diff --git a/.github/workflows/publish.pypi.yml b/.github/workflows/publish.pypi.yml index ef40de16f4..66c9ccfaac 100644 --- a/.github/workflows/publish.pypi.yml +++ b/.github/workflows/publish.pypi.yml @@ -3,6 +3,18 @@ on: release: types: [created] workflow_dispatch: + inputs: + package: + description: "Build all root packages or only the SDK" + type: choice + default: all + options: + - all + - sdk + publish: + description: "Publish built packages (false builds only)" + type: boolean + default: false permissions: contents: read @@ -25,10 +37,37 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: ๐Ÿ›ž Create Wheels - # Packaging tools are small; avoid restoring/saving the runtime pip cache. + env: + PACKAGE_SELECTION: ${{ github.event_name == 'workflow_dispatch' && inputs.package || 'all' }} run: | - make create_pypi_wheels - - name: ๐Ÿš€ Publish to PyPi + if [ "$PACKAGE_SELECTION" = "sdk" ]; then + python -m pip install "setuptools>=83.0.0" "wheel>=0.38.1,<=0.45.1" + make create_sdk_wheel + else + make create_pypi_wheels + fi + - name: ๐Ÿงน Separate SDK publication + # Keep SDK publication in this existing trusted workflow. Its isolated + # build can precede Workflows without resolving the consumer lock. + run: | + mkdir -p sdk-dist + mv dist/pypi/inference_sdk-*.whl sdk-dist/ + - name: ๐Ÿ“ค Upload built wheels + uses: actions/upload-artifact@v4 + with: + name: inference-wheels + path: | + dist/pypi/*.whl + sdk-dist/*.whl + - name: ๐Ÿš€ Publish SDK to PyPI + if: ${{ github.event_name == 'release' || inputs.publish == true }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: sdk-dist/ + # A dependency bootstrap may publish the SDK before the root release. + skip-existing: true + - name: ๐Ÿš€ Publish root packages to PyPI + if: ${{ github.event_name == 'release' || (inputs.publish == true && inputs.package != 'sdk') }} uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/pypi/ diff --git a/Makefile b/Makefile index e28c336e7c..a18adffec8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: style check_code_quality download_fonts create_workflows_wheel create_streamvision_wheel create_models_wheel create_isolation_wheels create_pypi_wheels +.PHONY: style check_code_quality download_fonts create_workflows_wheel create_streamvision_wheel create_models_wheel create_isolation_wheels create_pypi_wheels create_sdk_wheel serve: SKIP_CODEGEN=1 python -m zensical serve @@ -100,7 +100,7 @@ create_wheels: create_workflows_wheel create_models_wheel create_streamvision_wh rm -rf build/* python .release/pypi/inference.cli.setup.py bdist_wheel -create_wheels_for_gpu_notebook: create_workflows_wheel create_streamvision_wheel +create_wheels_for_gpu_notebook: create_workflows_wheel create_models_wheel create_streamvision_wheel python -m pip install --upgrade pip python -m pip install setuptools wheel twine requests rm -rf build/* @@ -112,6 +112,10 @@ create_wheels_for_gpu_notebook: create_workflows_wheel create_streamvision_wheel rm -rf build/* python .release/pypi/inference.cli.setup.py bdist_wheel +create_sdk_wheel: + rm -rf build/* + ${PYTHON} .release/pypi/inference.sdk.setup.py bdist_wheel --dist-dir dist/pypi + create_inference_cli_whl: ${PYTHON} -m pip install --upgrade pip ${PYTHON} -m pip install setuptools wheel requests diff --git a/docker/dockerfiles/Dockerfile.onnx.cpu b/docker/dockerfiles/Dockerfile.onnx.cpu index 475092c81c..d1cc8f3bd2 100644 --- a/docker/dockerfiles/Dockerfile.onnx.cpu +++ b/docker/dockerfiles/Dockerfile.onnx.cpu @@ -59,7 +59,7 @@ ARG TARGETPLATFORM WORKDIR /build COPY . . RUN make create_wheels -RUN pip3 install --find-links dist/ dist/roboflow_workflows*.whl dist/streamvision*.whl dist/inference_cli*.whl dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl "setuptools~=84.2.1" +RUN pip3 install --find-links dist/ dist/roboflow_workflows*.whl dist/inference_models*.whl dist/streamvision*.whl dist/inference_cli*.whl dist/inference_core*.whl dist/inference_cpu*.whl dist/inference_sdk*.whl "setuptools~=84.2.1" RUN pip3 install "inference-models[model-blob-cache]" RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ]; then pip3 install -r requirements/requirements.vino.txt; rm -rf ~/.cache/pip; fi diff --git a/docker/dockerfiles/Dockerfile.onnx.gpu b/docker/dockerfiles/Dockerfile.onnx.gpu index 95ab058982..629577034f 100644 --- a/docker/dockerfiles/Dockerfile.onnx.gpu +++ b/docker/dockerfiles/Dockerfile.onnx.gpu @@ -292,6 +292,7 @@ RUN ln -sf /usr/bin/python3 /usr/bin/python RUN /bin/make create_wheels_for_gpu_notebook && \ python3 -m pip install --no-cache-dir --find-links dist/ \ dist/roboflow_workflows*.whl \ + dist/inference_models*.whl \ dist/streamvision*.whl \ dist/inference_cli*.whl \ dist/inference_core*.whl \ diff --git a/docs/fine-tuned/rfdetr-obb.md b/docs/fine-tuned/rfdetr-obb.md new file mode 100644 index 0000000000..533dd13f04 --- /dev/null +++ b/docs/fine-tuned/rfdetr-obb.md @@ -0,0 +1,198 @@ +# Native RF-DETR oriented object detection + +This implementation adds the `oriented-object-detection` task for an exported +RF-DETR OBB model. It does not register the artifact as ordinary object detection. + +## Artifact and geometry contract + +The producer registers a standard RF-DETR ONNX package containing `weights.onnx`, +`class_names.txt`, and `inference_config.json`. The loader generates +`model_config.json` for its cache and offline loading. The registry identity is +architecture `rfdetr`, task `oriented-object-detection`, backend `onnx`, with model +variant `rfdetr-obb-preview-2xl`. The registry also accepts the producer's +`obb-detection` task alias; responses use `oriented-object-detection`. + +The model metadata response supplies `orientedBoxMetadata` with a `classHeading` +map and nonnegative `classHeadingRevision`, taken from the model's frozen dataset +version. This field is optional for other tasks and required by the native OBB +loader. It follows the model through cached and offline loads. The runtime does +not infer class heading capability from the graph's heading head. + +The graph emits raw `dets[B,Q,5]`, `labels[B,Q,C]`, and `heading[B,Q,4]` before +selection. `dets` contains normalized center and ordered side lengths, followed by +a clockwise-radian angle. `labels` contains class logits; output zero is background. +`class_names.txt` identifies output columns, and the package's standard +`class_names_operations` removes the background class. Response class IDs are +zero-based indices after that remapping, rather than COCO category IDs. Frozen +class heading policy is matched by class name. Zero-area padded query slots are +discarded before selection. Nonfinite graph values and missing heading outputs +are errors. + +For a directed class, heading side `k` adds `k*pi/2` to the raw angle and swaps +width/height iff `k` is odd. This preserves the exact footprint and designates the +predicted front. Undirected classes retain the raw ordered rectangle axis. +No independent front vector is persisted. + +The package declares letterbox resizing, RGB padding 114, division by 255 and +ImageNet normalization. For an image of width `w`, height `h` and square model +input size `S`, the runtime resizes to `round(w*S/max(w,h))` by +`round(h*S/max(w,h))`, with each dimension at least one pixel. Pillow bilinear +resizing and integer centered padding match the producer's training transform. + +Rounding can make the realized width and height ratios differ. The inverse maps +centers using their respective ratios and ordered sides using the mean of those +two ratios, matching the training-target transform; angles remain unchanged. +The producer's current postprocessing uses a continuous max-side inverse. +Comparison with that postprocessing and validation against trained weights remain +pending. Roboflow static crop, grayscale and contrast preprocessing from +`inference_config.json` runs before model normalization. The inverse includes the +static-crop offset. Returned geometry remains floating point in original +input-image coordinates. + +## Response and SDK + +`POST /infer/oriented_object_detection` accepts the shared image/confidence request +fields. Each prediction contains `obb: [cx, cy, width, height, theta]`, `class_id`, +`class`, `confidence`, `heading`, and `detection_id`. The response includes +`task_type: "oriented-object-detection"` and frozen `class_heading` semantics. +Detection IDs use a fresh namespace for each input result, so repeated calls and +separate models produce distinct IDs. IDs remain stable through Workflow transport. +Bounding AABBs and corners are compatibility views derived from native geometry. + +The current Inference SDK in API v1 mode routes the native task to this endpoint. It disables +client resizing for this task so a second rounded resize cannot alter the native +shape before the server's declared preprocessing. Select API v1 explicitly when connecting to a hosted URL that otherwise defaults to the legacy API v0 client; API v0 client downsizing is not part of the advertised native path. + +## Workflows + +`roboflow_core/roboflow_oriented_object_detection_model@v1` produces the separate +`oriented_object_detection_prediction` kind. Numeric confidence, class filtering, +row selection, row combination, dynamic crop offsets, original-coordinate output, +serialization, and `roboflow_core/oriented_box_visualization@v1` retain the ordered +footprint and class front. The visualization highlights the native front edge for +directed classes. NumPy and tensor carriers retain the same semantic fields. + +Inside Workflows, ordered corners carry geometry/front through coordinate changes. +A general affine/perspective warp can produce a quadrilateral: serialization then +emits `geometry_type: "quadrilateral"` and ordered `corners`, without inventing an +OBB fit. Such results require an explicit rectangle-fit policy before use as native +rectangle annotations. Isotropic scaling and crop offsets preserve native tuples +without integer rounding. + +The existing Detections Merge block fits a union AABB and is not compatible with +this native kind. Row combination concatenates detections without refitting them. +Query-language `DetectionsShift` and `DetectionsOffset` are also excluded and reject +native inputs at execution, because their geometry operations only update AABBs. +Opt-in `roboflow_core/detections_consensus@v2` compares native footprints using +rotated IoU and retains the highest-confidence contributor of the selected class, +including its ordered geometry, front and identity. It reuses v1 voting and class +selection, with configured confidence aggregation. Its geometry overlap code ports +Agis Kounelis's [PR #2420](https://github.com/roboflow/inference/pull/2420) at +`ee7fe5288255df1c8401877feda3ddaca0118eef`. Frozen class policies survive consensus +and combination, including empty outputs; incompatible policies or mixed tasks +raise errors. Ordinary tensor inputs remain unsupported by v2; v1 retains its +existing behavior. +Dataset writeback and annotation acceptance need the platform's native adapters and +workspace admission checks; ordinary AABB upload paths are not advertised for OBB. + +## Validation and deployment matrix + +| Target | Implementation and validation | +| --- | --- | +| Local CPU ONNX model package | Implemented; synthetic three-output ONNX fixtures use the standard producer package files and frozen class metadata; pinned producer letterbox pixels and target geometry tested across five image shapes | +| Server request consumer | Implemented; metadata retrieval, AutoModel loading, cache/offline reloads, native request lifecycle and original-coordinate response tested locally with synthetic fixtures | +| Current Inference SDK | Native route exercised through the request adapter and SDK with a synthetic ONNX fixture; its response passes native preview and evaluation parsers; no hosted request executed | +| Local Workflow model, transport and visualization | Implemented; NumPy/tensor semantic and coordinate tests | +| Query-language `DetectionsShift` / `DetectionsOffset` | Excluded; native NumPy/tensor inputs reject before geometry mutation | +| Hosted production serving | Requires compatible package releases, trained artifact registration and serving deployment; no hosted/cloud run claimed | +| CUDA, TensorRT, CoreML, browser inferencejs | Excluded from this initial backend registration | +| Legacy `roboflow-python` `version.model` | Excluded; current Inference SDK support does not add legacy task dispatch | + +Synthetic fixtures establish package and transport compatibility; they do not +establish trained-model accuracy or an end-to-end platform training run. No model +package release, cloud/GPU run, deployment or feature activation is part of these +local validation results. Existing authorized model reads remain independent +of the platform's default-off gate for new OBB authoring/jobs. + +## Package and worker release gate + +This feature is pending release. The registry versions below are forward +requirements, not evidence of a published or deployed native provider: + +| Distribution | Required version | Consumer | +| --- | --- | --- | +| `inference-sdk` | `1.8.4` | Remote native task dispatch in Workflows and model evaluation | +| `inference-models` | `0.39.8` | Native RF-DETR ONNX registry/runtime | +| `roboflow-workflows` | `0.2.16` | Canonical native response entities, blocks and consensus v2 | +| `inference`, `inference-cpu`, `inference-gpu`, `inference-core` | `1.8.4` | HTTP server and model adapters | + +### Build validation before release + +Build candidate packages from the same reviewed checkout: + +```bash +make create_sdk_wheel +(cd inference_models && uv build) +make download_fonts +(cd workflows && uv build --wheel --out-dir ../dist) +``` + +These commands produce local distributions without publishing them. Same-checkout +wheels can validate source compatibility and package isolation in a disposable +environment. They do not establish registry availability or make a committed +registry lock valid. The existing SDK and Models workflows also support manual +build-only validation; publishing is a separate release operation. + +### Dependency order and release gates + +SDK and Models changes must merge before their normal, separately approved +releases. Consumer pins and registry locks require those published dependencies. +With the existing registry-lock gates, this requires a merge boundary between the +SDK/Models changes and the Workflows/server consumer changes; the combined change +cannot satisfy that order by publishing from an unmerged branch. This dependency +order is required for release; no package publication or deployment is established +by the local build checks above. + +1. Merge the SDK and Models changes, then release SDK `1.8.4` and Models `0.39.8` + through their existing release workflows after separate release approval. + Verify the registry wheels' metadata and contents against the merged source. +2. Once those registry releases are verified, update the consumer pins and run + `cd workflows && uv lock`. Verify the genuine generated registry lock and a + fresh constrained installation before merging the consumer changes. Use the + same-checkout Workflows wheel when testing its unpublished version; that + installed-wheel check is separate from the registry-lock check. Update the + Streamvision registry lock if its dependency graph requires it, and verify + which Workflows release it actually selects. + The Models project lock must also pass + `cd inference_models && uv lock --check`. Local wheel overrides, fabricated + hashes, and vendored wheels do not satisfy the registry-lock gate. +3. After the consumer changes merge, release Workflows `0.2.16` through its + existing workflow. Verify the registry wheel contains + `core_steps/common/oriented_entities.py`, and verify clean ordinary server + imports with the released dependency set. Workflows must be available before + root distributions requiring its exact version become consumable. +4. Release the root `1.8.4` packages after their consumer changes merge and build + server images. `make create_wheels` builds the root release set, including the + SDK; the root workflow handles the SDK upload separately. Full CPU/GPU builds + explicitly install their checkout Models wheel; slim builds resolve the new + registry pin. Verify installed model registration and a CPU ONNX fixture in + actual deployed images and US/EU targets. +5. Deploy compatible serving and queue workers before enabling platform native + producers. Native custom Auto Label jobs use direct HTTP and require no queue + SDK/Supervision upgrade. Unavailable endpoints or non-native responses fail the + job without an ordinary detection fallback. Native Workflow Auto Label + admission is a separate unsupported path. + +Before publication, a PyPI project owner must verify the project's Trusted +Publisher settings. The expected repository is `roboflow/inference`, with +`publish.pypi.yml` for SDK/root distributions, +`publish.pypi.inference_exp.yml` for Models, and +`publish.pypi.roboflow_workflows.yml` for Workflows. The workflow filename is part +of the trusted identity, and any configured environment or protection must match. +Private publisher settings have not been verified by the local checks. See +[PyPI's publisher setup](https://docs.pypi.org/trusted-publishers/adding-a-publisher/). + +Python Supervision's native parser/NMM feature is independent. Its development +version is outside the platform's existing `<0.31.0` cap; the server/Workflows +native carrier constructs compatible `Detections` directly and does not require +that upgrade. Existing ordinary/legacy task discriminators remain valid. diff --git a/inference/core/entities/requests/inference.py b/inference/core/entities/requests/inference.py index 5d1958bfce..9432040eb1 100644 --- a/inference/core/entities/requests/inference.py +++ b/inference/core/entities/requests/inference.py @@ -431,16 +431,44 @@ class LMMInferenceRequest(CVInferenceRequest): def request_from_type(model_type, request_dict): - """Uses original request id""" + """Reconstruct a task-specific request, preserving its identifier. + + Args: + model_type: Canonical model task type. + request_dict: Serialized inference request. + + Returns: + The request model for the specified task. + + Raises: + ValueError: If the task type is unsupported. + """ if model_type == "classification": request = ClassificationInferenceRequest(**request_dict) elif model_type == "instance-segmentation": request = InstanceSegmentationInferenceRequest(**request_dict) elif model_type == "object-detection": request = ObjectDetectionInferenceRequest(**request_dict) + elif model_type == "oriented-object-detection": + request = OrientedObjectDetectionInferenceRequest(**request_dict) elif model_type == "semantic-segmentation": request = SemanticSegmentationInferenceRequest(**request_dict) else: raise ValueError(f"Unknown task type {model_type}") request.id = request_dict.get("id", request.id) return request + + +class OrientedObjectDetectionInferenceRequest(ObjectDetectionInferenceRequest): + """Run a native OBB model using numeric confidence and shared image fields.""" + + confidence: float = Field( + default=0.5, + ge=0, + le=1, + description="Minimum sigmoid class confidence for native OBB.", + ) + disable_active_learning: bool = Field( + default=True, + description="Native OBB writeback requires a compatible platform adapter.", + ) diff --git a/inference/core/entities/responses/inference.py b/inference/core/entities/responses/inference.py index 772813042d..de968995bc 100644 --- a/inference/core/entities/responses/inference.py +++ b/inference/core/entities/responses/inference.py @@ -4,6 +4,10 @@ from uuid import uuid4 from pydantic import BaseModel, Field, ValidationError, field_serializer +from roboflow_workflows.core_steps.common.oriented_entities import ( + OrientedObjectDetectionInferenceResponse, + OrientedObjectDetectionPrediction, +) from inference.core.workflows.core_steps.common.inference_response_entities import ( # noqa: F401 CvInferenceResponse, @@ -454,6 +458,8 @@ def response_from_type(model_type, response_dict): return InstanceSegmentationInferenceResponse(**response_dict) elif model_type == "semantic-segmentation": return SemanticSegmentationInferenceResponse(**response_dict) + elif model_type == "oriented-object-detection": + return OrientedObjectDetectionInferenceResponse(**response_dict) elif model_type == "object-detection": return ObjectDetectionInferenceResponse(**response_dict) else: diff --git a/inference/core/interfaces/http/http_api.py b/inference/core/interfaces/http/http_api.py index a709bd650f..791e7d33c3 100644 --- a/inference/core/interfaces/http/http_api.py +++ b/inference/core/interfaces/http/http_api.py @@ -69,6 +69,7 @@ KeypointsDetectionInferenceRequest, LMMInferenceRequest, ObjectDetectionInferenceRequest, + OrientedObjectDetectionInferenceRequest, SemanticSegmentationInferenceRequest, ensure_wire_safe_mask_format, ) @@ -123,6 +124,7 @@ LMMInferenceResponse, MultiLabelClassificationInferenceResponse, ObjectDetectionInferenceResponse, + OrientedObjectDetectionInferenceResponse, SemanticSegmentationInferenceResponse, StubResponse, ) @@ -2227,6 +2229,27 @@ def infer_embeddings( service_secret=service_secret, ) + @app.post( + "/infer/oriented_object_detection", + response_model=Union[ + OrientedObjectDetectionInferenceResponse, + List[OrientedObjectDetectionInferenceResponse], + ], + summary="Native oriented object detection", + ) + @with_route_exceptions + @usage_collector("request") + def infer_oriented_objects( + inference_request: OrientedObjectDetectionInferenceRequest, + countinference: Optional[bool] = None, + service_secret: Optional[str] = None, + ): + return process_inference_request( + inference_request, + countinference=countinference, + service_secret=service_secret, + ) + @app.post( "/infer/keypoints_detection", response_model=Union[KeypointsDetectionInferenceResponse, StubResponse], @@ -5102,6 +5125,8 @@ def legacy_infer_from_request( elif task_type == "classification": inference_request_type = ClassificationInferenceRequest args = {"include_anomaly_map": include_anomaly_map} + elif task_type == "oriented-object-detection": + inference_request_type = OrientedObjectDetectionInferenceRequest elif task_type == "keypoint-detection": inference_request_type = KeypointsDetectionInferenceRequest args = {"keypoint_confidence": keypoint_confidence} @@ -5140,7 +5165,7 @@ def legacy_infer_from_request( inference_response = self.model_manager.infer_from_request_sync( inference_request.model_id, inference_request, - active_learning_eligible=True, + active_learning_eligible=task_type != "oriented-object-detection", background_tasks=background_tasks, ) logger.debug("Response ready.") diff --git a/inference/core/interfaces/workflows_models_provider.py b/inference/core/interfaces/workflows_models_provider.py index c40c24fef1..a10f33c3bf 100644 --- a/inference/core/interfaces/workflows_models_provider.py +++ b/inference/core/interfaces/workflows_models_provider.py @@ -46,6 +46,7 @@ KeypointsDetectionInferenceRequest, LMMInferenceRequest, ObjectDetectionInferenceRequest, + OrientedObjectDetectionInferenceRequest, SemanticSegmentationInferenceRequest, ) from inference.core.entities.requests.moondream2 import Moondream2InferenceRequest @@ -150,6 +151,41 @@ def run_object_detection( ) return self._dump(self._infer(model_id=model_id, request=request)) + def run_oriented_object_detection( + self, + model_id, + images, + api_key=None, + confidence=0.5, + max_detections=300, + class_filter=None, + ): + """Run native OBB inference for a Workflow model block. + + Args: + model_id: Registered native model. + images: Input images or crop references. + api_key: Model access credential. + confidence: Numeric class threshold. + max_detections: Per-image detection cap. + class_filter: Optional accepted names. + + Returns: + Native response dictionaries. + """ + request = OrientedObjectDetectionInferenceRequest( + api_key=api_key, + model_id=model_id, + image=images, + confidence=confidence, + max_detections=max_detections, + class_filter=class_filter, + source=_WORKFLOW_SOURCE, + disable_active_learning=True, + ) + responses = self._dump(self._infer(model_id=model_id, request=request)) + return responses + def run_classification( self, model_id: str, diff --git a/inference/core/models/inference_models_adapters.py b/inference/core/models/inference_models_adapters.py index a07ee1d88f..f2eb419451 100644 --- a/inference/core/models/inference_models_adapters.py +++ b/inference/core/models/inference_models_adapters.py @@ -2603,3 +2603,104 @@ def _as_action_recognition_model(model: Any, model_id: str) -> ActionRecognition if isinstance(model, Cosmos3EdgeReasoner): return Cosmos3EdgeActionRecognition.from_reasoner(reasoner=model) raise ValueError(f"Model {model_id} does not support action recognition.") + + +class InferenceModelsOrientedObjectDetectionAdapter( + InferenceModelsObjectDetectionAdapter +): + """Serve native OBB packages through the shared image loading and request lifecycle.""" + + def __init__(self, model_id: str, api_key: str = None, **kwargs): + super().__init__(model_id=model_id, api_key=api_key, **kwargs) + self.task_type = "oriented-object-detection" + self.img_size_h = self.img_size_w = self._model.resolution + + def postprocess(self, predictions, preprocess_return_metadata, **kwargs): + """Serialize native tuples without converting them into axis-aligned boxes. + + Args: + predictions: Raw three-output ONNX predictions. + preprocess_return_metadata: Realized letterbox metadata. + **kwargs: Confidence, cap and optional class filter. + + Returns: + Native response for each input image. + """ + from inference.core.entities.responses.inference import ( + OrientedObjectDetectionInferenceResponse, + OrientedObjectDetectionPrediction, + ) + + results = self._model.post_process( + predictions, preprocess_return_metadata, **kwargs + ) + responses = [] + for result, meta in zip(results, preprocess_return_metadata): + native = [] + for obb, confidence, class_id, heading, detection_id, name in zip( + result.obb.tolist(), + result.confidence.tolist(), + result.class_id.tolist(), + result.heading.tolist(), + result.detection_id, + result.class_names, + ): + if kwargs.get("class_filter") and name not in kwargs["class_filter"]: + continue + native.append( + OrientedObjectDetectionPrediction( + obb=obb, + confidence=confidence, + class_id=class_id, + heading=heading, + detection_id=detection_id, + **{"class": name}, + ) + ) + responses.append( + OrientedObjectDetectionInferenceResponse( + predictions=native, + image=InferenceResponseImage( + width=meta.original_size.width, height=meta.original_size.height + ), + class_heading=self._model.metadata.classHeading, + ) + ) + return responses + + def draw_predictions(self, inference_request, inference_response): + """Render the native footprint and front on the request image. + + Args: + inference_request: Image request. + inference_response: Native prediction response. + + Returns: + JPEG-encoded visualization bytes. + """ + import math + + image = Image.fromarray(load_image_rgb(inference_request.image)) + draw = ImageDraw.Draw(image) + for prediction in inference_response.predictions: + cx, cy, width, height, angle = prediction.obb + c, s = math.cos(angle), math.sin(angle) + corners = [ + (cx + x * c - y * s, cy + x * s + y * c) + for x, y in ( + (-width / 2, -height / 2), + (width / 2, -height / 2), + (width / 2, height / 2), + (-width / 2, height / 2), + ) + ] + draw.line(corners + [corners[0]], fill="#A351FB", width=2) + if prediction.heading: + draw.line( + [(cx, cy), (cx + c * width / 2, cy + s * width / 2)], + fill="#FFB633", + width=3, + ) + output = BytesIO() + image.save(output, format="JPEG") + return output.getvalue() diff --git a/inference/core/version.py b/inference/core/version.py index 407e16e906..91c731ee61 100644 --- a/inference/core/version.py +++ b/inference/core/version.py @@ -1,4 +1,4 @@ -__version__ = "1.8.3" +__version__ = "1.8.4" if __name__ == "__main__": diff --git a/inference/models/utils.py b/inference/models/utils.py index 0efbb4f47c..8b71a50578 100644 --- a/inference/models/utils.py +++ b/inference/models/utils.py @@ -1425,6 +1425,18 @@ def get_roboflow_model(*args, **kwargs): InferenceModelsDepthEstimationAdapter ) + InferenceModelsOrientedObjectDetectionAdapter = _LazyModelClass( + "inference.core.models.inference_models_adapters:" + "InferenceModelsOrientedObjectDetectionAdapter" + ) + + ROBOFLOW_MODEL_TYPES[("oriented-object-detection", "rfdetr-obb-preview-2xl")] = ( + InferenceModelsOrientedObjectDetectionAdapter + ) + ROBOFLOW_MODEL_TYPES[("oriented-object-detection", "rfdetr")] = ( + InferenceModelsOrientedObjectDetectionAdapter + ) + # RFDETR keypoint detection is inference_models-only (no legacy implementation), # so we add entries directly rather than swapping existing ones. ROBOFLOW_MODEL_TYPES[("keypoint-detection", "rfdetr-keypoint-preview")] = ( diff --git a/inference_models/inference_models/models/auto_loaders/auto_resolution_cache.py b/inference_models/inference_models/models/auto_loaders/auto_resolution_cache.py index f89f00d796..6dc2887dc9 100644 --- a/inference_models/inference_models/models/auto_loaders/auto_resolution_cache.py +++ b/inference_models/inference_models/models/auto_loaders/auto_resolution_cache.py @@ -21,6 +21,9 @@ ModelArchitecture, TaskType, ) +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) from inference_models.utils.file_system import read_json from inference_models.weights_providers.entities import ( ModelDependency, @@ -49,6 +52,7 @@ class AutoResolutionCacheEntry(BaseModel): created_at: datetime model_features: Optional[dict] = Field(default=None) recommended_parameters: Optional[RecommendedParameters] = Field(default=None) + oriented_box_metadata: Optional[OrientedBoxMetadata] = Field(default=None) trusted_source: Optional[bool] = Field(default=None) diff --git a/inference_models/inference_models/models/auto_loaders/core.py b/inference_models/inference_models/models/auto_loaders/core.py index 63aebdf88b..4b9ed85576 100644 --- a/inference_models/inference_models/models/auto_loaders/core.py +++ b/inference_models/inference_models/models/auto_loaders/core.py @@ -114,6 +114,9 @@ ) from inference_models.models.auto_loaders.ranking import rank_model_packages from inference_models.models.base.image_embeddings import ImageEmbeddingModel +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) from inference_models.runtime_introspection.core import x_ray_runtime_environment from inference_models.utils.content_addressed_artifact_cache import ( ContentAddressedArtifactCache, @@ -2288,6 +2291,7 @@ def _warm_up_point_model_directory(model_dir: str) -> None: model_dependencies_instances=model_dependencies_instances, model_dependencies_directories=model_dependencies_directories, recommended_parameters=model_metadata.recommended_parameters, + oriented_box_metadata=model_metadata.oriented_box_metadata, max_package_loading_attempts=max_package_loading_attempts, model_download_file_lock_acquire_timeout=model_download_file_lock_acquire_timeout, verify_hash_while_download=verify_hash_while_download, @@ -2630,6 +2634,14 @@ def dependency_directory_pointer(model_dir: str) -> None: model_init_kwargs["recommended_parameters"] = ( cache_entry.recommended_parameters ) + if cache_entry.oriented_box_metadata != package_config.oriented_box_metadata: + raise CorruptedModelPackageError( + message="Cached oriented-box semantics differ from package manifest." + ) + if cache_entry.oriented_box_metadata is not None: + model_init_kwargs["oriented_box_metadata"] = ( + cache_entry.oriented_box_metadata + ) model = load_model_with_capabilities( model_class, model_package_cache_dir, @@ -2887,6 +2899,7 @@ def attempt_loading_matching_model_packages( point_model_directory: Optional[Callable[[str], None]] = None, requested_model_id: Optional[str] = None, content_addressed_artifact_cache: Optional[ContentAddressedArtifactCache] = None, + oriented_box_metadata: Optional[OrientedBoxMetadata] = None, ) -> AnyModel: if requested_model_id is None: requested_model_id = model_id @@ -2924,6 +2937,7 @@ def attempt_loading_matching_model_packages( model_dependencies_instances=model_dependencies_instances, model_dependencies_directories=model_dependencies_directories, recommended_parameters=recommended_parameters, + oriented_box_metadata=oriented_box_metadata, verify_hash_while_download=verify_hash_while_download, download_files_without_hash=download_files_without_hash, content_addressed_artifact_cache=content_addressed_artifact_cache, @@ -3117,6 +3131,7 @@ def initialize_model( requested_model_id: Optional[str] = None, api_key: Optional[str] = None, content_addressed_artifact_cache: Optional[ContentAddressedArtifactCache] = None, + oriented_box_metadata: Optional[OrientedBoxMetadata] = None, ) -> Tuple[AnyModel, str]: if not isinstance(model_id, str) or not model_id.strip(): raise CorruptedModelPackageError( @@ -3190,6 +3205,10 @@ def initialize_model( "dynamic_batch_size_supported": (model_package.dynamic_batch_size_supported), "static_batch_size": model_package.static_batch_size, } + if oriented_box_metadata is not None: + expected_manifest_fields["oriented_box_metadata"] = ( + oriented_box_metadata.model_dump(mode="json") + ) if model_package.package_source == PackageSourceType.LOCAL_CACHE: package_path_for_lock = resolve_existing_model_package_cache_path( model_id=cache_model_id, @@ -3256,6 +3275,7 @@ def initialize_model( model_dependencies_instances=model_dependencies_instances, model_dependencies_directories=model_dependencies_directories, recommended_parameters=recommended_parameters, + oriented_box_metadata=oriented_box_metadata, model_download_file_lock_acquire_timeout=model_download_file_lock_acquire_timeout, verify_hash_while_download=verify_hash_while_download, download_files_without_hash=download_files_without_hash, @@ -3436,6 +3456,14 @@ def initialize_model( model_init_kwargs[MODEL_DEPENDENCIES_KEY] = model_dependencies_instances if resolved_recommended_parameters is not None: model_init_kwargs["recommended_parameters"] = resolved_recommended_parameters + if oriented_box_metadata is not None: + if OFFLINE_MODE: + cached_config = parse_model_config(config_path=config_path) + if cached_config.oriented_box_metadata != oriented_box_metadata: + raise CorruptedModelPackageError( + message="Offline oriented-box semantics differ from package manifest." + ) + model_init_kwargs["oriented_box_metadata"] = oriented_box_metadata model = load_model_with_capabilities( model_class, model_package_cache_dir, @@ -3471,6 +3499,7 @@ def initialize_model( trusted_source=model_package.trusted_source, model_dependencies=model_dependencies_manifest, recommended_parameters=recommended_parameters_manifest, + oriented_box_metadata=oriented_box_metadata, quantization=quantization, dynamic_batch_size_supported=model_package.dynamic_batch_size_supported, static_batch_size=model_package.static_batch_size, @@ -3498,6 +3527,7 @@ def initialize_model( model_dependencies=model_dependencies, model_features=model_package.model_features, recommended_parameters=resolved_recommended_parameters, + oriented_box_metadata=oriented_box_metadata, trusted_source=model_package.trusted_source, api_key=api_key, ) @@ -3584,6 +3614,7 @@ def dump_model_config_for_offline_use( canonical_model_id: Optional[str] = None, package_artifacts: Optional[List[dict]] = None, dependency_package_paths: Optional[List[dict]] = None, + oriented_box_metadata: Optional[OrientedBoxMetadata] = None, ) -> str: """Persist a versioned manifest used for safe offline package loading.""" if not isinstance(model_id, str) or not model_id.strip(): @@ -3624,6 +3655,10 @@ def dump_model_config_for_offline_use( "dependency_package_paths": dependency_package_paths, "model_id": model_id, } + if oriented_box_metadata is not None: + published_fields["oriented_box_metadata"] = oriented_box_metadata.model_dump( + mode="json" + ) target_file_dir, target_file_name = os.path.split(config_path) target_file_dir = target_file_dir or "." lock_path = os.path.join(target_file_dir, f".{target_file_name}.lock") @@ -3833,6 +3868,7 @@ def dump_auto_resolution_cache( trusted_source: Optional[bool] = None, canonical_model_id: Optional[str] = None, api_key: Optional[str] = None, + oriented_box_metadata: Optional[OrientedBoxMetadata] = None, ) -> None: if not use_auto_resolution_cache: return None @@ -3866,6 +3902,7 @@ def dump_auto_resolution_cache( model_dependencies=model_dependencies, model_features=model_features, recommended_parameters=recommended_parameters, + oriented_box_metadata=oriented_box_metadata, trusted_source=trusted_source, ) auto_resolution_cache.register( @@ -4111,6 +4148,12 @@ def parse_model_config(config_path: str) -> InferenceModelConfig: model_features = raw_config.get("model_features") model_dependencies = raw_config.get("model_dependencies") recommended_parameters = raw_config.get("recommended_parameters") + raw_oriented_box_metadata = raw_config.get("oriented_box_metadata") + oriented_box_metadata = ( + OrientedBoxMetadata.model_validate(raw_oriented_box_metadata) + if raw_oriented_box_metadata is not None + else None + ) trusted_source = raw_config.get("trusted_source") package_artifacts = raw_config.get("package_artifacts") dependency_package_paths = raw_config.get("dependency_package_paths") @@ -4204,6 +4247,7 @@ def parse_model_config(config_path: str) -> InferenceModelConfig: trusted_source=trusted_source, model_dependencies=model_dependencies, recommended_parameters=recommended_parameters, + oriented_box_metadata=oriented_box_metadata, quantization=raw_config.get("quantization"), dynamic_batch_size_supported=dynamic_batch_size_supported, static_batch_size=static_batch_size, @@ -4229,6 +4273,8 @@ def load_library_model_from_local_dir( set(model_config.model_features) if model_config.model_features else None ), ) + if model_config.oriented_box_metadata is not None: + model_init_kwargs["oriented_box_metadata"] = model_config.oriented_box_metadata return load_model_with_capabilities( model_class, model_dir, diff --git a/inference_models/inference_models/models/auto_loaders/entities.py b/inference_models/inference_models/models/auto_loaders/entities.py index 07699d5a58..5dbacb49d2 100644 --- a/inference_models/inference_models/models/auto_loaders/entities.py +++ b/inference_models/inference_models/models/auto_loaders/entities.py @@ -20,6 +20,9 @@ OpenVocabularyObjectDetectionModel, ) from inference_models.models.base.semantic_segmentation import SemanticSegmentationModel +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) ModelArchitecture = str TaskType = Optional[str] @@ -95,6 +98,7 @@ class InferenceModelConfig: # computed while reading the manifest so auto-resolution metadata can bind # every load-driving field, including custom model_module/model_class keys. manifest_content_hash: Optional[str] = None + oriented_box_metadata: Optional[OrientedBoxMetadata] = None def is_library_model(self) -> bool: return self.model_architecture is not None and self.backend_type is not None diff --git a/inference_models/inference_models/models/auto_loaders/models_registry.py b/inference_models/inference_models/models/auto_loaders/models_registry.py index 34d4799590..e4b0895116 100644 --- a/inference_models/inference_models/models/auto_loaders/models_registry.py +++ b/inference_models/inference_models/models/auto_loaders/models_registry.py @@ -10,6 +10,7 @@ from inference_models.utils.imports import LazyClass OBJECT_DETECTION_TASK = "object-detection" +ORIENTED_OBJECT_DETECTION_TASK = "oriented-object-detection" INSTANCE_SEGMENTATION_TASK = "instance-segmentation" SEMANTIC_SEGMENTATION_TASK = "semantic-segmentation" KEYPOINT_DETECTION_TASK = "keypoint-detection" @@ -383,6 +384,14 @@ class RegistryEntry: module_name="inference_models.models.perception_encoder.perception_encoder_pytorch", class_name="PerceptionEncoderTorch", ), + ("rfdetr", ORIENTED_OBJECT_DETECTION_TASK, BackendType.ONNX): LazyClass( + module_name="inference_models.models.rfdetr.rfdetr_oriented_object_detection_onnx", + class_name="RFDetrForOrientedObjectDetectionONNX", + ), + ("rfdetr", "obb-detection", BackendType.ONNX): LazyClass( + module_name="inference_models.models.rfdetr.rfdetr_oriented_object_detection_onnx", + class_name="RFDetrForOrientedObjectDetectionONNX", + ), ("rfdetr", OBJECT_DETECTION_TASK, BackendType.TRT): RegistryEntry( model_class=LazyClass( module_name="inference_models.models.rfdetr.rfdetr_object_detection_trt", diff --git a/inference_models/inference_models/models/common/roboflow/oriented_box_metadata.py b/inference_models/inference_models/models/common/roboflow/oriented_box_metadata.py new file mode 100644 index 0000000000..5b024fb6f4 --- /dev/null +++ b/inference_models/inference_models/models/common/roboflow/oriented_box_metadata.py @@ -0,0 +1,14 @@ +"""Frozen dataset semantics supplied by Core for oriented-box models.""" + +from pydantic import BaseModel, Field, StrictBool + + +class OrientedBoxMetadata(BaseModel): + """Preserve class front policy independently of a model's heading head.""" + + classHeading: dict[str, StrictBool] = Field( + description="Frozen generated-version class heading policy." + ) + classHeadingRevision: int = Field( + strict=True, ge=0, description="Frozen generated-version policy revision." + ) diff --git a/inference_models/inference_models/models/rfdetr/rfdetr_oriented_object_detection_onnx.py b/inference_models/inference_models/models/rfdetr/rfdetr_oriented_object_detection_onnx.py new file mode 100644 index 0000000000..2c495a1e1c --- /dev/null +++ b/inference_models/inference_models/models/rfdetr/rfdetr_oriented_object_detection_onnx.py @@ -0,0 +1,407 @@ +"""Native RF-DETR OBB ONNX consumer. Angles are clockwise image-space radians.""" + +from dataclasses import dataclass, field +from typing import List +from uuid import uuid4 + +import numpy as np +import torch +from PIL import Image + +from inference_models.entities import ImageDimensions +from inference_models.models.base.object_detection import ( + Detections, + ObjectDetectionModel, +) +from inference_models.models.common.model_packages import get_model_package_contents +from inference_models.models.common.onnx import create_onnx_inference_session +from inference_models.models.common.roboflow.model_packages import ( + ColorMode, + ImagePreProcessing, + ResizeMode, + align_training_input_size_with_model, + ensure_input_size_within_limit, + parse_class_names_file, + parse_inference_config, +) +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) +from inference_models.models.common.roboflow.pre_processing import ( + apply_pre_processing_to_numpy_image, +) +from inference_models.models.rfdetr.class_remapping import prepare_class_remapping + + +@dataclass +class OrientedDetections(Detections): + """Float native boxes; compatibility bounds and corners are derived from obb.""" + + obb: torch.Tensor = field(kw_only=True) + heading: torch.Tensor = field(kw_only=True) + detection_id: List[str] = field(kw_only=True) + class_names: List[str] = field(kw_only=True) + + def to_supervision(self): + """Convert to native Workflow transport without dropping front. + + Returns: + Supervision detections with canonical OBB and derived ordered corners. + """ + import supervision as sv + + result = sv.Detections( + xyxy=self.xyxy.detach().cpu().numpy(), + confidence=self.confidence.detach().cpu().numpy(), + class_id=self.class_id.detach().cpu().numpy(), + data={ + "xyxyxyxy": _corners(self.obb).detach().cpu().numpy(), + "heading": self.heading.detach().cpu().numpy(), + "native_obb": np.ones(len(self), dtype=bool), + "detection_id": np.asarray(self.detection_id), + "class_name": np.asarray(self.class_names), + }, + metadata=self.image_metadata or {}, + ) + return result + + +def _corners(obb): + signs = obb.new_tensor([[-1, -1], [1, -1], [1, 1], [-1, 1]]) + local = signs[None] * obb[:, None, 2:4] / 2 + c, s = obb[:, 4].cos(), obb[:, 4].sin() + x = local[:, :, 0] * c[:, None] - local[:, :, 1] * s[:, None] + y = local[:, :, 0] * s[:, None] + local[:, :, 1] * c[:, None] + return torch.stack((x, y), dim=-1) + obb[:, None, :2] + + +@dataclass(frozen=True) +class OrientedPreprocessingMetadata: + """Realized integer letterbox transform used by the trained model.""" + + original_size: ImageDimensions + scale_width: float + scale_height: float + pad_left: float + pad_top: float + inference_size: ImageDimensions + crop_offset_x: int = 0 + crop_offset_y: int = 0 + + +class RFDetrForOrientedObjectDetectionONNX(ObjectDetectionModel): + """Run the three-output native OBB artifact on the ONNX CPU execution provider. + + CPU ONNX is the advertised initial backend. Other backends require their own + validation before registration. Model preprocessing is the training Letterbox: + rounded max-side resize, integer centered padding and unchanged angle. + """ + + @classmethod + def from_pretrained(cls, model_name_or_path: str, **kwargs): + """Load a standard RF-DETR package with Core-owned frozen class semantics. + + Args: + model_name_or_path: Local package directory with standard RF-DETR ONNX files. + **kwargs: Shared loader options; onnx_execution_providers can select CPU only. + + Returns: + Initialized native model. + + Raises: + ValueError: For unsupported backend, malformed outputs or missing metadata. + """ + raw_metadata = kwargs.get("oriented_box_metadata") + if raw_metadata is None: + raise ValueError("Native OBB requires frozen orientedBoxMetadata from Core") + metadata = OrientedBoxMetadata.model_validate(raw_metadata) + providers = kwargs.get("onnx_execution_providers") or ["CPUExecutionProvider"] + if providers != ["CPUExecutionProvider"]: + raise ValueError("Native OBB currently supports CPUExecutionProvider only") + package = get_model_package_contents( + model_package_dir=model_name_or_path, + elements=["class_names.txt", "inference_config.json", "weights.onnx"], + ) + class_names = parse_class_names_file(package["class_names.txt"]) + maximum = kwargs.get("rf_detr_max_input_resolution") + config = parse_inference_config( + config_path=package["inference_config.json"], + allowed_resize_modes={ResizeMode.LETTERBOX}, + max_allowed_input_size=maximum, + ) + indices = list(range(len(class_names))) + if config.class_names_operations: + class_names, remapping = prepare_class_remapping( + class_names=class_names, + class_names_operations=config.class_names_operations, + device=torch.device("cpu"), + ) + indices = remapping.remaining_class_ids.tolist() + if not class_names or any( + name not in metadata.classHeading for name in class_names + ): + raise ValueError( + "Native OBB class names disagree with frozen class heading policy" + ) + session = create_onnx_inference_session( + model_path=package["weights.onnx"], providers=providers + ) + model = cls( + session=session, + metadata=metadata, + class_names=class_names, + output_indices=indices, + image_pre_processing=config.image_pre_processing, + ) + config = align_training_input_size_with_model( + config, + model_input_height=model.resolution, + model_input_width=model.resolution, + ) + ensure_input_size_within_limit( + inference_config=config, max_allowed_input_size=maximum + ) + network = config.network_input + if ( + network.color_mode != ColorMode.RGB + or network.input_channels != 3 + or network.padding_value != 114 + or network.scaling_factor != 255 + or network.normalization != ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) + ): + raise ValueError("Native OBB package preprocessing differs from training") + return model + + def __init__( + self, + *, + session, + metadata: OrientedBoxMetadata, + class_names: List[str], + output_indices: List[int], + image_pre_processing: ImagePreProcessing, + ): + self._session = session + self.metadata = metadata + self.image_pre_processing = image_pre_processing + inputs = session.get_inputs() + if len(inputs) != 1 or len(inputs[0].shape) != 4: + raise ValueError("OBB ONNX must have one BCHW image input") + self._input_name = inputs[0].name + self._input_batch_size = inputs[0].shape[0] + shape = inputs[0].shape + if not isinstance(shape[2], int) or shape[2] != shape[3]: + raise ValueError("OBB ONNX requires a static square input size") + self.resolution = shape[2] + outputs = {o.name: o.shape for o in session.get_outputs()} + if set(outputs) != {"dets", "labels", "heading"} or any( + len(s) != 3 for s in outputs.values() + ): + raise ValueError( + "Native OBB requires dets[B,Q,5], labels[B,Q,C], heading[B,Q,4]" + ) + if outputs["dets"][-1] != 5 or outputs["heading"][-1] != 4: + raise ValueError("Invalid OBB output geometry or heading shape") + if ( + not isinstance(outputs["labels"][-1], int) + or max(output_indices) >= outputs["labels"][-1] + ): + raise ValueError("OBB output class mapping exceeds labels dimension") + self._class_names = class_names + self._output_indices = output_indices + + @property + def class_names(self): + """Frozen native class names in artifact order.""" + return self._class_names + + def pre_process(self, images, **kwargs): + """Apply the same rounded letterbox and ImageNet normalization as training. + + Args: + images: RGB uint8 HWC NumPy images or CHW tensors; lists/batches accepted. + **kwargs: input_color_format may explicitly be rgb or bgr. + + Returns: + Float32 BCHW array and exact per-image inverse transform metadata. + """ + if isinstance(images, (torch.Tensor, np.ndarray)): + images = list(images) if images.ndim == 4 else [images] + if not len(images): + raise ValueError("Native OBB requires a nonempty image batch") + arrays, metas = [], [] + for image in images: + if isinstance(image, torch.Tensor): + image = image.detach().cpu().permute(1, 2, 0).numpy() + if image.dtype != np.uint8 or image.ndim != 3 or image.shape[2] != 3: + raise ValueError("Native OBB requires uint8 three-channel images") + if kwargs.get("input_color_format", "rgb") == "bgr": + image = image[:, :, ::-1] + original_height, original_width = image.shape[:2] + image, crop = apply_pre_processing_to_numpy_image( + image=image, + image_pre_processing=self.image_pre_processing, + network_input_channels=3, + input_color_mode=ColorMode.RGB, + pre_processing_overrides=kwargs.get("pre_processing_overrides"), + ) + h, w = image.shape[:2] + size = self.resolution + scale = size / max(h, w) + resized_width = max(1, round(w * scale)) + resized_height = max(1, round(h * scale)) + left, top = (size - resized_width) // 2, (size - resized_height) // 2 + resized = Image.fromarray(image).resize( + (resized_width, resized_height), resample=Image.Resampling.BILINEAR + ) + canvas = Image.new("RGB", (size, size), (114, 114, 114)) + canvas.paste(resized, (left, top)) + array = np.asarray(canvas).astype(np.float32) / 255.0 + array = ( + array - np.array([0.485, 0.456, 0.406], dtype=np.float32) + ) / np.array([0.229, 0.224, 0.225], dtype=np.float32) + arrays.append(array.transpose(2, 0, 1)) + metas.append( + OrientedPreprocessingMetadata( + original_size=ImageDimensions( + height=original_height, width=original_width + ), + scale_width=resized_width / w, + scale_height=resized_height / h, + pad_left=left, + pad_top=top, + inference_size=ImageDimensions(height=size, width=size), + crop_offset_x=crop.offset_x, + crop_offset_y=crop.offset_y, + ) + ) + batch = np.stack(arrays) + return batch, metas + + def forward(self, pre_processed_images, **kwargs): + """Execute the exported graph, respecting a fixed model batch dimension. + + Args: + pre_processed_images: Normalized float32 BCHW array. + **kwargs: Reserved shared inference options. + + Returns: + Named raw dets, labels and heading arrays before selection. + """ + batch_limit = ( + self._input_batch_size + if isinstance(self._input_batch_size, int) + else len(pre_processed_images) + ) + outputs = [] + for start in range(0, len(pre_processed_images), batch_limit): + batch = pre_processed_images[start : start + batch_limit] + count = len(batch) + if count < batch_limit: + batch = np.concatenate( + [batch, np.repeat(batch[-1:], batch_limit - count, axis=0)] + ) + values = self._session.run( + ["dets", "labels", "heading"], {self._input_name: batch} + ) + outputs.append([v[:count] for v in values]) + result = tuple(np.concatenate([o[i] for o in outputs]) for i in range(3)) + return result + + def post_process(self, model_results, pre_processing_meta, **kwargs): + """Select class/query rows and invert the exact training letterbox in float. + + Args: + model_results: Three raw ONNX arrays in dets, labels, heading order. + pre_processing_meta: Metadata returned by pre_process. + **kwargs: confidence (default .5), max_detections (default 300). + + Returns: + Native boxes with category identity, class semantics and unique detection IDs. + + Raises: + ValueError: If graph values, row alignment or heading are malformed. + """ + boxes, logits, headings = (torch.as_tensor(v) for v in model_results) + if ( + boxes.ndim != 3 + or boxes.shape[-1] != 5 + or headings.shape != (*boxes.shape[:2], 4) + or logits.shape[:2] != boxes.shape[:2] + or len(boxes) != len(pre_processing_meta) + ): + raise ValueError("Unaligned native OBB graph outputs") + if not all(torch.isfinite(v).all() for v in (boxes, logits, headings)): + raise ValueError( + "Native OBB geometry/logits must be finite with positive sides" + ) + threshold = kwargs.get("confidence") + threshold = 0.5 if threshold is None else threshold + if not isinstance(threshold, (int, float)) or not 0 <= threshold <= 1: + raise ValueError("Native OBB confidence must be a number in [0,1]") + cap = kwargs.get("max_detections") + cap = 300 if cap is None else cap + if not isinstance(cap, int) or cap < 0: + raise ValueError("Native OBB max_detections must be a nonnegative integer") + indices = torch.tensor(self._output_indices) + results = [] + for image_boxes, image_logits, image_heading, meta in zip( + boxes, logits, headings, pre_processing_meta + ): + scores = image_logits[:, indices].sigmoid() + # Exported graphs can have zero-filled unused query slots. They cannot + # describe a rectangle, regardless of the class logits in that slot. + scores = torch.where( + (image_boxes[:, 2:4] > 0).all(-1)[:, None], scores, -1.0 + ) + flat = scores.flatten() + selected = torch.argsort(flat, descending=True, stable=True)[:cap] + selected = selected[flat[selected] > threshold] + query, cls = selected // len(indices), selected % len(indices) + chosen = image_boxes[query].clone().float() + sides = image_heading[query].argmax(-1) + directed = torch.tensor( + [self.metadata.classHeading[self._class_names[int(c)]] for c in cls], + dtype=torch.bool, + ) + sides = torch.where(directed, sides, 0) + swap = sides % 2 == 1 + chosen[swap, 2:4] = chosen[swap, 2:4].flip(-1) + chosen[:, 4] = (chosen[:, 4] + sides * (torch.pi / 2) + torch.pi) % ( + 2 * torch.pi + ) - torch.pi + chosen[:, :4] *= self.resolution + chosen[:, 0] = ( + chosen[:, 0] - meta.pad_left + ) / meta.scale_width + meta.crop_offset_x + chosen[:, 1] = ( + chosen[:, 1] - meta.pad_top + ) / meta.scale_height + meta.crop_offset_y + # Training scales ordered sides by the mean of the realized resize + # ratios, while centers use each axis's own ratio. + chosen[:, 2:4] /= (meta.scale_width + meta.scale_height) / 2 + corners = _corners(chosen) + bounds = torch.cat((corners.amin(1), corners.amax(1)), dim=-1) + result_id = uuid4().hex + results.append( + OrientedDetections( + image_metadata={ + "native_obb": True, + "class_heading": dict(self.metadata.classHeading), + "image_dimensions": [ + meta.original_size.height, + meta.original_size.width, + ], + }, + xyxy=bounds, + confidence=flat[selected], + class_id=cls.to(device="cpu", dtype=torch.int32), + obb=chosen, + heading=directed, + detection_id=[ + f"{result_id}:q{int(q)}:c{int(c)}" for q, c in zip(query, cls) + ], + class_names=[self._class_names[int(c)] for c in cls], + ) + ) + return results diff --git a/inference_models/inference_models/weights_providers/entities.py b/inference_models/inference_models/weights_providers/entities.py index 5f1bcd865d..8c58638747 100644 --- a/inference_models/inference_models/weights_providers/entities.py +++ b/inference_models/inference_models/weights_providers/entities.py @@ -7,6 +7,9 @@ from pydantic.alias_generators import to_camel from inference_models.models.auto_loaders.entities import BackendType +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) class PackageSourceType(str, Enum): @@ -222,3 +225,4 @@ class ModelMetadata: model_variant: Optional[str] = field(default=None) model_dependencies: Optional[List[ModelDependency]] = field(default=None) recommended_parameters: Optional[RecommendedParameters] = field(default=None) + oriented_box_metadata: Optional[OrientedBoxMetadata] = field(default=None) diff --git a/inference_models/inference_models/weights_providers/offline_registry.py b/inference_models/inference_models/weights_providers/offline_registry.py index d85d01f26c..d43c569c96 100644 --- a/inference_models/inference_models/weights_providers/offline_registry.py +++ b/inference_models/inference_models/weights_providers/offline_registry.py @@ -33,6 +33,9 @@ resolve_existing_model_package_cache_path, slugify_model_id_to_os_safe_format, ) +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) from inference_models.utils.file_system import read_json from inference_models.weights_providers.entities import ( JetsonEnvironmentRequirements, @@ -520,6 +523,10 @@ def record_successful_load( else None ), } + if model_metadata.oriented_box_metadata is not None: + incoming_model["oriented_box_metadata"] = ( + model_metadata.oriented_box_metadata.model_dump(mode="json") + ) merged_model = { field_name: ( incoming_value @@ -585,6 +592,11 @@ def _record_to_model_metadata(record: dict) -> Optional[ModelMetadata]: model_variant=model_payload.get("model_variant"), model_dependencies=model_dependencies, recommended_parameters=recommended_parameters, + oriented_box_metadata=( + OrientedBoxMetadata.model_validate(model_payload["oriented_box_metadata"]) + if model_payload.get("oriented_box_metadata") is not None + else None + ), ) @@ -667,8 +679,7 @@ def _parse_registry_timestamp( except ValueError: pass LOGGER.warning( - "Offline-weights registry record contains unparsable timestamp for " - "%s: %r", + "Offline-weights registry record contains unparsable timestamp for " "%s: %r", context, value, ) diff --git a/inference_models/inference_models/weights_providers/roboflow.py b/inference_models/inference_models/weights_providers/roboflow.py index 87626ad68e..d774ff86b0 100644 --- a/inference_models/inference_models/weights_providers/roboflow.py +++ b/inference_models/inference_models/weights_providers/roboflow.py @@ -16,6 +16,9 @@ ROBOFLOW_API_KEY, SECURE_GATEWAY, ) +from inference_models.models.common.roboflow.oriented_box_metadata import ( + OrientedBoxMetadata, +) from inference_models.utils.secure_gateway import validate_secure_gateway_url LOCAL_API_KEY = "local" @@ -116,6 +119,11 @@ class RoboflowModelMetadata(BaseModel): model_architecture: str = Field(alias="modelArchitecture") model_variant: Optional[str] = Field(alias="modelVariant", default=None) task_type: Optional[str] = Field(alias="taskType", default=None) + oriented_box_metadata: Optional[OrientedBoxMetadata] = Field( + alias="orientedBoxMetadata", + default=None, + description="Frozen Core class heading semantics for native oriented boxes.", + ) model_dependencies: Optional[List[RoboflowModelDependencyV1]] = Field( alias="modelDependencies", default=None ) @@ -195,6 +203,7 @@ def get_roboflow_model( model_architecture=model_metadata.model_architecture, model_packages=parsed_model_packages, task_type=model_metadata.task_type, + oriented_box_metadata=model_metadata.oriented_box_metadata, model_variant=model_metadata.model_variant, model_dependencies=model_dependencies, recommended_parameters=model_metadata.recommended_parameters, diff --git a/inference_models/inference_models/weights_providers/roboflow_offline.py b/inference_models/inference_models/weights_providers/roboflow_offline.py index 4861344952..7dba8eaeb1 100644 --- a/inference_models/inference_models/weights_providers/roboflow_offline.py +++ b/inference_models/inference_models/weights_providers/roboflow_offline.py @@ -93,6 +93,7 @@ def _with_materialized_packages_only( model_architecture=model_metadata.model_architecture, model_packages=materialized, task_type=model_metadata.task_type, + oriented_box_metadata=model_metadata.oriented_box_metadata, model_variant=model_metadata.model_variant, model_dependencies=model_metadata.model_dependencies, recommended_parameters=model_metadata.recommended_parameters, diff --git a/inference_models/pyproject.toml b/inference_models/pyproject.toml index 97657b359b..252b717163 100644 --- a/inference_models/pyproject.toml +++ b/inference_models/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "inference-models" -version = "0.39.6" +version = "0.39.8" description = "The new inference engine for Computer Vision models" readme = "README.md" requires-python = ">=3.10,<3.14" @@ -288,6 +288,8 @@ requires-dist = [ [tool.uv] +# Older uv can reuse Jetson Torch metadata for the plain PyPI wheel. +required-version = ">=0.10.10" conflicts = [ [ { extra = "torch-cpu" }, diff --git a/inference_models/tests/unit_tests/models/auto_loaders/test_core.py b/inference_models/tests/unit_tests/models/auto_loaders/test_core.py index 6ad7cc6aa0..588a8402b3 100644 --- a/inference_models/tests/unit_tests/models/auto_loaders/test_core.py +++ b/inference_models/tests/unit_tests/models/auto_loaders/test_core.py @@ -34,7 +34,6 @@ AutoResolutionCache, AutoResolutionCacheEntry, ) -from inference_models.errors import InvalidParameterError from inference_models.models.auto_loaders.core import ( attempt_loading_matching_model_packages, attempt_loading_model_from_local_storage, @@ -56,10 +55,6 @@ InferenceModelConfig, SuppliedDependency, ) -from inference_models.weights_providers.entities import ( - ModelDependency, - RecommendedParameters, -) from inference_models.runtime_introspection.core import RuntimeXRayResult from inference_models.weights_providers import core as weights_providers_core from inference_models.weights_providers.entities import ( @@ -1004,6 +999,12 @@ def test_dump_model_config_for_offline_use_preserves_legacy_positional_slots( def test_new_offline_parameters_are_appended_to_existing_helper_signatures() -> None: + def legacy_parameters(function): + parameters = inspect.signature(function).parameters + assert list(parameters)[-1] == "oriented_box_metadata" + assert parameters["oriented_box_metadata"].default is None + return list(parameters)[:-1] + assert list(inspect.signature(dump_model_config_for_offline_use).parameters)[ :7 ] == [ @@ -1015,18 +1016,16 @@ def test_new_offline_parameters_are_appended_to_existing_helper_signatures() -> "on_file_created", "model_id", ] - assert list(inspect.signature(initialize_model).parameters)[-3:] == [ + assert legacy_parameters(initialize_model)[-3:] == [ "requested_model_id", "api_key", "content_addressed_artifact_cache", ] - assert list(inspect.signature(attempt_loading_matching_model_packages).parameters)[ - -2: - ] == [ + assert legacy_parameters(attempt_loading_matching_model_packages)[-2:] == [ "requested_model_id", "content_addressed_artifact_cache", ] - assert list(inspect.signature(dump_auto_resolution_cache).parameters)[-2:] == [ + assert legacy_parameters(dump_auto_resolution_cache)[-2:] == [ "canonical_model_id", "api_key", ] @@ -2856,22 +2855,32 @@ def test_create_symlinks_to_shared_blobs_when_hooks_not_provided( def test_resolve_preloaded_dependency_when_none_supplied() -> None: - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id="pkg-1") + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id="pkg-1" + ) assert resolve_preloaded_dependency(dependency, None) is None assert resolve_preloaded_dependency(dependency, {}) is None def test_resolve_preloaded_dependency_when_dependency_not_supplied() -> None: - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id="pkg-1") - supplied = {"other": SuppliedDependency(model_id="x", model_package_id="y", instance=object())} + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id="pkg-1" + ) + supplied = { + "other": SuppliedDependency( + model_id="x", model_package_id="y", instance=object() + ) + } assert resolve_preloaded_dependency(dependency, supplied) is None def test_resolve_preloaded_dependency_when_identity_matches() -> None: instance = object() - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id="pkg-1") + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id="pkg-1" + ) supplied = { "feature_extractor": SuppliedDependency( model_id="owlv2", model_package_id="pkg-1", instance=instance @@ -2881,9 +2890,13 @@ def test_resolve_preloaded_dependency_when_identity_matches() -> None: assert resolve_preloaded_dependency(dependency, supplied) is instance -def test_resolve_preloaded_dependency_when_metadata_package_unset_and_supplied_package_none() -> None: +def test_resolve_preloaded_dependency_when_metadata_package_unset_and_supplied_package_none() -> ( + None +): instance = object() - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id=None) + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id=None + ) supplied = { "feature_extractor": SuppliedDependency( model_id="owlv2", model_package_id=None, instance=instance @@ -2893,8 +2906,12 @@ def test_resolve_preloaded_dependency_when_metadata_package_unset_and_supplied_p assert resolve_preloaded_dependency(dependency, supplied) is instance -def test_resolve_preloaded_dependency_when_metadata_package_unset_but_supplied_concrete() -> None: - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id=None) +def test_resolve_preloaded_dependency_when_metadata_package_unset_but_supplied_concrete() -> ( + None +): + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id=None + ) supplied = { "feature_extractor": SuppliedDependency( model_id="owlv2", model_package_id="pkg-a", instance=object() @@ -2906,7 +2923,9 @@ def test_resolve_preloaded_dependency_when_metadata_package_unset_but_supplied_c def test_resolve_preloaded_dependency_when_model_id_mismatches() -> None: - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id="pkg-1") + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id="pkg-1" + ) supplied = { "feature_extractor": SuppliedDependency( model_id="other-base", model_package_id="pkg-1", instance=object() @@ -2918,7 +2937,9 @@ def test_resolve_preloaded_dependency_when_model_id_mismatches() -> None: def test_resolve_preloaded_dependency_when_package_id_mismatches() -> None: - dependency = ModelDependency(name="feature_extractor", model_id="owlv2", model_package_id="pkg-1") + dependency = ModelDependency( + name="feature_extractor", model_id="owlv2", model_package_id="pkg-1" + ) supplied = { "feature_extractor": SuppliedDependency( model_id="owlv2", model_package_id="pkg-2", instance=object() diff --git a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py index f44f3781f1..b0f61cc2ac 100644 --- a/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py +++ b/inference_models/tests/unit_tests/models/auto_loaders/test_resolved_model_metadata.py @@ -59,12 +59,17 @@ def test_auto_model_reports_canonical_package_after_fresh_and_cached_loads( model_packages=[package], ) monkeypatch.setattr(core, "get_model_from_provider", lambda **kwargs: metadata) - model_class = SimpleNamespace( - from_pretrained=lambda *args, **kwargs: SimpleNamespace() - ) + initialization_options = [] + + def load_package(*args, **kwargs): + initialization_options.append(kwargs) + return SimpleNamespace() + + model_class = SimpleNamespace(from_pretrained=load_package) monkeypatch.setattr(core, "resolve_model_class", lambda **kwargs: model_class) first = core.AutoModel.from_pretrained("alias/1", backend="onnx", device="cpu") + assert "oriented_box_metadata" not in initialization_options[0] first_metadata = getattr(first, "resolved_model", None) assert isinstance(first_metadata, ResolvedModelMetadata) diff --git a/inference_models/uv.lock b/inference_models/uv.lock index 6bc011cae2..c1b78cfea8 100644 --- a/inference_models/uv.lock +++ b/inference_models/uv.lock @@ -1351,7 +1351,7 @@ wheels = [ [[package]] name = "inference-models" -version = "0.39.6" +version = "0.39.8" source = { virtual = "." } dependencies = [ { name = "accelerate" }, @@ -4466,15 +4466,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] -[[package]] -name = "pkgconfig" -version = "1.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/fd/0adde075cd3bfecd557bc7d757e00e231d34d8a6edb4c8d1642759254c21/pkgconfig-1.6.0.tar.gz", hash = "sha256:4a5a6631ce937fafac457104a40d558785a658bbdca5c49b6295bc3fd651907f", size = 5691, upload-time = "2026-03-06T11:26:01.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/75/6f/f7ec07fba48f07c555cc4099481df644fbbc12067879072c17ac229f6556/pkgconfig-1.6.0-py3-none-any.whl", hash = "sha256:98e71754855e9563838d952a160eb577edabb57782e49853edb5381927e6bea1", size = 7086, upload-time = "2026-03-06T11:26:07.688Z" }, -] - [[package]] name = "platformdirs" version = "4.3.8" @@ -5056,7 +5047,6 @@ version = "2.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cffi" }, - { name = "pkgconfig" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7b/88/f73dae807ec68b228fba72507105e3ba80a561dc0bade0004ce24fd118fc/pyvips-2.2.3.tar.gz", hash = "sha256:43bceced0db492654c93008246a58a508e0373ae1621116b87b322f2ac72212f", size = 56626, upload-time = "2024-04-28T11:19:58.158Z" } diff --git a/inference_sdk/http/client.py b/inference_sdk/http/client.py index 527d04e4d0..88597b2582 100644 --- a/inference_sdk/http/client.py +++ b/inference_sdk/http/client.py @@ -34,6 +34,7 @@ INSTANCE_SEGMENTATION_TASK, KEYPOINTS_DETECTION_TASK, OBJECT_DETECTION_TASK, + ORIENTED_OBJECT_DETECTION_TASK, ApiKeyTransport, HTTPClientMode, ImagesReference, @@ -138,6 +139,7 @@ def _warn_about_default_api_key_transport_once() -> None: NEW_INFERENCE_ENDPOINTS = { INSTANCE_SEGMENTATION_TASK: "/infer/instance_segmentation", OBJECT_DETECTION_TASK: "/infer/object_detection", + ORIENTED_OBJECT_DETECTION_TASK: "/infer/oriented_object_detection", CLASSIFICATION_TASK: "/infer/classification", KEYPOINTS_DETECTION_TASK: "/infer/keypoints_detection", } @@ -3548,7 +3550,10 @@ def _determine_client_downsizing_parameters( model_description: Optional[ModelDescription], default_max_input_size: int, ) -> Tuple[Optional[int], Optional[int]]: - if client_downsizing_disabled: + if client_downsizing_disabled or ( + model_description is not None + and model_description.task_type == ORIENTED_OBJECT_DETECTION_TASK + ): return None, None if ( model_description is None diff --git a/inference_sdk/http/entities.py b/inference_sdk/http/entities.py index 0dad1e4492..1283ff37ee 100644 --- a/inference_sdk/http/entities.py +++ b/inference_sdk/http/entities.py @@ -27,6 +27,7 @@ TaskType = str CLASSIFICATION_TASK = "classification" OBJECT_DETECTION_TASK = "object-detection" +ORIENTED_OBJECT_DETECTION_TASK = "oriented-object-detection" INSTANCE_SEGMENTATION_TASK = "instance-segmentation" KEYPOINTS_DETECTION_TASK = "keypoint-detection" ACTION_RECOGNITION_TASK = "action-recognition" @@ -225,7 +226,7 @@ def to_api_call_parameters( """ if client_mode is HTTPClientMode.V0: return self.to_legacy_call_parameters() - if task_type == OBJECT_DETECTION_TASK: + if task_type in {OBJECT_DETECTION_TASK, ORIENTED_OBJECT_DETECTION_TASK}: return self.to_object_detection_parameters() if task_type == INSTANCE_SEGMENTATION_TASK: return self.to_instance_segmentation_parameters() diff --git a/inference_sdk/http/utils/post_processing.py b/inference_sdk/http/utils/post_processing.py index b50da9fe8c..5e29eb8199 100644 --- a/inference_sdk/http/utils/post_processing.py +++ b/inference_sdk/http/utils/post_processing.py @@ -243,6 +243,13 @@ def adjust_prediction_to_client_scaling_factor( if predictions_should_not_be_post_processed(prediction=prediction): return prediction + if prediction.get("task_type") == "oriented-object-detection": + for row in prediction["predictions"]: + if "obb" in row: + row["obb"] = [v / scaling_factor for v in row["obb"][:4]] + [ + row["obb"][4] + ] + return prediction if "points" in prediction["predictions"][0]: prediction["predictions"] = ( adjust_prediction_with_bbox_and_points_to_client_scaling_factor( diff --git a/requirements/requirements.cpu.txt b/requirements/requirements.cpu.txt index a3892839c1..dd67517c72 100644 --- a/requirements/requirements.cpu.txt +++ b/requirements/requirements.cpu.txt @@ -1,3 +1,3 @@ onnxruntime>=1.15.1,<1.22.0 nvidia-ml-py<13.0.0 -inference-models[torch-cpu,onnx-cpu]==0.39.6 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu,onnx-cpu]==0.39.8 # keep in sync between requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.gpu.cu13.txt b/requirements/requirements.gpu.cu13.txt index 3240ee2657..aa090d8d05 100644 --- a/requirements/requirements.gpu.cu13.txt +++ b/requirements/requirements.gpu.cu13.txt @@ -4,6 +4,6 @@ # is a CUDA 12 build), so the Dockerfile installs it in a dedicated step from # Microsoft's onnxruntime-cuda-13 feed. A plain pin here would silently # resolve to the CUDA 12 wheel. -inference-models[torch-cu130]==0.39.6 +inference-models[torch-cu130]==0.39.8 pynvvideocodec~=2.1.0; platform_machine == "x86_64" onnxruntime-gpu>=1.27.0,<1.30.0 diff --git a/requirements/requirements.gpu.txt b/requirements/requirements.gpu.txt index 8bd8a2fd0e..414a0fe368 100644 --- a/requirements/requirements.gpu.txt +++ b/requirements/requirements.gpu.txt @@ -1,3 +1,3 @@ onnxruntime-gpu>=1.15.1,<1.22.0 -inference-models[torch-cu124,onnx-cu12]==0.39.6 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cu124,onnx-cu12]==0.39.8 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt pynvvideocodec>=2.1.0,<3.0.0; platform_machine == "x86_64" diff --git a/requirements/requirements.jetson.txt b/requirements/requirements.jetson.txt index 1fe9d7732b..c46b6b2303 100644 --- a/requirements/requirements.jetson.txt +++ b/requirements/requirements.jetson.txt @@ -1,3 +1,3 @@ pypdfium2>=4.11.0,<5.0.0 PyYAML~=6.0.0 -inference-models==0.39.6 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models==0.39.8 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.vino.txt b/requirements/requirements.vino.txt index 34b0eff87d..cf9a529cf2 100644 --- a/requirements/requirements.vino.txt +++ b/requirements/requirements.vino.txt @@ -1,2 +1,2 @@ onnxruntime-openvino>=1.15.0,<1.22.0 -inference-models[torch-cpu]==0.39.6 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt +inference-models[torch-cpu]==0.39.8 # keep in sync between requirements.jetson requirements.gpu.txt, requirements.cpu.txt, requirements.vino.txt diff --git a/requirements/requirements.workflows.txt b/requirements/requirements.workflows.txt index bdf9c3fd0c..5a06c49919 100644 --- a/requirements/requirements.workflows.txt +++ b/requirements/requirements.workflows.txt @@ -1,2 +1,2 @@ # keep in sync with workflows/pyproject.toml version -roboflow-workflows[enterprise]==0.2.14 +roboflow-workflows[enterprise]==0.2.16 diff --git a/tests/inference/unit_tests/models/model_export_inventory.json b/tests/inference/unit_tests/models/model_export_inventory.json index 24ddc2e1de..aabb635c50 100644 --- a/tests/inference/unit_tests/models/model_export_inventory.json +++ b/tests/inference/unit_tests/models/model_export_inventory.json @@ -180,6 +180,7 @@ "InferenceModelsKeyPointsDetectionAdapter": "inference.core.models.inference_models_adapters:InferenceModelsKeyPointsDetectionAdapter", "InferenceModelsMoondream2Adapter": "inference.models.moondream2.moondream2_inference_models:InferenceModelsMoondream2Adapter", "InferenceModelsObjectDetectionAdapter": "inference.core.models.inference_models_adapters:InferenceModelsObjectDetectionAdapter", + "InferenceModelsOrientedObjectDetectionAdapter": "inference.core.models.inference_models_adapters:InferenceModelsOrientedObjectDetectionAdapter", "InferenceModelsOwlV2Adapter": "inference.models.owlv2.owlv2_inference_models:InferenceModelsOwlV2Adapter", "InferenceModelsPPOCRAdapter": "inference.models.pp_ocr.pp_ocr_inference_models:InferenceModelsPPOCRAdapter", "InferenceModelsPaligemmaAdapter": "inference.models.paligemma.paligemma_inference_models:InferenceModelsPaligemmaAdapter", @@ -271,6 +272,7 @@ "InferenceModelsKeyPointsDetectionAdapter": "inference.core.models.inference_models_adapters:InferenceModelsKeyPointsDetectionAdapter", "InferenceModelsMoondream2Adapter": "inference.models.moondream2.moondream2_inference_models:InferenceModelsMoondream2Adapter", "InferenceModelsObjectDetectionAdapter": "inference.core.models.inference_models_adapters:InferenceModelsObjectDetectionAdapter", + "InferenceModelsOrientedObjectDetectionAdapter": "inference.core.models.inference_models_adapters:InferenceModelsOrientedObjectDetectionAdapter", "InferenceModelsOwlV2Adapter": "inference.models.owlv2.owlv2_inference_models:InferenceModelsOwlV2Adapter", "InferenceModelsPPOCRAdapter": "inference.models.pp_ocr.pp_ocr_inference_models:InferenceModelsPPOCRAdapter", "InferenceModelsPaligemmaAdapter": "inference.models.paligemma.paligemma_inference_models:InferenceModelsPaligemmaAdapter", diff --git a/tests/inference/unit_tests/models/registry_inventory.json b/tests/inference/unit_tests/models/registry_inventory.json index 66a46c5c82..b6c3a49dd8 100644 --- a/tests/inference/unit_tests/models/registry_inventory.json +++ b/tests/inference/unit_tests/models/registry_inventory.json @@ -3104,6 +3104,16 @@ "tr-ocr" ] ], + "inference.core.models.inference_models_adapters:InferenceModelsOrientedObjectDetectionAdapter": [ + [ + "oriented-object-detection", + "rfdetr" + ], + [ + "oriented-object-detection", + "rfdetr-obb-preview-2xl" + ] + ], "inference.core.models.inference_models_adapters:InferenceModelsSemanticSegmentationAdapter": [ [ "semantic-segmentation", @@ -4400,6 +4410,16 @@ "tr-ocr" ] ], + "inference.core.models.inference_models_adapters:InferenceModelsOrientedObjectDetectionAdapter": [ + [ + "oriented-object-detection", + "rfdetr" + ], + [ + "oriented-object-detection", + "rfdetr-obb-preview-2xl" + ] + ], "inference.core.models.inference_models_adapters:InferenceModelsSemanticSegmentationAdapter": [ [ "semantic-segmentation", diff --git a/tests/inference/unit_tests/models/test_native_obb.py b/tests/inference/unit_tests/models/test_native_obb.py new file mode 100644 index 0000000000..1355ca5b7a --- /dev/null +++ b/tests/inference/unit_tests/models/test_native_obb.py @@ -0,0 +1,647 @@ +"""Execute an exported ONNX artifact through native model and request consumers.""" + +import hashlib +import json +import math +import os +from pathlib import Path + +import numpy as np +import onnx +import pytest +import torch +from onnx import TensorProto, helper, numpy_helper + +from inference_models.models.rfdetr.rfdetr_oriented_object_detection_onnx import ( + RFDetrForOrientedObjectDetectionONNX, + _corners, +) + + +def metadata(): + return {"classHeading": {"plane": True, "stone": False}, "classHeadingRevision": 3} + + +def inference_config(size=64): + return { + "image_pre_processing": {}, + "network_input": { + "training_input_size": {"height": size, "width": size}, + "dynamic_spatial_size_supported": False, + "color_mode": "rgb", + "resize_mode": "letterbox", + "padding_value": 114, + "input_channels": 3, + "scaling_factor": 255, + "normalization": [[0.485, 0.456, 0.406], [0.229, 0.224, 0.225]], + }, + "class_names_operations": [ + {"type": "class_name_removal", "class_name": "background_class83422"} + ], + } + + +def package( + tmp_path, *, side=0, box=(0.5, 0.5, 0.4, 0.2, 0.3), malformed=None, class_index=1 +): + dets = np.array([[box]], np.float32) + labels = np.array([[[100.0, -12.0, -12.0]]], np.float32) + labels[0, 0, class_index] = 12.0 + heading = np.full((1, 1, 4), -10.0, np.float32) + heading[0, 0, side] = 10.0 + values = {"dets": dets, "labels": labels, "heading": heading} + if malformed: + values[malformed][...] = np.nan + nodes = [ + helper.make_node("Constant", [], [name], value=numpy_helper.from_array(v)) + for name, v in values.items() + ] + graph = helper.make_graph( + nodes, + "native-obb", + [helper.make_tensor_value_info("images", TensorProto.FLOAT, [1, 3, 64, 64])], + [ + helper.make_tensor_value_info(n, TensorProto.FLOAT, v.shape) + for n, v in values.items() + ], + ) + model = helper.make_model( + graph, opset_imports=[helper.make_opsetid("", 17)], ir_version=9 + ) + onnx.save(model, tmp_path / "weights.onnx") + (tmp_path / "class_names.txt").write_text("background_class83422\nplane\nstone\n") + (tmp_path / "inference_config.json").write_text(json.dumps(inference_config())) + return RFDetrForOrientedObjectDetectionONNX.from_pretrained( + str(tmp_path), oriented_box_metadata=metadata() + ) + + +@pytest.mark.parametrize("side", range(4)) +@pytest.mark.parametrize("wh", [(0.4, 0.2), (0.2, 0.4), (0.3, 0.3)]) +def test_all_front_sides_preserve_footprint_and_directed_squares(tmp_path, side, wh): + model = package(tmp_path, side=side, box=(0.5, 0.5, *wh, 0.3)) + result = model(np.zeros((35, 73, 3), np.uint8), confidence=0.5)[0] + assert result.class_id.tolist() == [ + 0 + ] # remapped class identity, never background output0 + assert result.class_names == ["plane"] + assert result.heading.tolist() == [True] + scale = (64 / 73 + 31 / 35) / 2 + center = [36.5, (32 - 16) / (31 / 35)] + expected = torch.tensor([[*center, wh[0] * 64 / scale, wh[1] * 64 / scale, 0.3]]) + actual = _corners(result.obb)[0].numpy() + reference = _corners(expected)[0].numpy() + np.testing.assert_allclose( + sorted(map(tuple, actual)), sorted(map(tuple, reference)), atol=1e-5 + ) + expected_angle = (0.3 + side * math.pi / 2 + math.pi) % (2 * math.pi) - math.pi + assert result.obb[0, 4].item() == pytest.approx(expected_angle, abs=1e-6) + assert result.obb[0, 2:4].tolist() == pytest.approx( + [wh[side % 2] * 64 / scale, wh[1 - side % 2] * 64 / scale] + ) + assert result.obb[0, :2].tolist() == pytest.approx([36.5, 16 / (31 / 35)]) + assert result.xyxy.dtype.is_floating_point + + +def test_detection_ids_are_unique_across_images_calls_and_models(tmp_path): + model = package(tmp_path) + image = np.zeros((35, 73, 3), np.uint8) + results = [ + *model([image, image], confidence=0), + *model(image, confidence=0), + *package(tmp_path)(image, confidence=0), + ] + ids = [identity for result in results for identity in result.detection_id] + assert len(ids) == 8 + assert all(ids) + assert len(set(ids)) == len(ids) + for result in results: + assert ( + result.to_supervision().data["detection_id"].tolist() == result.detection_id + ) + + +def test_realized_preprocessing_matches_model_similarity_on_odd_image(tmp_path): + model = package(tmp_path) + image = np.zeros((35, 73, 3), np.uint8) + image[17:19, 35:38] = 255 + arrays, metas = model.pre_process([image]) + meta = metas[0] + assert meta.scale_width == 64 / 73 + assert meta.scale_height == 31 / 35 + assert meta.pad_top == 16 + assert arrays.shape == (1, 3, 64, 64) + assert arrays.dtype == np.float32 + assert ( + model([image, image], confidence=0.5)[1].obb.tolist() + == model(image, confidence=0.5)[0].obb.tolist() + ) + + +@pytest.mark.parametrize( + "height,width,pixel_hash,transformed", + [ + ( + 35, + 73, + "d25653931528abaaccae0c1623b7be03ef70baeac69d6b554a4e1a2f6aaac2f6", + [27.51999855, 27.47000122, 14.79557228, 5.2432189], + ), + ( + 73, + 35, + "e4b2fbf4e9dc0f229cdfd0d208c65357b0dc225bf39e921bfabda636ceecc1c7", + [29.32999992, 23.68000031, 7.09376717, 10.93585682], + ), + ( + 79, + 101, + "f112d0893712cccaa52a14568df33bc73268a0b0badaf72e4013ab3fbfe812f3", + [27.52000046, 25.5, 14.71126556, 8.50504971], + ), + ( + 1, + 99, + "472d724451529a66f3bef12c7ce213b617e6cb1529dfa4559368a230011cdc1b", + [27.52000046, 31.37000084, 18.74500084, 0.1399495], + ), + ( + 64, + 64, + "2ffe74f47a7bb7350e913f6b9259080cbe3cee97b2d313d5e2fe2942108d98e9", + [27.52000046, 23.68000031, 14.72000027, 10.88000011], + ), + ], +) +def test_matches_pinned_producer_pixels_and_inverts_target_geometry( + tmp_path, height, width, pixel_hash, transformed +): + # Fixtures from RFDT c54293a Letterbox(64), applied to this gradient and + # obb=[width*.43,height*.37,width*.23,height*.17,.3]. + image = ( + np.arange(height * width * 3, dtype=np.uint32).reshape(height, width, 3) % 251 + ).astype(np.uint8) + model = package(tmp_path, box=(*[value / 64 for value in transformed], 0.3)) + inputs, _ = model.pre_process([image]) + rgb = np.rint( + ( + inputs[0].transpose(1, 2, 0) * np.array([0.229, 0.224, 0.225]) + + np.array([0.485, 0.456, 0.406]) + ) + * 255 + ).astype(np.uint8) + assert hashlib.sha256(rgb.tobytes()).hexdigest() == pixel_hash + result = model(image)[0] + assert result.obb[0].tolist() == pytest.approx( + [width * 0.43, height * 0.37, width * 0.23, height * 0.17, 0.3], abs=1e-5 + ) + + +@pytest.mark.parametrize("name", ["dets", "labels", "heading"]) +def test_nonfinite_predictions_are_rejected(tmp_path, name): + model = package(tmp_path, malformed=name) + with pytest.raises(ValueError, match="finite"): + model(np.zeros((64, 64, 3), np.uint8)) + + +@pytest.mark.parametrize("field", ["classHeading", "classHeadingRevision"]) +def test_missing_frozen_core_metadata_rejects(tmp_path, field): + package(tmp_path) + value = metadata() + del value[field] + with pytest.raises(ValueError): + RFDetrForOrientedObjectDetectionONNX.from_pretrained( + str(tmp_path), oriented_box_metadata=value + ) + + +def test_no_heading_policy_is_inferred_from_four_way_head(tmp_path): + package(tmp_path) + with pytest.raises(ValueError, match="orientedBoxMetadata"): + RFDetrForOrientedObjectDetectionONNX.from_pretrained(str(tmp_path)) + + +def test_axial_class_ignores_heading_head_and_keeps_ordered_geometry(tmp_path): + model = package(tmp_path, side=1, class_index=2) + result = model(np.zeros((64, 64, 3), np.uint8))[0] + assert result.class_id.tolist() == [1] + assert result.class_names == ["stone"] + assert result.heading.tolist() == [False] + assert result.obb[0].tolist() == pytest.approx([32, 32, 25.6, 12.8, 0.3]) + + +def test_empty_confidence_result_keeps_geometry_rank(tmp_path): + result = package(tmp_path)(np.zeros((64, 64, 3), np.uint8), confidence=1.0)[0] + assert result.obb.shape == (0, 5) + assert result.to_supervision().data["xyxyxyxy"].shape == (0, 4, 2) + + +def test_optional_actual_exported_model(tmp_path): + fixture = os.environ.get("OBB_EXPORTED_ONNX") + if fixture is None: + pytest.skip("Set OBB_EXPORTED_ONNX to the actual RF-DETR export fixture") + (tmp_path / "weights.onnx").symlink_to(Path(fixture)) + (tmp_path / "class_names.txt").write_text("background_class83422\nplane\nstone\n") + (tmp_path / "inference_config.json").write_text(json.dumps(inference_config())) + model = RFDetrForOrientedObjectDetectionONNX.from_pretrained( + str(tmp_path), oriented_box_metadata=metadata() + ) + result = model(np.zeros((35, 73, 3), np.uint8), confidence=0, max_detections=4)[0] + assert result.obb.shape == (4, 5) + assert torch.isfinite(result.obb).all() + assert set(result.class_id.tolist()) <= {0, 1} + assert len(set(result.detection_id)) == 4 + + +def test_auto_loader_and_request_consumer(tmp_path, monkeypatch): + package(tmp_path, side=3) + (tmp_path / "model_config.json").write_text( + json.dumps( + { + "model_architecture": "rfdetr", + "task_type": "oriented-object-detection", + "backend_type": "onnx", + "oriented_box_metadata": metadata(), + } + ) + ) + from inference_models import AutoModel + + loaded = AutoModel.from_pretrained( + str(tmp_path), allow_direct_local_storage_loading=True + ) + assert isinstance(loaded, RFDetrForOrientedObjectDetectionONNX) + from inference.core.entities.requests.inference import ( + InferenceRequestImage, + OrientedObjectDetectionInferenceRequest, + ) + from inference.core.models import inference_models_adapters as adapters + + monkeypatch.setattr( + adapters, "ALLOW_INFERENCE_MODELS_DIRECTLY_ACCESS_LOCAL_PACKAGES", True + ) + adapter = adapters.InferenceModelsOrientedObjectDetectionAdapter(str(tmp_path)) + request = OrientedObjectDetectionInferenceRequest( + model_id="native/1", + image=InferenceRequestImage( + type="numpy_object", value=np.zeros((35, 73, 3), np.uint8) + ), + confidence=0.5, + disable_active_learning=True, + ) + response = adapter.infer_from_request(request) + assert response.task_type == "oriented-object-detection" + assert response.image.width == 73 + assert response.image.height == 35 + row = response.predictions[0] + assert row.class_id == 0 and row.heading + assert row.obb[:2] == pytest.approx([36.5, 16 / (31 / 35)]) + assert row.obb[4] == pytest.approx(0.3 - math.pi / 2, abs=1e-6) + assert "x" not in row.model_dump() + + +def test_static_crop_preprocessing_is_inverted_before_response(tmp_path): + package(tmp_path) + (tmp_path / "inference_config.json").write_text( + json.dumps( + { + **inference_config(), + "image_pre_processing": { + "static-crop": { + "enabled": True, + "x_min": 10, + "x_max": 60, + "y_min": 10, + "y_max": 30, + } + }, + } + ) + ) + model = RFDetrForOrientedObjectDetectionONNX.from_pretrained( + str(tmp_path), oriented_box_metadata=metadata() + ) + result = model(np.zeros((35, 73, 3), np.uint8), confidence=0.5)[0] + assert result.obb[0, :2].tolist() == pytest.approx([25, 6.5]) + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_workflow_block_executes_installed_onnx_request_and_native_transport( + tmp_path, monkeypatch, tensor +): + package(tmp_path, side=2) + (tmp_path / "model_config.json").write_text( + json.dumps( + { + "model_architecture": "rfdetr", + "task_type": "oriented-object-detection", + "backend_type": "onnx", + "oriented_box_metadata": metadata(), + } + ) + ) + from roboflow_workflows.core_steps.common.entities import StepExecutionMode + from roboflow_workflows.core_steps.common.oriented import serialise_native_obb + from roboflow_workflows.core_steps.models.roboflow.oriented_object_detection import ( + v1, + ) + from roboflow_workflows.execution_engine.entities.base import ( + Batch, + ImageParentMetadata, + OriginCoordinatesSystem, + WorkflowImageData, + ) + + from inference.core.interfaces.workflows_models_provider import ( + ModelManagerModelsProvider, + ) + from inference.core.models import inference_models_adapters as adapters + + monkeypatch.setattr( + adapters, "ALLOW_INFERENCE_MODELS_DIRECTLY_ACCESS_LOCAL_PACKAGES", True + ) + monkeypatch.setattr(v1, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + adapter = adapters.InferenceModelsOrientedObjectDetectionAdapter(str(tmp_path)) + + class LocalManager: + def add_model(self, **kwargs): + pass + + def infer_from_request_sync(self, model_id, request, **kwargs): + response = adapter.infer_from_request(request) + self.detection_id = response[0].predictions[0].detection_id + return response + + manager = LocalManager() + block = v1.RoboflowOrientedObjectDetectionModelBlockV1( + ModelManagerModelsProvider(manager), None, StepExecutionMode.LOCAL + ) + image = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="image0"), + workflow_root_ancestor_metadata=ImageParentMetadata( + "root", OriginCoordinatesSystem(13, 17, 300, 200) + ), + numpy_image=np.zeros((35, 73, 3), np.uint8), + ) + result = block.run( + Batch([image], indices=[(0,)]), + model_id="native/1", + confidence=0.5, + max_detections=3, + class_filter=["plane"], + ) + native = serialise_native_obb(result[0]["predictions"]) + assert native["task_type"] == "oriented-object-detection" + assert native["predictions"][0]["obb"] == pytest.approx( + [ + 36.5, + 16 / (31 / 35), + 0.4 * 64 / ((64 / 73 + 31 / 35) / 2), + 0.2 * 64 / ((64 / 73 + 31 / 35) / 2), + 0.3 - math.pi, + ], + abs=1e-5, + ) + assert native["predictions"][0]["detection_id"] == manager.detection_id + assert native["predictions"][0]["parent_id"] == "image0" + assert native["predictions"][0]["heading"] + + from roboflow_workflows.core_steps.common.tensor_native import ( + native_detections_to_root_coordinates, + ) + from roboflow_workflows.core_steps.common.utils import ( + sv_detections_to_root_coordinates, + ) + from roboflow_workflows.core_steps.transformations.per_class_confidence_filter import ( + v1 as confidence_numpy, + ) + from roboflow_workflows.core_steps.transformations.per_class_confidence_filter import ( + v1_tensor as confidence_tensor, + ) + + filtering = confidence_tensor if tensor else confidence_numpy + empty = filtering.PerClassConfidenceFilterBlockV1().run( + predictions=[result[0]["predictions"]], class_thresholds={"plane": 1.0} + )[0]["predictions"] + rooted = ( + native_detections_to_root_coordinates(empty) + if tensor + else sv_detections_to_root_coordinates(empty) + ) + serialized = serialise_native_obb(rooted) + assert serialized["predictions"] == [] + assert serialized["image"] == {"width": 300, "height": 200} + assert serialized["class_heading"] == native["class_heading"] + + +def test_unused_frozen_classes_are_retained_without_output_mapping(tmp_path): + package(tmp_path) + frozen = metadata() + frozen["classHeading"]["unused"] = True + loaded = RFDetrForOrientedObjectDetectionONNX.from_pretrained( + str(tmp_path), oriented_box_metadata=frozen + ) + result = loaded(np.zeros((64, 64, 3), np.uint8), confidence=0.5)[0] + assert result.image_metadata["class_heading"]["unused"] is True + + +@pytest.mark.parametrize("task", ["oriented-object-detection", "obb-detection"]) +def test_core_metadata_and_standard_train_package_survive_cache_and_offline( + tmp_path, monkeypatch, requests_mock, task +): + from inference_models.configuration import ROBOFLOW_API_HOST + from inference_models.models.auto_loaders import ( + auto_negotiation, + auto_resolution_cache, + core, + model_cache_paths, + ) + from inference_models.weights_providers import offline_registry + + source = tmp_path / "source" + source.mkdir() + package(source, side=1) + assert {p.name for p in source.iterdir()} == { + "weights.onnx", + "class_names.txt", + "inference_config.json", + } + files = [] + for artifact in source.iterdir(): + content = artifact.read_bytes() + url = f"https://obb-fixture.example/{artifact.name}" + requests_mock.get(url, content=content) + files.append( + { + "fileHandle": artifact.name, + "downloadUrl": url, + "md5Hash": hashlib.md5(content).hexdigest(), + } + ) + metadata_url = f"{ROBOFLOW_API_HOST}/models/v1/external/weights" + requests_mock.get( + metadata_url, + json={ + "modelMetadata": { + "type": "external-model-metadata-v1", + "modelId": "native/1", + "modelArchitecture": "rfdetr", + "taskType": task, + "orientedBoxMetadata": metadata(), + "modelPackages": [ + { + "type": "external-model-package-v1", + "packageId": "nativeonnx", + "packageManifest": { + "type": "onnx-model-package-v1", + "backendType": "onnx", + "quantization": "fp32", + "dynamicBatchSize": False, + "staticBatchSize": 1, + "opset": 17, + }, + "packageFiles": files, + "trustedSource": True, + } + ], + } + }, + ) + for module in (model_cache_paths, auto_resolution_cache, offline_registry): + monkeypatch.setattr(module, "INFERENCE_HOME", str(tmp_path / "cache")) + monkeypatch.setattr( + auto_negotiation, + "get_selected_onnx_execution_providers", + lambda: ["CPUExecutionProvider"], + ) + monkeypatch.setattr(core, "OFFLINE_MODE_WARM_UP", True) + options = dict( + backend="onnx", + device="cpu", + api_key="fixture-key", + onnx_execution_providers=["CPUExecutionProvider"], + ) + fresh = core.AutoModel.from_pretrained("native/1", **options) + monkeypatch.setattr(core, "OFFLINE_MODE_WARM_UP", False) + requests_mock.get( + metadata_url, exc=AssertionError("metadata network request after warmup") + ) + cached = core.AutoModel.from_pretrained("native/1", **options) + monkeypatch.setattr(core, "OFFLINE_MODE", True) + offline = core.AutoModel.from_pretrained( + "native/1", use_auto_resolution_cache=False, **options + ) + image = np.zeros((35, 73, 3), np.uint8) + expected = fresh(image)[0] + for loaded in (cached, offline): + assert loaded.metadata == fresh.metadata + result = loaded(image)[0] + torch.testing.assert_close(result.obb, expected.obb) + assert result.class_id.tolist() == [0] + assert result.heading.tolist() == [True] + + from inference.core.entities.requests.inference import ( + OrientedObjectDetectionInferenceRequest, + ) + from inference.core.models import inference_models_adapters as adapters + from inference_sdk import InferenceConfiguration, InferenceHTTPClient + + adapter = adapters.InferenceModelsOrientedObjectDetectionAdapter( + "native/1", + api_key="fixture-key", + onnx_execution_providers=["CPUExecutionProvider"], + ) + server_url = "http://native-sdk-fixture" + requests_mock.get( + f"{server_url}/model/registry", + json={ + "models": [ + { + "model_id": "native/1", + "task_type": "oriented-object-detection", + "batch_size": 1, + "input_height": 64, + "input_width": 64, + } + ] + }, + ) + + def infer_request(request, context): + payload = request.json() + assert payload["confidence"] == 0.5 + payload["disable_active_learning"] = True + parsed = OrientedObjectDetectionInferenceRequest.model_validate(payload) + response = adapter.infer_from_request(parsed) + return response.model_dump(by_alias=True, exclude_none=True) + + requests_mock.post( + f"{server_url}/infer/oriented_object_detection", json=infer_request + ) + client = InferenceHTTPClient( + api_url=server_url, api_key="fixture-key" + ).select_api_v1() + client.configure( + InferenceConfiguration( + confidence_threshold=0.5, client_downsizing_disabled=False + ) + ) + response = client.infer(image, model_id="native/1") + assert response["task_type"] == "oriented-object-detection" + assert response["image"] == {"width": 73, "height": 35} + assert response["class_heading"] == metadata()["classHeading"] + assert response["predictions"][0]["obb"] == pytest.approx(expected.obb[0].tolist()) + batch = client.infer([image, image], model_id="native/1") + assert len(batch) == 2 + assert all(item["image"] == response["image"] for item in batch) + assert all( + item["predictions"][0]["obb"] == pytest.approx(expected.obb[0].tolist()) + for item in batch + ) + if os.environ.get("OBB_SMOKE_RESPONSE"): + Path(os.environ["OBB_SMOKE_RESPONSE"]).write_text(json.dumps(response)) + + manifest = ( + Path( + model_cache_paths.generate_model_package_cache_path( + model_id="native/1", package_id="nativeonnx" + ) + ) + / "model_config.json" + ) + contents = json.loads(manifest.read_text()) + contents["oriented_box_metadata"]["classHeading"]["plane"] = False + manifest.write_text(json.dumps(contents)) + from inference_models.errors import ModelPackageAlternativesExhaustedError + + with pytest.raises( + ModelPackageAlternativesExhaustedError, match="Offline oriented-box semantics" + ): + core.AutoModel.from_pretrained( + "native/1", use_auto_resolution_cache=False, **options + ) + + +@pytest.mark.parametrize("task", ["oriented-object-detection", "object-detection"]) +def test_request_factory_preserves_native_or_ordinary_request(task): + from inference.core.entities.requests.inference import ( + ObjectDetectionInferenceRequest, + OrientedObjectDetectionInferenceRequest, + request_from_type, + ) + + request_class = ( + OrientedObjectDetectionInferenceRequest + if task == "oriented-object-detection" + else ObjectDetectionInferenceRequest + ) + original = request_class( + model_id="native/1", + image={"type": "base64", "value": "AA=="}, + confidence=0.5, + class_filter=["plane"], + id="dispatch-request", + ) + restored = request_from_type(task, original.model_dump()) + assert type(restored) is request_class + assert restored.model_dump() == original.model_dump() diff --git a/workflows/CHANGELOG.md b/workflows/CHANGELOG.md index eb82f556fc..9407f37c5e 100644 --- a/workflows/CHANGELOG.md +++ b/workflows/CHANGELOG.md @@ -18,6 +18,8 @@ for contributor and maintainer responsibilities. ### Execution engine +- Add native `oriented_object_detection_prediction` with original-image float geometry and frozen class heading semantics. Native model, visualization, row filters, combination, crop offsets and serialization retain ordered fronts in NumPy and tensor carriers. Projective transforms retain ordered quadrilaterals instead of silently fitting new rectangle annotations. + - **Inner Workflow literal bindings keep their type.** A literal bound in an Inner Workflow step's `parameter_bindings` (for example `0.3`, `true`, a list or an object) now reaches the child workflow's steps with that type when the @@ -50,13 +52,31 @@ for contributor and maintainer responsibilities. - SAM 3 Interactive (`roboflow_core/sam3_interactive@v1`): optional `output_format`, `polygons` (default, output unchanged) or `rle`, with the same values as SAM 3 v3. With `rle` the block asks SAM 3 for RLE masks on every execution path and returns one prediction per prompt that keeps the model's mask in `rle_mask` (decoded into `mask` for downstream blocks), declared as `rle_instance_segmentation_prediction` first, so serialized outputs carry `rle_mask` instead of polygon `points`. A server that answers polygons anyway has them rasterised into the prompt's mask. The tensor-native variant already carried RLE masks; with `rle` it declares and serializes them as RLE, and, like the NumPy block, keeps one mask per prompt for a polygon answer, drops empty masks and reports COCO box extents (a one-pixel mask is a 1x1 box). `ModelsProvider.run_sam2_segmentation` gains `format`, which the block sends only for `rle`. - Vision Events (`roboflow_core/roboflow_vision_events@v1`) and Vision Event Bundle (`roboflow_core/vision_event_bundle@v1`) blocks gain an optional `value_base_units` input: what the event is worth, in base units of the use case's currency (e.g. cents for USD). The currency always comes from the use case. The value is sent as `valueBaseUnits` to the Vision Events API and in the bundle `payload.json`, and as `value_base_units` to the local event store. A typed value or workflow input must be a whole number (`true`/`false` are rejected). A value from another step is rounded to whole base units; one that is not a finite number or falls outside ยฑ10,000,000,000 is dropped with a warning and the event is still recorded. Left empty, the use case's default value applies. `operator_feedback` events never carry a value, so the blocks omit it for them. +- Opt-in Detections Consensus v2 ports mask/rotated overlap voting from contributor PR #2420 and preserves a winning native OBB contributor, front, identity and frozen class policy through NumPy and tensor transport. Ordinary tensor inputs remain explicitly unsupported by v2. + ### Changed +- Native OBB requires `inference-models>=0.39.8` and `inference-sdk>=1.8.4`; existing Supervision `<0.31.0` support is retained. Package `0.2.16` must follow those dependency releases. + - Remove duplicated model lists and counts from `meta_vlm@v3` descriptions; available models are defined by the registry-backed dropdown. ### Fixed +- Keep native OBB image headers through filtering, combination, crop/scale/root transforms and empty selections; derive Perspective bounds from ordered corners. + +- Preserve native class names across colliding model-local IDs, initialize external native input lineage, render empty tensor predictions, admit native perspective connections without rounding corners, and declare native blocks in workload discovery. + +- Keep legacy oriented-corner scaling and perspective rounding unchanged; float corner transport is specific to native OBB metadata. + +- Reject native oriented detections at ordinary detection conversion and input boundaries; use the dedicated native kind and model block. + +- Reject native oriented detections in query-language `DetectionsShift`, `DetectionsOffset`, and `DetectionsRename` before mutation; these operations are excluded from native kind compatibility. + +- Clip native Dynamic Crop extraction windows to image boundaries while retaining the complete ordered geometry relative to the visible crop. + +- Detections Consensus v2 compares complete segmentation masks even when foreground extends beyond the detector boxes. + - `roboflow_core/asset_library_attributes@v1` normalizes metadata keys before sending them: characters outside `[A-Za-z0-9_.]` become `_`, and a leading `.` gets a `_` prefix. If two keys of one image normalize to the same key (for example `traffic light` and `traffic_light`), the block raises an error instead of letting one value overwrite the other. Before, one invalid key (for example a class name with a space) made the API reject every attribute for that image. - Root-coordinate recovery (`sv_detections_to_root_coordinates`, applied to workflow outputs by default) re-encodes `rle_mask` from the re-anchored dense masks. A block that ran on a crop and returns RLE with dense masks, such as SAM 3 v3 with `output_format: rle`, used to serialize root-coordinate boxes next to an RLE still sized and placed in the crop. RLE-only predictions (`mask=None`) are unchanged, as in `scale_sv_detections`. diff --git a/workflows/pyproject.toml b/workflows/pyproject.toml index 91dbd600b6..f6653b4949 100644 --- a/workflows/pyproject.toml +++ b/workflows/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "roboflow-workflows" -version = "0.2.14" +version = "0.2.16" description = "Roboflow Workflows execution engine and block library" readme = "README.md" license = {file = "LICENSE"} @@ -23,8 +23,8 @@ dependencies = [ "urllib3>=2.7.0,<3.0.0", "torch>=2.0.1,<3.0.0", "torchvision", - "inference-models>=0.39.5", - "inference-sdk>=1.8.1", + "inference-models>=0.39.8", + "inference-sdk>=1.8.4", "aiohttp~=3.14.4", "msgpack", "websocket-client~=1.9.0", diff --git a/workflows/roboflow_workflows/core_steps/common/deserializers.py b/workflows/roboflow_workflows/core_steps/common/deserializers.py index 0544ba4beb..b5d33b01a2 100644 --- a/workflows/roboflow_workflows/core_steps/common/deserializers.py +++ b/workflows/roboflow_workflows/core_steps/common/deserializers.py @@ -8,6 +8,7 @@ import pybase64 import supervision as sv from pydantic import ValidationError +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.core_steps.common.utils import ( add_inference_keypoints_to_sv_detections, filter_out_invalid_polygons, @@ -251,6 +252,34 @@ def deserialize_detections_kind( parameter: str, detections: Any, ) -> sv.Detections: + """Deserialize predictions declared as an ordinary detection kind. + + Args: + parameter: Runtime parameter name used in validation errors and lineage. + detections: Serialized predictions or an existing Supervision carrier. + + Returns: + Ordinary detections with image and parent metadata. + + Raises: + RuntimeInputError: If the input is malformed or carries native OBB semantics. + """ + native_payload = ( + isinstance(detections, dict) + and detections.get("task_type") == "oriented-object-detection" + ) + native_carrier = isinstance(detections, sv.Detections) and ( + bool(detections.metadata.get(NATIVE_OBB)) + or np.any(detections.data.get(NATIVE_OBB, False)) + ) + if native_payload or native_carrier: + raise RuntimeInputError( + public_message=f"Runtime parameter `{parameter}` contains native oriented " + "detections; declare oriented_object_detection_prediction instead of " + "an ordinary detection kind.", + context="workflow_execution | runtime_input_validation", + ) + if isinstance(detections, sv.Detections): return detections if not isinstance(detections, dict): diff --git a/workflows/roboflow_workflows/core_steps/common/deserializers_tensor.py b/workflows/roboflow_workflows/core_steps/common/deserializers_tensor.py index 67020fb11d..13cd248201 100644 --- a/workflows/roboflow_workflows/core_steps/common/deserializers_tensor.py +++ b/workflows/roboflow_workflows/core_steps/common/deserializers_tensor.py @@ -16,6 +16,7 @@ from roboflow_workflows.core_steps.common.deserializers import ( deserialize_video_metadata_kind, ) +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.core_steps.common.tensor_native import ( native_detections_from_inference_predictions, ) @@ -159,6 +160,28 @@ def _parse_image_metadata_fields(parameter: str, image: Any): return parent_metadata, workflow_root_ancestor_metadata, video_metadata +def _reject_native_oriented_detections(parameter: str, detections: Any) -> None: + """Keep native OBB semantics out of ordinary tensor detection kinds.""" + native_payload = ( + isinstance(detections, dict) + and detections.get("task_type") == "oriented-object-detection" + ) + native_carrier = isinstance(detections, (Detections, InstanceDetections)) and ( + bool((detections.image_metadata or {}).get(NATIVE_OBB)) + or any( + (row or {}).get(NATIVE_OBB, False) + for row in (detections.bboxes_metadata or []) + ) + ) + if native_payload or native_carrier: + raise RuntimeInputError( + public_message=f"Runtime parameter `{parameter}` contains native oriented " + "detections; declare oriented_object_detection_prediction instead of " + "an ordinary detection kind.", + context="workflow_execution | runtime_input_validation", + ) + + def deserialize_detections_kind( parameter: str, detections: Any, @@ -173,7 +196,19 @@ def deserialize_detections_kind( prediction dicts (center ``x``/``y``/``width``/``height``). Lineage carried by the serialised predictions is reconstructed onto the placeholder image so crop-aware coordinate recovery downstream keeps working. + + Args: + parameter: Runtime parameter name used in validation errors and lineage. + detections: Ordinary serialized predictions or an existing tensor carrier. + + Returns: + Tensor detections preserving ordinary prediction geometry and lineage. + + Raises: + RuntimeInputError: If the input is malformed or carries native OBB semantics. """ + _reject_native_oriented_detections(parameter=parameter, detections=detections) + if isinstance(detections, (Detections, InstanceDetections)): return detections raw_predictions = _validate_serialized_detections( @@ -197,7 +232,19 @@ def deserialize_rle_detections_kind( then rebuilds ``InstancesRLEMasks`` from each serialised prediction's COCO RLE (``RLE_MASK_KEY_IN_INFERENCE_RESPONSE``) and returns a native ``InstanceDetections``. Used for the rle-instance-seg and semantic-seg kinds. + + Args: + parameter: Runtime parameter name used in validation errors and lineage. + detections: Ordinary RLE predictions or an existing tensor carrier. + + Returns: + Tensor detections preserving ordinary segmentation masks and lineage. + + Raises: + RuntimeInputError: If the input is malformed or carries native OBB semantics. """ + _reject_native_oriented_detections(parameter=parameter, detections=detections) + if isinstance(detections, (Detections, InstanceDetections)): return detections raw_predictions = _validate_serialized_detections( diff --git a/workflows/roboflow_workflows/core_steps/common/oriented.py b/workflows/roboflow_workflows/core_steps/common/oriented.py new file mode 100644 index 0000000000..5d9bbf5750 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/common/oriented.py @@ -0,0 +1,398 @@ +"""Native OBB Workflow transport. Ordered corners carry the front through warps. + +Inside a Workflow the first edge points along the native positive width/front +axis. Perspective/anisotropic warps retain that order as quadrilaterals; they +are never silently refitted into native rectangle annotations. +""" + +import math +from copy import deepcopy +from dataclasses import replace + +import cv2 +import numpy as np +import supervision as sv + +NATIVE_OBB = "native_obb" +CORNERS = "xyxyxyxy" +_NATIVE_FRAME_KEYS = ( + "image_dimensions", + "parent_id", + "parent_coordinates", + "parent_dimensions", + "root_parent_id", + "root_parent_coordinates", + "root_parent_dimensions", + "scaling_relative_to_parent", + "scaling_relative_to_root_parent", +) + + +def _corners(obb): + cx, cy, w, h, angle = obb + c, s = math.cos(angle), math.sin(angle) + local = np.array( + [[-w / 2, -h / 2], [w / 2, -h / 2], [w / 2, h / 2], [-w / 2, h / 2]] + ) + return local @ np.array([[c, s], [-s, c]]) + [cx, cy] + + +def _rectangle(corners): + if corners.shape != (4, 2) or not np.isfinite(corners).all(): + raise ValueError("Malformed native OBB ordered corners") + edges = np.roll(corners, -1, axis=0) - corners + next_edges = np.roll(edges, -1, axis=0) + turns = edges[:, 0] * next_edges[:, 1] - edges[:, 1] * next_edges[:, 0] + if not (np.all(turns > 0) or np.all(turns < 0)): + raise ValueError( + "Native OBB corners must form a nondegenerate convex footprint" + ) + u, v = corners[1] - corners[0], corners[3] - corners[0] + w, h = np.linalg.norm(u), np.linalg.norm(v) + if w <= 0 or h <= 0: + raise ValueError("Native OBB cannot have a zero side") + if ( + not np.allclose( + corners[0] + corners[2], corners[1] + corners[3], atol=1e-5, rtol=1e-6 + ) + or abs(np.dot(u, v)) > 1e-6 * w * h + ): + return None + center = corners.mean(0) + return [ + float(center[0]), + float(center[1]), + float(w), + float(h), + math.atan2(u[1], u[0]), + ] + + +def native_obb_from_inference(response): + """Parse native tuples or ordered Workflow quadrilaterals without an AABB cast. + + Args: + response: Native inference response dict, including image dimensions. + + Returns: + Supervision transport with float ordered corners and class/front metadata. + + Raises: + ValueError: If geometry, direction, class semantics or identity are absent. + """ + if response.get("task_type") != "oriented-object-detection": + raise ValueError("Native OBB task identity is required") + rows = response["predictions"] + names, ids, confidences, corners, headings, detection_ids = [], [], [], [], [], [] + class_heading = response["class_heading"] + if not isinstance(class_heading, dict) or any( + type(value) is not bool for value in class_heading.values() + ): + raise ValueError("Native OBB requires frozen boolean class semantics") + for row in rows: + name = row["class"] + if ( + type(row.get("heading")) is not bool + or class_heading.get(name) is not row["heading"] + ): + raise ValueError("Native OBB heading must match frozen class semantics") + if "obb" in row: + obb = np.asarray(row["obb"], dtype=float) + if obb.shape != (5,) or not np.isfinite(obb).all() or (obb[2:4] <= 0).any(): + raise ValueError("Malformed native OBB geometry") + polygon = _corners(obb) + elif row.get("geometry_type") == "quadrilateral": + polygon = np.asarray(row["corners"], dtype=float) + _rectangle(polygon) + else: + raise ValueError("Native geometry is required") + if not isinstance(row.get("detection_id"), str) or not row["detection_id"]: + raise ValueError("Native OBB detection identity is required") + names.append(name) + ids.append(row["class_id"]) + confidences.append(row["confidence"]) + corners.append(polygon) + headings.append(row["heading"]) + detection_ids.append(row["detection_id"]) + corners = np.asarray(corners, dtype=np.float64).reshape(-1, 4, 2) + if not np.isfinite(confidences).all() or any( + not 0 <= value <= 1 for value in confidences + ): + raise ValueError("Native OBB confidence must be finite and in [0,1]") + bounds = ( + np.concatenate((corners.min(1), corners.max(1)), axis=1) + if rows + else np.empty((0, 4)) + ) + height, width = response["image"]["height"], response["image"]["width"] + parent_ids = [row.get("parent_id") for row in rows] + if any(value is not None for value in parent_ids) and any( + not isinstance(value, str) or not value for value in parent_ids + ): + raise ValueError("Native OBB parent identity must be provided for every row") + + result = sv.Detections( + xyxy=bounds, + class_id=np.asarray(ids, dtype=int), + confidence=np.asarray(confidences, dtype=float), + data={ + CORNERS: corners, + "heading": np.asarray(headings, dtype=bool), + "class_name": np.asarray(names, dtype=str), + "detection_id": np.asarray(detection_ids, dtype=str), + NATIVE_OBB: np.ones(len(rows), dtype=bool), + "image_dimensions": np.tile([height, width], (len(rows), 1)), + }, + metadata={ + NATIVE_OBB: True, + "class_heading": dict(class_heading), + "image_dimensions": [height, width], + }, + ) + if parent_ids and all(parent_ids): + result.data["parent_id"] = np.asarray(parent_ids, dtype=str) + + result = _native_obb_to_sv(result) + return result + + +def serialise_native_obb(detections): + """Serialize a native Workflow result from its current ordered corners. + + Args: + detections: Native supervision transport after transformations/selection. + + Returns: + Native response; nonrectangular warps explicitly emit quadrilaterals. + + Raises: + ValueError: If front metadata or geometry has been dropped. + """ + detections = _native_obb_to_sv(detections) + rows = [] + if not isinstance(detections.metadata.get("class_heading"), dict): + raise ValueError("Native OBB frozen class_heading metadata is required") + class_heading = dict(detections.metadata["class_heading"]) + for i in range(len(detections)): + data = detections.data + if CORNERS not in data or "heading" not in data: + raise ValueError("Native OBB corners and heading are required") + corners = np.asarray(data[CORNERS][i], dtype=np.float64) + rectangle = _rectangle(corners) + name = str(data["class_name"][i]) + if not isinstance(data["heading"][i], (bool, np.bool_)): + raise ValueError("Native OBB heading must be boolean") + heading = bool(data["heading"][i]) + if name in class_heading and class_heading[name] != heading: + raise ValueError("Conflicting native class heading semantics") + class_heading[name] = heading + row = { + "class": name, + "class_id": int(detections.class_id[i]), + "confidence": float(detections.confidence[i]), + "heading": heading, + "detection_id": str(data["detection_id"][i]), + } + if rectangle is None: + row.update(geometry_type="quadrilateral", corners=corners.tolist()) + else: + row["obb"] = rectangle + if "parent_id" in data: + row["parent_id"] = str(data["parent_id"][i]) + rows.append(row) + dimensions = detections.metadata.get("image_dimensions") + if dimensions is None: + raise ValueError("Native OBB image dimensions are required") + result = { + "predictions": rows, + "image": {"height": int(dimensions[0]), "width": int(dimensions[1])}, + "task_type": "oriented-object-detection", + "class_heading": class_heading, + } + return result + + +def deserialize_native_obb(parameter, detections): + """Deserialize a native Workflow input. + + Args: + parameter: Workflow parameter name. + detections: Native response dict or supervision transport. + + Returns: + Validated native supervision transport. + """ + from roboflow_workflows.environment import ENABLE_TENSOR_DATA_REPRESENTATION + + result = ( + native_obb_from_inference(detections) + if isinstance(detections, dict) + else _native_obb_to_sv(detections) + ) + if isinstance(detections, dict): + # Match ordinary prediction inputs, importing lazily to avoid the + # deserializer registry's dependency on this module. + from roboflow_workflows.core_steps.common.deserializers import ( + _attach_parent_coordinates_and_dimensions, + ) + + rows = detections["predictions"] + for prefix in ("parent", "root_parent"): + key = f"{prefix}_id" + result.data.setdefault( + key, np.asarray([row.get(key, parameter) for row in rows], dtype=str) + ) + _attach_parent_coordinates_and_dimensions( + parameter=parameter, + raw_detections=rows, + parsed_detections=result, + origin_key=f"{prefix}_origin", + coordinates_key=f"{prefix}_coordinates", + dimensions_key=f"{prefix}_dimensions", + ) + result = _native_obb_to_sv(result) + serialise_native_obb(result) + if ENABLE_TENSOR_DATA_REPRESENTATION: + result = native_obb_to_tensor(result) + return result + + +def annotate_native_obb(image, detections, *, thickness=2): + """Draw each current footprint and the designated front edge. + + Args: + image: BGR uint8 HWC image, modified in place. + detections: Native transport. + thickness: Line width in pixels. + + Returns: + Annotated input image. + """ + if not len(detections): + return image + + for i, corners in enumerate(detections.data[CORNERS]): + points = np.rint(corners).astype(np.int32) + cv2.polylines(image, [points], True, (251, 81, 163), thickness) + if detections.data["heading"][i]: + cv2.line( + image, tuple(points[1]), tuple(points[2]), (51, 182, 255), thickness + 1 + ) + return image + + +def native_obb_to_tensor(detections): + """Convert native corner transport to the standard tensor detection carrier. + + Args: + detections: Native supervision detections with image/crop lineage. + + Returns: + Tensor detections; ordered corners and heading remain per-row metadata. + """ + import torch + + from inference_models.models.base.object_detection import Detections + + detections = _native_obb_to_sv(detections) + metadata = dict(detections.metadata) + metadata["class_names"] = ( + { + int(c): str(n) + for c, n in zip(detections.class_id, detections.data["class_name"]) + } + if len(detections) + else {} + ) + entries = [ + {key: value[i] for key, value in detections.data.items()} + for i in range(len(detections)) + ] + result = Detections( + xyxy=torch.as_tensor(detections.xyxy), + class_id=torch.as_tensor(detections.class_id), + confidence=torch.as_tensor(detections.confidence), + image_metadata=metadata, + bboxes_metadata=entries, + ) + return result + + +def _native_obb_to_sv(detections): + if isinstance(detections, sv.Detections): + result = deepcopy(detections) + else: + from roboflow_workflows.core_steps.common.tensor_native import HOST_MIRROR_KEYS + + entries = detections.bboxes_metadata or [] + # Device host mirrors are derived caches, not SV detection columns. + keys = set().union(*(e.keys() for e in entries)).difference(HOST_MIRROR_KEYS) + result = sv.Detections( + xyxy=detections.xyxy.detach().cpu().numpy(), + confidence=detections.confidence.detach().cpu().numpy(), + class_id=detections.class_id.detach().cpu().numpy(), + data={key: np.asarray([e[key] for e in entries]) for key in keys}, + metadata=deepcopy(detections.image_metadata or {}), + ) + # The image header owns the frame. Rows only fill absent boundary fields; + # compatibility copies must follow the header before further transformations. + for key in _NATIVE_FRAME_KEYS: + if key not in result.metadata and len(result) and key in result.data: + value = result.data[key][0] + result.metadata[key] = ( + value.tolist() if isinstance(value, np.ndarray) else value + ) + result = _native_obb_in_frame(result, result.metadata) + return result + + +def _native_obb_in_frame(detections, metadata): + """Project compatibility columns into an already selected image frame.""" + # An empty first source can lack a header ID; keep row identity for consensus. + data = { + key: value + for key, value in detections.data.items() + if key not in _NATIVE_FRAME_KEYS or (key == "parent_id" and key not in metadata) + } + for key in _NATIVE_FRAME_KEYS: + if key in metadata: + data[key] = np.asarray([metadata[key]] * len(detections)) + result = replace(detections, data=data, metadata=metadata) + return result + + +def merge_native_semantics(first, second): + """Reconcile frozen capabilities before combining native prediction rows. + + Args: + first: NumPy or tensor predictions. + second: Predictions in the same representation and image frame. + + Returns: + Combined frozen class map for native inputs, otherwise None. + + Raises: + ValueError: For mixed tasks or conflicting frozen class capabilities. + """ + maps = [] + native = [] + for prediction in (first, second): + metadata = ( + prediction.metadata + if isinstance(prediction, sv.Detections) + else (prediction.image_metadata or {}) + ) + native.append(bool(metadata.get(NATIVE_OBB))) + maps.append(metadata.get("class_heading")) + if not any(native): + return None + if not all(native) or any(not isinstance(value, dict) for value in maps): + raise ValueError( + "Native OBB combination requires frozen native semantics on both inputs" + ) + combined = dict(maps[0]) + for name, heading in maps[1].items(): + if name in combined and combined[name] is not heading: + raise ValueError("Conflicting native class heading semantics") + combined[name] = heading + return combined diff --git a/workflows/roboflow_workflows/core_steps/common/oriented_entities.py b/workflows/roboflow_workflows/core_steps/common/oriented_entities.py new file mode 100644 index 0000000000..4d6ed1ceb2 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/common/oriented_entities.py @@ -0,0 +1,89 @@ +"""Native OBB response entities shared by server and Workflow schema introspection.""" + +from typing import Dict, List, Literal, Optional, Tuple, Union + +from pydantic import BaseModel, Field, StrictBool, field_validator +from roboflow_workflows.core_steps.common.inference_response_entities import ( + CvInferenceResponse, + WithVisualizationResponse, +) + + +class OrientedObjectDetectionPrediction(BaseModel): + """Native rectangle/front tuple, expressed in original-image pixels and CW radians.""" + + obb: List[float] = Field( + min_length=5, + max_length=5, + description="Ordered cx, cy, width, height, clockwise angle; sides remain ordered.", + ) + confidence: float = Field(ge=0, le=1, description="Sigmoid class confidence.") + class_name: str = Field(alias="class", description="Frozen class name.") + class_id: int = Field(description="Native dataset category identifier.") + heading: StrictBool = Field(description="Whether this class has a semantic front.") + detection_id: str = Field( + description="Stable query/category identity within an input result." + ) + parent_id: Optional[str] = Field( + default=None, description="Input image region identity." + ) + + @field_validator("obb") + @classmethod + def _check_obb(cls, value): + import math + + if not all(math.isfinite(v) for v in value) or value[2] <= 0 or value[3] <= 0: + raise ValueError("OBB requires finite geometry and positive ordered sides") + return value + + +class OrientedObjectDetectionInferenceResponse( + CvInferenceResponse, WithVisualizationResponse +): + """Native task response; AABB geometry is never substituted for an OBB.""" + + predictions: List[OrientedObjectDetectionPrediction] = Field( + description="Native oriented detections." + ) + task_type: Literal["oriented-object-detection"] = Field( + default="oriented-object-detection", description="Native task identity." + ) + class_heading: Dict[str, bool] = Field( + description="Frozen class heading capabilities." + ) + + +class OrientedQuadrilateralPrediction(BaseModel): + """Ordered native footprint after a general affine/projective transform.""" + + geometry_type: Literal["quadrilateral"] = Field( + description="Explicit nonrectangular footprint." + ) + corners: List[Tuple[float, float]] = Field( + min_length=4, + max_length=4, + description="Four ordered float pixel corners; first edge follows the native front axis.", + ) + confidence: float = Field(ge=0, le=1, description="Retained class confidence.") + class_name: str = Field(alias="class", description="Frozen class name.") + class_id: int = Field(description="Native category identifier.") + heading: StrictBool = Field(description="Whether the class has semantic front.") + detection_id: str = Field(description="Stable source detection identity.") + parent_id: Optional[str] = Field( + default=None, description="Parent image/crop identity." + ) + + +class OrientedWorkflowInferenceResponse(CvInferenceResponse): + """Wire schema for native rectangle or explicitly warped quadrilateral results.""" + + predictions: List[ + Union[OrientedObjectDetectionPrediction, OrientedQuadrilateralPrediction] + ] = Field(description="Native predictions after supported Workflow operations.") + task_type: Literal["oriented-object-detection"] = Field( + default="oriented-object-detection", description="Native task identity." + ) + class_heading: Dict[str, bool] = Field( + description="Frozen class heading capabilities." + ) diff --git a/workflows/roboflow_workflows/core_steps/common/query_language/entities/operations.py b/workflows/roboflow_workflows/core_steps/common/query_language/entities/operations.py index a7ff8ffbd1..9463bfbcff 100644 --- a/workflows/roboflow_workflows/core_steps/common/query_language/entities/operations.py +++ b/workflows/roboflow_workflows/core_steps/common/query_language/entities/operations.py @@ -27,6 +27,7 @@ KEYPOINT_DETECTION_PREDICTION_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, TIMESTAMP_KIND, WILDCARD_KIND, @@ -199,6 +200,7 @@ class DetectionsPropertyExtract(OperationDefinition): "compound": False, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -216,6 +218,7 @@ class DetectionsToDictionary(OperationDefinition): "compound": False, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -248,11 +251,13 @@ class DetectionsSelection(OperationDefinition): "compound": False, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], "output_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -283,11 +288,13 @@ class DetectionsFilter(OperationDefinition): "compound": True, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], "output_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -306,11 +313,13 @@ class SortDetections(OperationDefinition): "compound": False, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], "output_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -372,6 +381,7 @@ class RandomNumber(OperationDefinition): "into random floating point value. To be used mainly to sample predictions or images.", "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, WILDCARD_KIND, @@ -473,6 +483,7 @@ class SequenceLength(OperationDefinition): "compound": False, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, LIST_OF_VALUES_KIND, @@ -596,11 +607,13 @@ class PickDetectionsByParentClass(OperationDefinition): "compound": False, "input_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], "output_kind": [ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -954,6 +967,7 @@ class IsEmpty(UnaryOperator): LIST_OF_VALUES_KIND, DICTIONARY_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -974,6 +988,7 @@ class IsNotEmpty(UnaryOperator): LIST_OF_VALUES_KIND, DICTIONARY_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], diff --git a/workflows/roboflow_workflows/core_steps/common/query_language/operations/detection/base.py b/workflows/roboflow_workflows/core_steps/common/query_language/operations/detection/base.py index 2e9a401fa1..a2bc150a64 100644 --- a/workflows/roboflow_workflows/core_steps/common/query_language/operations/detection/base.py +++ b/workflows/roboflow_workflows/core_steps/common/query_language/operations/detection/base.py @@ -9,6 +9,7 @@ SPEED_KEY_IN_SV_DETECTIONS, VELOCITY_KEY_IN_SV_DETECTIONS, ) +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.core_steps.common.query_language.entities.enums import ( DetectionsProperty, ) @@ -123,6 +124,9 @@ def _extract_detection_property( def _extract_class_name_tensor_native(detection: tuple) -> str: + if detection[6].get(NATIVE_OBB): + return str(detection[5]["class_name"]) + class_names = detection[6].get(CLASS_NAMES_KEY) if class_names is None: raise OperationError( diff --git a/workflows/roboflow_workflows/core_steps/common/query_language/operations/detections/base.py b/workflows/roboflow_workflows/core_steps/common/query_language/operations/detections/base.py index 06b663ef26..9986d72082 100644 --- a/workflows/roboflow_workflows/core_steps/common/query_language/operations/detections/base.py +++ b/workflows/roboflow_workflows/core_steps/common/query_language/operations/detections/base.py @@ -4,6 +4,7 @@ import numpy as np import supervision as sv import torch +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.core_steps.common.query_language.entities.enums import ( DetectionsProperty, DetectionsSelectionMode, @@ -477,13 +478,21 @@ def _pick_detections_by_parent_class_impl( ) -> sv.Detections: class_names = detections.data.get("class_name") if class_names is None or len(class_names) == 0: - return sv.Detections.empty() + return ( + detections[:0] + if detections.metadata.get(NATIVE_OBB) + else sv.Detections.empty() + ) if not isinstance(class_names, np.ndarray): class_names = np.array(class_names) parent_mask = class_names == parent_class parent_detections = detections[parent_mask] if len(parent_detections) == 0: - return sv.Detections.empty() + return ( + detections[:0] + if detections.metadata.get(NATIVE_OBB) + else sv.Detections.empty() + ) dependent_detections = detections[~parent_mask] dependent_detections_anchors = dependent_detections.get_anchors_coordinates( anchor=Position.CENTER @@ -667,6 +676,9 @@ def _resolve_class_names( ) -> List[str]: if _detections_count(detections) == 0: return [] + if (detections.image_metadata or {}).get(NATIVE_OBB): + return [str(entry["class_name"]) for entry in detections.bboxes_metadata] + class_names = _class_names_lookup(detections, operation_name=operation_name) result = [] for class_id_scalar in detections.class_id.tolist(): @@ -1441,10 +1453,18 @@ def _parent_class_row_flags( the divergence where the old ``_resolve_class_names`` call raised on any unmapped row, even ones the query doesn't touch. + Native OBB uses authoritative per-row names instead of model-local ID maps. + A *truly absent* map (no ``CLASS_NAMES_KEY`` at all) remains a producer contract violation and still raises via ``_class_names_lookup`` โ€” the single accepted divergence from numpy, which returns empty there. """ + if (detections.image_metadata or {}).get(NATIVE_OBB): + return [ + name == parent_class + for name in _resolve_class_names(detections, operation_name) + ] + class_names_map = _class_names_lookup(detections, operation_name=operation_name) return [ class_names_map.get(int(class_id_scalar)) == parent_class @@ -1578,9 +1598,49 @@ def filter_detections( ) +def _reject_native_obb_operation(value: Any, *, operation_name: str) -> None: + """Reject operations that cannot preserve native geometry or frozen classes.""" + if _is_key_point_prediction(value): + value = value[1] + + if isinstance(value, sv.Detections): + is_native = bool(value.metadata.get(NATIVE_OBB)) or np.any( + value.data.get(NATIVE_OBB, False) + ) + elif isinstance(value, TENSOR_NATIVE_DETECTIONS_TYPES): + is_native = bool((value.image_metadata or {}).get(NATIVE_OBB)) or any( + (row or {}).get(NATIVE_OBB, False) for row in (value.bboxes_metadata or []) + ) + else: + return + + if is_native: + raise InvalidInputTypeError( + public_message=f"Executing {operation_name}(...) does not support native " + "oriented detections with frozen geometry and class semantics.", + context="step_execution | roboflow_query_language_evaluation", + ) + + def offset_detections( value: Any, offset_x: int, offset_y: int, **kwargs ) -> Union[sv.Detections, TensorNativeDetections]: + """Expand or contract ordinary detection bounds around their centers. + + Args: + value: NumPy or tensor detections with axis-aligned geometry. + offset_x: Amount to add to each box's width. + offset_y: Amount to add to each box's height. + **kwargs: Additional operation execution context. + + Returns: + Copied detections with adjusted axis-aligned bounds. + + Raises: + InvalidInputTypeError: If detections are native oriented or unsupported. + """ + _reject_native_obb_operation(value, operation_name="offset_detections") + if ENABLE_TENSOR_DATA_REPRESENTATION: return _offset_detections_tensor_native( value=value, offset_x=offset_x, offset_y=offset_y, **kwargs @@ -1593,6 +1653,22 @@ def offset_detections( def shift_detections( value: Any, shift_x: int, shift_y: int, **kwargs ) -> Union[sv.Detections, TensorNativeDetections]: + """Translate ordinary detection bounds without changing their sizes. + + Args: + value: NumPy or tensor detections with axis-aligned geometry. + shift_x: Horizontal translation in pixels. + shift_y: Vertical translation in pixels. + **kwargs: Additional operation execution context. + + Returns: + Copied detections with translated axis-aligned bounds. + + Raises: + InvalidInputTypeError: If detections are native oriented or unsupported. + """ + _reject_native_obb_operation(value, operation_name="shift_detections") + if ENABLE_TENSOR_DATA_REPRESENTATION: return _shift_detections_tensor_native( value=value, shift_x=shift_x, shift_y=shift_y, **kwargs @@ -1626,6 +1702,24 @@ def rename_detections( global_parameters: Dict[str, Any], **kwargs, ) -> Union[sv.Detections, TensorNativeDetections]: + """Rename ordinary detection classes using the configured mapping. + + Args: + detections: Ordinary NumPy or tensor detections. + class_map: Name replacements or the parameter supplying them. + strict: Whether every observed class must appear in the mapping. + new_classes_id_offset: Starting ID for remapped classes in non-strict mode. + global_parameters: Runtime parameters resolving named configuration values. + **kwargs: Additional operation execution context. + + Returns: + Copied detections with remapped names and class IDs. + + Raises: + InvalidInputTypeError: If detections carry native frozen class semantics. + """ + _reject_native_obb_operation(detections, operation_name="rename_detections") + if ENABLE_TENSOR_DATA_REPRESENTATION: return _rename_detections_tensor_native( detections=detections, diff --git a/workflows/roboflow_workflows/core_steps/common/serializers.py b/workflows/roboflow_workflows/core_steps/common/serializers.py index a14d7ac7ea..8c335b9e45 100644 --- a/workflows/roboflow_workflows/core_steps/common/serializers.py +++ b/workflows/roboflow_workflows/core_steps/common/serializers.py @@ -78,6 +78,10 @@ def serialise_sv_detections(detections: sv.Detections) -> dict: + if detections.metadata.get("native_obb") or "native_obb" in detections.data: + from roboflow_workflows.core_steps.common.oriented import serialise_native_obb + + return serialise_native_obb(detections) serialized_detections = [] image_dimensions = None for xyxy, mask, confidence, class_id, tracker_id, data in detections: diff --git a/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py b/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py index 65f1c29fe8..721ffdeca8 100644 --- a/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py +++ b/workflows/roboflow_workflows/core_steps/common/serializers_tensor.py @@ -124,6 +124,13 @@ def serialise_sv_detections( despite the ``sv`` prefix it now consumes native tensor objects, not ``sv.Detections``. """ + if (detections.image_metadata or {}).get("native_obb") or any( + (entry or {}).get("native_obb") for entry in (detections.bboxes_metadata or []) + ): + from roboflow_workflows.core_steps.common.oriented import serialise_native_obb + + return serialise_native_obb(detections) + serialized_image_metadata, serialized_detections, _ = _serialise_sv_detections( detections=detections ) diff --git a/workflows/roboflow_workflows/core_steps/common/utils.py b/workflows/roboflow_workflows/core_steps/common/utils.py index 7e9792867f..7cc29250be 100644 --- a/workflows/roboflow_workflows/core_steps/common/utils.py +++ b/workflows/roboflow_workflows/core_steps/common/utils.py @@ -24,6 +24,7 @@ KEYPOINT_PADDING_CLASS_NAME, validate_keypoints_padding, ) +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB, _native_obb_to_sv from roboflow_workflows.errors import RuntimeInputError from roboflow_workflows.execution_engine.constants import ( DETECTION_ID_KEY, @@ -184,10 +185,30 @@ def convert_inference_detections_batch_to_sv_detections( predictions_key: str = "predictions", image_key: str = "image", ) -> List[sv.Detections]: + """Convert ordinary inference responses to Supervision detections. + + Args: + predictions: Batch of ordinary inference response dictionaries. + predictions_key: Response key containing prediction rows. + image_key: Response key containing image dimensions. + + Returns: + Detection carriers with identities and image dimensions. + + Raises: + ValueError: If a response requires the dedicated native OBB kind and block. + """ batch_of_detections: List[sv.Detections] = [] for p in predictions: width, height = p[image_key][WIDTH_KEY], p[image_key][HEIGHT_KEY] raw_predictions = filter_out_invalid_polygons(predictions=p[predictions_key]) + if p.get("task_type") == "oriented-object-detection": + raise ValueError( + "Native oriented detections require the " + "oriented_object_detection_prediction kind and the " + "Oriented Object Detection model block." + ) + detections = sv.Detections.from_inference({**p, "predictions": raw_predictions}) parent_ids = [d.get(PARENT_ID_KEY, "") for d in raw_predictions] detection_ids = [_get_or_create_detection_id(d) for d in raw_predictions] @@ -314,9 +335,21 @@ def attach_parent_coordinates_to_detections( coordinates_key: str, dimensions_key: str, ) -> sv.Detections: + """Attach frame lineage while retaining native image headers. + + Args: + detections: Predictions to update in place. + parent_metadata: Known ancestor identity and coordinate frame. + parent_id_key: Parent identity field. + coordinates_key: Parent offset field. + dimensions_key: Parent dimensions field. + + Returns: + Predictions with row lineage and, for natives, a matching image header. + """ parent_coordinates_system = parent_metadata.origin_coordinates - if len(detections) == 0: - # Per-row data cannot retain lineage when there are no rows. + if len(detections) == 0 or detections.metadata.get(NATIVE_OBB): + # Native frames must survive later removal of every row. detections.metadata.update( { parent_id_key: parent_metadata.parent_id, @@ -359,7 +392,20 @@ def attach_parent_coordinates_to_detections( def sv_detections_to_root_coordinates( detections: sv.Detections, keypoints_key: str = KEYPOINTS_XY_KEY_IN_SV_DETECTIONS ) -> sv.Detections: - detections_copy = deepcopy(detections) + """Return predictions expressed in their root image frame. + + Args: + detections: Predictions carrying root lineage. + keypoints_key: Row field containing keypoint coordinates. + + Returns: + A copy with translated geometry and synchronized native frame metadata. + """ + detections_copy = ( + _native_obb_to_sv(detections) + if detections.metadata.get(NATIVE_OBB) + else deepcopy(detections) + ) if len(detections_copy) == 0: root_dimensions = detections_copy.metadata.get(ROOT_PARENT_DIMENSIONS_KEY) if root_dimensions is not None: @@ -405,6 +451,12 @@ def sv_detections_to_root_coordinates( detections_copy[IMAGE_DIMENSIONS_KEY] = np.array( [[origin_height, origin_width]] * len(detections_copy) ) + if detections_copy.metadata.get(NATIVE_OBB): + detections_copy.metadata.update( + image_dimensions=[origin_height, origin_width], + scaling_relative_to_parent=1.0, + scaling_relative_to_root_parent=1.0, + ) root_parent_id = detections_copy[ROOT_PARENT_ID_KEY][0] shift_x, shift_y = detections_copy[ROOT_PARENT_COORDINATES_KEY][0] detections_copy.xyxy += [shift_x, shift_y, shift_x, shift_y] @@ -553,8 +605,9 @@ def scale_sv_detections( Dataset Upload, when using different X/Y scales. An anisotropic transform cannot be represented by the existing scalar metadata. """ - detections_copy = deepcopy(detections) - if len(detections_copy) == 0: + native = bool(detections.metadata.get(NATIVE_OBB)) + detections_copy = _native_obb_to_sv(detections) if native else deepcopy(detections) + if not native and len(detections_copy) == 0: return detections_copy if isinstance(scale, (tuple, list)): @@ -570,10 +623,46 @@ def scale_sv_detections( "terminal result that will not undergo root-coordinate recovery." ) + if native: + dimensions = detections_copy.metadata.get(IMAGE_DIMENSIONS_KEY) + if target_size_wh is not None: + detections_copy.metadata[IMAGE_DIMENSIONS_KEY] = [ + int(target_size_wh[1]), + int(target_size_wh[0]), + ] + elif dimensions is not None: + detections_copy.metadata[IMAGE_DIMENSIONS_KEY] = ( + np.rint(np.asarray(dimensions) * [scale_y, scale_x]) + .astype(int) + .tolist() + ) + if update_scaling_metadata: + for key in ( + SCALING_RELATIVE_TO_PARENT_KEY, + SCALING_RELATIVE_TO_ROOT_PARENT_KEY, + ): + detections_copy.metadata[key] = ( + detections_copy.metadata.get(key, 1.0) * scale_x + ) + if not len(detections_copy): + detections_copy = _native_obb_to_sv(detections_copy) + return detections_copy + xyxy = detections_copy.xyxy.astype(np.float64, copy=True) xyxy[:, [0, 2]] *= scale_x xyxy[:, [1, 3]] *= scale_y - detections_copy.xyxy = xyxy.round() + if ( + detections_copy.metadata.get(NATIVE_OBB) + and ORIENTED_BOX_COORDINATES in detections_copy.data + ): + corners = detections_copy.data[ORIENTED_BOX_COORDINATES].astype( + np.float64, copy=True + ) + corners *= np.asarray([scale_x, scale_y]) + detections_copy.data[ORIENTED_BOX_COORDINATES] = corners + detections_copy.xyxy = np.concatenate((corners.min(1), corners.max(1)), axis=1) + else: + detections_copy.xyxy = xyxy.round() if keypoints_key in detections_copy.data: for i in range(len(detections_copy[keypoints_key])): diff --git a/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v1.py b/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v1.py index 61d891482f..7594f393c3 100644 --- a/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v1.py +++ b/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v1.py @@ -3,7 +3,18 @@ from collections import Counter from enum import Enum from functools import lru_cache -from typing import Dict, Generator, List, Literal, Optional, Set, Tuple, Type, Union +from typing import ( + Callable, + Dict, + Generator, + List, + Literal, + Optional, + Set, + Tuple, + Type, + Union, +) from uuid import uuid4 import numpy as np @@ -263,6 +274,27 @@ def run( detections_merge_coordinates_aggregation: AggregationMode, detections_merge_mask_aggregation: MaskAggregationMode, ) -> BlockResult: + """Fuse each image's prediction sources using the block's consensus strategy. + + Args: + predictions_batches: One batch per model source, aligned by image. + required_votes: Minimum matching sources for an accepted detection. + class_aware: Restrict votes to matching class names. + iou_threshold: Minimum overlap for a candidate vote. + confidence: Minimum aggregated detection confidence. + classes_to_consider: Optional accepted class names. + required_objects: Minimum total or per-class consensus count. + presence_confidence_aggregation: Confidence aggregation for presence. + detections_merge_confidence_aggregation: Output confidence/class policy. + detections_merge_coordinates_aggregation: Ordinary box coordinate policy. + detections_merge_mask_aggregation: Ordinary mask merge policy. + + Returns: + Per-image predictions and object-presence information. + + Raises: + ValueError: If no prediction source is supplied or image lineage differs. + """ if len(predictions_batches) < 1: raise ValueError( f"Consensus step requires at least one source of predictions." @@ -274,7 +306,7 @@ def run( object_present, presence_confidence, consensus_detections, - ) = agree_on_consensus_for_all_detections_sources( + ) = self._agree_on_consensus( detections_from_sources=list(detections_from_sources), required_votes=required_votes, class_aware=class_aware, @@ -296,6 +328,10 @@ def run( ) return results + def _agree_on_consensus(self, **kwargs): + result = agree_on_consensus_for_all_detections_sources(**kwargs) + return result + def does_not_detect_objects_in_any_source( detections_from_sources: List[sv.Detections], @@ -345,7 +381,23 @@ def get_detections_from_different_sources_with_max_overlap( iou_threshold: float, class_aware: bool, detections_already_considered: Set[str], + *, + _iou_calculator: Optional[Callable[..., float]] = None, ) -> Dict[int, Tuple[sv.Detections, float]]: + """Select the strongest unconsumed localization match from each other source. + + Args: + detection: Single detection receiving candidate votes. + source: Index of its prediction source. + detections_from_sources: All sources in the same image frame. + iou_threshold: Minimum overlap for a vote. + class_aware: Restrict votes to the same class name. + detections_already_considered: Identities consumed by previous consensus. + _iou_calculator: Optional version-specific localization strategy. + + Returns: + Source index to matching detection and overlap score. + """ current_max_overlap = {} for other_source, other_detection in enumerate_detections( detections_from_sources=detections_from_sources, @@ -358,7 +410,7 @@ def get_detections_from_different_sources_with_max_overlap( and detection["class_name"][0] != other_detection["class_name"][0] ): continue - iou_value = calculate_iou( + iou_value = (_iou_calculator or calculate_iou)( detection_a=detection, detection_b=other_detection, ) @@ -401,7 +453,33 @@ def agree_on_consensus_for_all_detections_sources( detections_merge_confidence_aggregation: AggregationMode, detections_merge_coordinates_aggregation: AggregationMode, detections_merge_mask_aggregation: MaskAggregationMode, + *, + _iou_calculator: Optional[Callable[..., float]] = None, + _detection_merger: Optional[Callable[..., sv.Detections]] = None, ) -> Tuple[str, bool, Dict[str, float], sv.Detections]: + """Compute consensus across the prediction sources for one image. + + Args: + detections_from_sources: Predictions in the same image frame. + required_votes: Minimum distinct matching sources. + class_aware: Restrict votes to the same class name. + iou_threshold: Minimum overlap for a vote. + confidence: Minimum merged detection confidence. + classes_to_consider: Optional accepted class names. + required_objects: Minimum total or per-class object count. + presence_confidence_aggregation: Presence confidence policy. + detections_merge_confidence_aggregation: Output confidence/class policy. + detections_merge_coordinates_aggregation: Ordinary box coordinate policy. + detections_merge_mask_aggregation: Ordinary mask merge policy. + _iou_calculator: Optional version-specific localization strategy. + _detection_merger: Optional version-specific contributor merge strategy. + + Returns: + Parent identity, presence flag, presence confidence and fused detections. + + Raises: + ValueError: If nonempty sources do not share one parent image identity. + """ if does_not_detect_objects_in_any_source( detections_from_sources=detections_from_sources ): @@ -423,6 +501,8 @@ def agree_on_consensus_for_all_detections_sources( detections_already_considered, ) = get_consensus_for_single_detection( detection=detection, + _iou_calculator=_iou_calculator, + _detection_merger=_detection_merger, source_id=source_id, detections_from_sources=detections_from_sources, iou_threshold=iou_threshold, @@ -465,13 +545,37 @@ def get_consensus_for_single_detection( detections_merge_coordinates_aggregation: AggregationMode, detections_merge_mask_aggregation: MaskAggregationMode, detections_already_considered: Set[str], + *, + _iou_calculator: Optional[Callable[..., float]] = None, + _detection_merger: Optional[Callable[..., sv.Detections]] = None, ) -> Tuple[List[sv.Detections], Set[str]]: + """Build at most one consensus detection and mark its contributors consumed. + + Args: + detection: Candidate anchor detection. + source_id: Anchor source index. + detections_from_sources: All sources in the same image frame. + iou_threshold: Minimum overlap for a vote. + class_aware: Restrict votes to matching class names. + required_votes: Minimum distinct matching sources. + confidence: Minimum aggregated detection confidence. + detections_merge_confidence_aggregation: Output confidence/class policy. + detections_merge_coordinates_aggregation: Ordinary coordinate policy. + detections_merge_mask_aggregation: Ordinary mask policy. + detections_already_considered: Previously consumed detection identities. + _iou_calculator: Optional version-specific localization strategy. + _detection_merger: Optional version-specific contributor merge strategy. + + Returns: + Accepted consensus detections and updated consumed identities. + """ if detection and detection["detection_id"][0] in detections_already_considered: return [], detections_already_considered consensus_detections = [] detections_with_max_overlap = ( get_detections_from_different_sources_with_max_overlap( detection=detection, + _iou_calculator=_iou_calculator, source=source_id, detections_from_sources=detections_from_sources, iou_threshold=iou_threshold, @@ -502,7 +606,7 @@ def get_consensus_for_single_detection( [detection] + [matched_value[0] for matched_value in detections_with_max_overlap.values()] ) - merged_detection = merge_detections( + merged_detection = (_detection_merger or merge_detections)( detections=detections_to_merge, confidence_aggregation_mode=detections_merge_confidence_aggregation, boxes_aggregation_mode=detections_merge_coordinates_aggregation, diff --git a/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v2.py b/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v2.py new file mode 100644 index 0000000000..e586b106e5 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/fusion/detections_consensus/v2.py @@ -0,0 +1,451 @@ +"""Opt-in mask/OBB consensus, ported from contributor PR #2420. + +Reuse v1 voting and aggregation. Native geometry, front and identity come from +one highest-confidence contributor of the class chosen by that aggregation. +""" + +import math +from functools import lru_cache +from typing import Callable, List, Literal, Optional, Tuple + +import numpy as np +import supervision as sv +from pydantic import AliasChoices, ConfigDict, Field +from roboflow_workflows.core_steps.common.oriented import ( + NATIVE_OBB, + _native_obb_in_frame, + _native_obb_to_sv, + merge_native_semantics, + native_obb_to_tensor, + serialise_native_obb, +) +from roboflow_workflows.core_steps.fusion.detections_consensus import v1 +from roboflow_workflows.core_steps.fusion.detections_consensus.v1 import ( + AggregationMode, + MaskAggregationMode, +) +from roboflow_workflows.environment import ENABLE_TENSOR_DATA_REPRESENTATION +from roboflow_workflows.execution_engine.constants import DETECTION_ID_KEY +from roboflow_workflows.execution_engine.entities.base import OutputDefinition +from roboflow_workflows.execution_engine.entities.tensor_native_types import ( + TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, +) +from roboflow_workflows.execution_engine.entities.types import ( + INSTANCE_SEGMENTATION_PREDICTION_KIND, + KEYPOINT_DETECTION_PREDICTION_KIND, + OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import DependentResource +from shapely.geometry import Polygon +from supervision.config import ORIENTED_BOX_COORDINATES + +_MaskGeometry = Tuple[np.ndarray, Tuple[int, int, int, int], int] + +_SOURCE_ORDER = "_consensus_source_order" +_NATIVE_KIND = ( + TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND + if ENABLE_TENSOR_DATA_REPRESENTATION + else ORIENTED_OBJECT_DETECTION_PREDICTION_KIND +) +_INPUT_KINDS = [_NATIVE_KIND] +if not ENABLE_TENSOR_DATA_REPRESENTATION: + _INPUT_KINDS += [ + OBJECT_DETECTION_PREDICTION_KIND, + INSTANCE_SEGMENTATION_PREDICTION_KIND, + KEYPOINT_DETECTION_PREDICTION_KIND, + ] + + +class BlockManifest(v1.BlockManifest): + """Select geometry-aware voting explicitly with the v2 block type.""" + + model_config = ConfigDict( + json_schema_extra={ + **v1.BlockManifest.model_config["json_schema_extra"], + "version": "v2", + "long_description": ( + "Combine detections using v1 voting and class/confidence aggregation. " + "Compare masks when both ordinary inputs carry masks, rotated corners " + "when both carry them, and AABBs otherwise. Native OBB inputs require " + "frozen class heading semantics and ordered corners: the output keeps " + "the highest-confidence contributor of the chosen output class, " + "including its footprint, front and identity (source-order ties). " + "Native and ordinary inputs cannot be mixed. Native inputs support " + "NumPy and tensor carriers through CPU geometry conversion. Ordinary " + "tensor inputs are unsupported; use v1 for those inputs." + ), + } + ) + type: Literal["roboflow_core/detections_consensus@v2"] = Field( + description="Explicit opt-in to mask and oriented-geometry voting." + ) + predictions_batches: List[Selector(kind=_INPUT_KINDS)] = Field( + min_length=1, + validation_alias=AliasChoices("predictions_batches", "predictions"), + description="Predictions for the same image. Native OBB sources must share frozen class heading semantics. Ordinary inputs are supported in NumPy mode; tensor mode accepts native OBB only.", + examples=[["$inputs.first", "$inputs.second"]], + ) + detections_merge_coordinates_aggregation: AggregationMode = Field( + default=AggregationMode.AVERAGE, + description="Ordinary box coordinate aggregation: average, largest (max), or smallest (min). Native OBB geometry and front are retained atomically from the highest-confidence contributor of the aggregated output class, with source-order ties; this setting does not refit native OBBs.", + examples=["average"], + ) + + @classmethod + @lru_cache(maxsize=None) + def describe_outputs(cls) -> List[OutputDefinition]: + """Describe consensus outputs, including the native oriented carrier. + + Returns: + Output definitions with native geometry compatibility. + """ + outputs = [ + output.model_copy(deep=True) for output in super().describe_outputs() + ] + outputs[0].kind = [_NATIVE_KIND] + ( + [OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND] + if not ENABLE_TENSOR_DATA_REPRESENTATION + else [] + ) + return outputs + + def discover_work_operations(self) -> List[WorkOperation]: + """Declare geometry-aware detection processing. + + Returns: + The work performed by this block. + """ + return [WorkOperation.DETECTION_PROCESSING] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + """Declare unrestricted execution. + + Args: + ignore_environment_restrictions: Whether to omit host restrictions. + + Returns: + A complete empty restriction declaration. + """ + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + def discover_dependent_resources(self) -> List[DependentResource]: + """Declare that no external resources are required. + + Returns: + A complete empty resource declaration. + """ + return [] + + +class DetectionsConsensusBlockV2(v1.DetectionsConsensusBlockV1): + """Run geometry-aware voting with v1's established consensus traversal.""" + + @classmethod + def get_manifest(cls): + """Return the explicit v2 manifest. + + Returns: + Manifest supporting native oriented predictions. + """ + return BlockManifest + + def _agree_on_consensus(self, **kwargs): + sources = kwargs.pop("detections_from_sources") + tensor = [not isinstance(source, sv.Detections) for source in sources] + if any(tensor) and not all(tensor): + raise ValueError("Consensus sources must use the same carrier") + + if any(tensor): + from inference_models.models.base.object_detection import Detections + + if any(not isinstance(source, Detections) for source in sources): + raise ValueError("Unsupported consensus tensor carrier") + sources = [_native_obb_to_sv(source) for source in sources] + if not all(_is_native(source) for source in sources): + raise ValueError("V2 ordinary tensor consensus is unsupported; use v1") + + native = [_is_native(source) for source in sources] + policy = None + if any(native): + if not all(native): + raise ValueError("Native OBB consensus cannot mix ordinary detections") + sources = [_native_obb_to_sv(source) for source in sources] + policy_carrier = sources[0][:0] + for source in sources: + # Validate every source before filtering/voting, including empty maps. + serialise_native_obb(source) + if any( + type(value) is not bool + for value in source.metadata["class_heading"].values() + ): + raise ValueError( + "Native OBB requires frozen boolean class semantics" + ) + if any( + source.metadata["class_heading"].get(str(name)) is not bool(heading) + for name, heading in zip( + source.data.get("class_name", []), + source.data.get("heading", []), + ) + ): + raise ValueError( + "Native OBB heading must match frozen class semantics" + ) + if NATIVE_OBB in source.data and not np.all(source.data[NATIVE_OBB]): + raise ValueError( + "Native OBB consensus cannot mix ordinary detections" + ) + policy = merge_native_semantics(policy_carrier, source) + policy_carrier.metadata = { + **policy_carrier.metadata, + "class_heading": policy, + } + if not v1.does_not_detect_objects_in_any_source(sources): + v1.get_parent_id_of_detections_from_sources(sources) + sources = [ + _native_obb_in_frame(source, policy_carrier.metadata) + for source in sources + ] + for index, source in enumerate(sources): + # Native names and geometry remain per-row; temporary SV views + # share the first image header so optional keys do not break merge. + source.data[_SOURCE_ORDER] = np.full(len(source), index) + corners = source.data.get(ORIENTED_BOX_COORDINATES) + if len(source): + source.xyxy = np.concatenate( + (corners.min(1), corners.max(1)), axis=1 + ) + + result = v1.agree_on_consensus_for_all_detections_sources( + detections_from_sources=sources, + _iou_calculator=( + calculate_iou if any(native) else _make_mask_iou_calculator(sources) + ), + _detection_merger=merge_detections, + **kwargs, + ) + parent_id, present, confidence, detections = result + if any(native): + if not len(detections): + detections = policy_carrier + detections.data.pop(_SOURCE_ORDER, None) + if all(tensor): + if not len(detections): + detections.data.setdefault("class_name", np.asarray([], dtype=str)) + detections = native_obb_to_tensor(detections) + return parent_id, present, confidence, detections + + +def _is_native(detection): + return bool(detection.metadata.get(NATIVE_OBB)) or bool( + np.any(detection.data.get(NATIVE_OBB, False)) + ) + + +def _make_mask_iou_calculator( + sources: List[sv.Detections], +) -> Callable[[sv.Detections, sv.Detections], float]: + """Keep lazily derived foreground geometry within one consensus call.""" + masked_sources = [ + source for source in sources if len(source) and source.mask is not None + ] + if len(masked_sources) < 2 or any( + not isinstance(source.mask, np.ndarray) for source in masked_sources + ): + return calculate_iou + if len({source.mask.shape[1:] for source in masked_sources}) != 1: + return calculate_iou + + identities = [] + for source in masked_sources: + source_ids = source.data.get(DETECTION_ID_KEY) + if source_ids is None or any( + not isinstance(value, str) for value in source_ids + ): + return calculate_iou + identities.extend(source_ids) + # v1 does not require unique input IDs; aliases must not share cached geometry. + if len(set(identities)) != len(identities): + return calculate_iou + + geometry: Optional[dict[str, _MaskGeometry]] = None + + def _calculate_pair_iou( + detection_a: sv.Detections, detection_b: sv.Detections + ) -> float: + if detection_a.mask is None or detection_b.mask is None: + return calculate_iou(detection_a, detection_b) + + nonlocal geometry + if geometry is None: + geometry = {} + for source in masked_sources: + boxes = sv.mask_to_xyxy(source.mask) + for identity, mask, box in zip( + source.data[DETECTION_ID_KEY], source.mask, boxes + ): + x_min, y_min, x_max, y_max = map(int, box) + # Supervision bounds are inclusive; extraction windows are half-open. + x_max += 1 + y_max += 1 + area = int(np.count_nonzero(mask[y_min:y_max, x_min:x_max])) + geometry[identity] = (mask, (x_min, y_min, x_max, y_max), area) + + mask_a, bounds_a, area_a = geometry[detection_a.data[DETECTION_ID_KEY][0]] + mask_b, bounds_b, area_b = geometry[detection_b.data[DETECTION_ID_KEY][0]] + if not area_a or not area_b: + return 0.0 + + x_min, y_min = max(bounds_a[0], bounds_b[0]), max(bounds_a[1], bounds_b[1]) + x_max, y_max = min(bounds_a[2], bounds_b[2]), min(bounds_a[3], bounds_b[3]) + intersection = int( + np.count_nonzero( + np.logical_and( + mask_a[y_min:y_max, x_min:x_max], mask_b[y_min:y_max, x_min:x_max] + ) + ) + ) + union = area_a + area_b - intersection + return float(intersection) / float(union) + + return _calculate_pair_iou + + +def _mask_pair_iou(detection_a: sv.Detections, detection_b: sv.Detections) -> float: + """Compare full foreground masks; detector boxes may not contain them.""" + mask_a, mask_b = detection_a.mask[0], detection_b.mask[0] + intersection = np.logical_and(mask_a, mask_b).sum() + union = np.logical_or(mask_a, mask_b).sum() + if union == 0: + return 0.0 + return float(intersection) / float(union) + + +def _oriented_box_pair_iou(corners_a: np.ndarray, corners_b: np.ndarray) -> float: + """Exact oriented-box IoU for a pair of (4, 2) corner arrays. + + Computed via polygon intersection rather than coordinate-dependent rasterization. + """ + polygon_a = Polygon(corners_a) + polygon_b = Polygon(corners_b) + if not polygon_a.is_valid or not polygon_b.is_valid: + return 0.0 + intersection = polygon_a.intersection(polygon_b).area + union = polygon_a.area + polygon_b.area - intersection + if union <= 0: + return 0.0 + return float(intersection) / float(union) + + +def calculate_iou(detection_a: sv.Detections, detection_b: sv.Detections) -> float: + """Compute IoU on the geometry the detections actually carry. + + Dispatch order: segmentation masks when both sides have them, oriented + box corners (``data["xyxyxyxy"]``) when both sides have them, and the + axis-aligned ``xyxy`` boxes otherwise. The v1 block always compared + ``xyxy``, which over-matches detections whose boxes overlap even though + their masks or rotated bodies do not. + + Mask pixels can extend beyond detector boxes, so masks are compared before + the box gate. Oriented corners are contained in their axis-aligned bounds. + + Args: + detection_a: One detection in the shared image frame. + detection_b: One candidate from another prediction source. + + Returns: + Intersection over union of the available geometry. + + Raises: + ValueError: For mixed native/ordinary or incomplete native geometry. + """ + native_a = _is_native(detection_a) + native_b = _is_native(detection_b) + if native_a or native_b: + if not (native_a and native_b): + raise ValueError("Native OBB consensus cannot mix ordinary detections") + for detection in (detection_a, detection_b): + if ( + ORIENTED_BOX_COORDINATES not in detection.data + or "heading" not in detection.data + ): + raise ValueError("Native OBB overlap requires corners and heading") + if not native_a and detection_a.mask is not None and detection_b.mask is not None: + return _mask_pair_iou(detection_a, detection_b) + box_iou = float(sv.box_iou_batch(detection_a.xyxy, detection_b.xyxy)[0][0]) + if math.isnan(box_iou) or box_iou == 0.0: + return 0.0 + if ( + ORIENTED_BOX_COORDINATES in detection_a.data + and ORIENTED_BOX_COORDINATES in detection_b.data + ): + corners_a = np.asarray( + detection_a.data[ORIENTED_BOX_COORDINATES], dtype=np.float64 + ).reshape(4, 2) + corners_b = np.asarray( + detection_b.data[ORIENTED_BOX_COORDINATES], dtype=np.float64 + ).reshape(4, 2) + return _oriented_box_pair_iou(corners_a, corners_b) + return box_iou + + +def merge_detections( + detections, + confidence_aggregation_mode, + boxes_aggregation_mode, + mask_aggregation_mode, +): + """Merge ordinary predictions or retain one native class contributor. + + Args: + detections: Matched single-image predictions. + confidence_aggregation_mode: Existing class and confidence selection policy. + boxes_aggregation_mode: Ordinary AABB aggregation policy. + mask_aggregation_mode: Ordinary mask aggregation policy. + + Returns: + One consensus detection with native contributor geometry kept atomically. + """ + if not _is_native(detections): + result = v1.merge_detections( + detections, + confidence_aggregation_mode, + boxes_aggregation_mode, + mask_aggregation_mode, + ) + return result + + serialise_native_obb(detections) + class_name, class_id = v1.AGGREGATION_MODE2CLASS_SELECTOR[ + confidence_aggregation_mode + ](detections) + indices = np.flatnonzero( + (detections.class_id == class_id) & (detections["class_name"] == class_name) + ) + order = detections.data.get(_SOURCE_ORDER, np.arange(len(detections))) + winner_index = indices[ + np.lexsort((order[indices], -detections.confidence[indices]))[0] + ] + winner = detections[int(winner_index)] + winner.confidence = np.array( + [ + v1.aggregate_field_values( + detections=detections, + field="confidence", + aggregation_mode=confidence_aggregation_mode, + ) + ], + dtype=np.float64, + ) + return winner diff --git a/workflows/roboflow_workflows/core_steps/loader.py b/workflows/roboflow_workflows/core_steps/loader.py index 579fc2bbb9..8f49e1a58a 100644 --- a/workflows/roboflow_workflows/core_steps/loader.py +++ b/workflows/roboflow_workflows/core_steps/loader.py @@ -5,6 +5,14 @@ from roboflow_workflows.core_steps.analytics.data_aggregator.v1 import ( DataAggregatorBlockV1, ) +from roboflow_workflows.core_steps.common.oriented import ( + deserialize_native_obb, + serialise_native_obb, +) +from roboflow_workflows.core_steps.models.roboflow.oriented_object_detection.v1 import ( + NativeOBBVisualizationBlockV1, + RoboflowOrientedObjectDetectionModelBlockV1, +) from roboflow_workflows.environment import ( ALLOW_WORKFLOW_BLOCKS_ACCESSING_ENVIRONMENTAL_VARIABLES, ALLOW_WORKFLOW_BLOCKS_ACCESSING_LOCAL_STORAGE, @@ -17,6 +25,9 @@ WORKFLOWS_INNER_WORKFLOW_REMOTE_TARGET, WORKFLOWS_STEP_EXECUTION_MODE, ) +from roboflow_workflows.execution_engine.entities.types import ( + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, +) if not ENABLE_TENSOR_DATA_REPRESENTATION: from roboflow_workflows.core_steps.analytics.detection_event_log.v1 import ( @@ -278,6 +289,7 @@ TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_KEYPOINT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_QR_CODE_DETECTION_KIND, TENSOR_NATIVE_RLE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_SEMANTIC_SEGMENTATION_PREDICTION_KIND, @@ -391,6 +403,10 @@ from roboflow_workflows.core_steps.fusion.detections_consensus.v1_tensor import ( DetectionsConsensusBlockV1, ) +from roboflow_workflows.core_steps.fusion.detections_consensus.v2 import ( + DetectionsConsensusBlockV2, +) + if not ENABLE_TENSOR_DATA_REPRESENTATION: from roboflow_workflows.core_steps.fusion.detections_list_rollup.v1 import ( DetectionsListRollUpBlockV1, @@ -1612,6 +1628,7 @@ } KINDS_SERIALIZERS = { + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND.name: serialise_native_obb, IMAGE_KIND.name: serialise_image, VIDEO_METADATA_KIND.name: serialize_video_metadata_kind, ACTION_RECOGNITION_PREDICTION_KIND.name: serialize_action_recognition_prediction_kind, @@ -1660,6 +1677,7 @@ # swap above already selects serializers_tensor.serialize_wildcard_kind, whose # native arms serialise native values routed to `*` outputs to the standard dicts. KINDS_DESERIALIZERS = { + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND.name: deserialize_native_obb, IMAGE_KIND.name: deserialize_image_kind, VIDEO_METADATA_KIND.name: deserialize_video_metadata_kind, ACTION_RECOGNITION_PREDICTION_KIND.name: deserialize_action_recognition_prediction_kind, @@ -1770,6 +1788,8 @@ def _should_filter_block(block_class: Type[WorkflowBlock]) -> bool: def load_blocks() -> List[Type[WorkflowBlock]]: blocks = [ + RoboflowOrientedObjectDetectionModelBlockV1, + NativeOBBVisualizationBlockV1, AbsoluteStaticCropBlockV1, DynamicCropBlockV1, DetectionsFilterBlockV1, @@ -1836,6 +1856,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: CornerVisualizationBlockV1, CropVisualizationBlockV1, DetectionsConsensusBlockV1, + DetectionsConsensusBlockV2, DetectionsDifferenceBlockV1, DetectionsStitchBlockV1, OverlapAnalysisBlockV1, @@ -2034,6 +2055,11 @@ def load_blocks() -> List[Type[WorkflowBlock]]: def load_kinds() -> List[Kind]: return [ + ( + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND + if not ENABLE_TENSOR_DATA_REPRESENTATION + else TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND + ), WILDCARD_KIND, IMAGE_KIND, VIDEO_METADATA_KIND, diff --git a/workflows/roboflow_workflows/core_steps/models/roboflow/oriented_object_detection/__init__.py b/workflows/roboflow_workflows/core_steps/models/roboflow/oriented_object_detection/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/workflows/roboflow_workflows/core_steps/models/roboflow/oriented_object_detection/v1.py b/workflows/roboflow_workflows/core_steps/models/roboflow/oriented_object_detection/v1.py new file mode 100644 index 0000000000..20c933a3f4 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/models/roboflow/oriented_object_detection/v1.py @@ -0,0 +1,290 @@ +"""Native OBB model block with the same explicit task identity as the server.""" + +from typing import List, Literal, Optional, Union + +from pydantic import ConfigDict, Field +from roboflow_workflows.core_steps.common.entities import StepExecutionMode +from roboflow_workflows.core_steps.common.oriented import ( + native_obb_from_inference, + native_obb_to_tensor, +) +from roboflow_workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_batch_of_sv_detections, +) +from roboflow_workflows.environment import ( + ENABLE_TENSOR_DATA_REPRESENTATION, + LOCAL_INFERENCE_API_URL, +) +from roboflow_workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from roboflow_workflows.execution_engine.entities.types import ( + IMAGE_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, + ROBOFLOW_MODEL_ID_KIND, + ImageInputField, + RoboflowModelField, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import ( + DependentResource, + WorkflowBlock, + WorkflowBlockManifest, + roboflow_platform_model, +) +from roboflow_workflows.prototypes.models_provider import ModelsProvider + +from inference_sdk import InferenceConfiguration, InferenceHTTPClient + + +class BlockManifest(WorkflowBlockManifest): + """Native OBB inputs and output kind, without ordinary detection compatibility.""" + + model_config = ConfigDict( + json_schema_extra={ + "name": "Oriented Object Detection Model", + "version": "v1", + "block_type": "model", + "short_description": "Predict native rectangles and class-specific fronts.", + "license": "Apache-2.0", + } + ) + type: Literal["roboflow_core/roboflow_oriented_object_detection_model@v1"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_id: Union[Selector(kind=[ROBOFLOW_MODEL_ID_KIND]), str] = RoboflowModelField + confidence: float = Field( + default=0.5, ge=0, le=1, description="Minimum sigmoid class confidence." + ) + max_detections: int = Field( + default=300, gt=0, description="Maximum native detections per image." + ) + class_filter: Optional[List[str]] = Field( + default=None, description="Optional frozen class names to retain." + ) + + @classmethod + def get_compatible_task_types(cls): + return ["oriented-object-detection"] + + @classmethod + def get_parameters_accepting_batches(cls): + return ["images"] + + @classmethod + def describe_outputs(cls): + return [ + OutputDefinition( + name="predictions", kind=[ORIENTED_OBJECT_DETECTION_PREDICTION_KIND] + ) + ] + + @classmethod + def get_execution_engine_compatibility(cls): + return ">=1.3.0,<2.0.0" + + def discover_work_operations(self) -> List[WorkOperation]: + """Declare native model inference. + + Returns: + The work performed by this block. + """ + return [WorkOperation.MODEL_INFERENCE] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + """Declare unrestricted execution. + + Args: + ignore_environment_restrictions: Whether to omit host restrictions. + + Returns: + A complete empty restriction declaration. + """ + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + def discover_dependent_resources(self) -> List[DependentResource]: + """Declare the selected native model. + + Returns: + The model dependency, retaining selectors for discovery. + """ + return [roboflow_platform_model(model_id=self.model_id)] + + +class RoboflowOrientedObjectDetectionModelBlockV1(WorkflowBlock): + """Execute the native model and preserve geometry/front in Workflow transport.""" + + def __init__(self, model_manager: ModelsProvider, api_key, step_execution_mode): + self._model_manager = model_manager + self._api_key = api_key + self._mode = step_execution_mode + + @classmethod + def get_init_parameters(cls): + return ["model_manager", "api_key", "step_execution_mode"] + + @classmethod + def get_manifest(cls): + return BlockManifest + + def run( + self, + images: Batch[WorkflowImageData], + model_id, + confidence, + max_detections, + class_filter, + ): + """Infer and attach the exact image/crop lineage to native detections. + + Args: + images: Workflow input images or crops. + model_id: Native registered model version. + confidence: Numeric sigmoid threshold. + max_detections: Per-image output cap. + class_filter: Optional accepted class names. + + Returns: + Native prediction output for each input image. + """ + if self._mode is StepExecutionMode.LOCAL: + self._model_manager.add_model(model_id=model_id, api_key=self._api_key) + responses = self._model_manager.run_oriented_object_detection( + model_id=model_id, + images=[i.to_inference_format(numpy_preferred=True) for i in images], + api_key=self._api_key, + confidence=confidence, + max_detections=max_detections, + class_filter=class_filter, + ) + elif self._mode is StepExecutionMode.REMOTE: + client = InferenceHTTPClient( + api_url=LOCAL_INFERENCE_API_URL, api_key=self._api_key + ) + client.select_api_v1() + client.configure( + InferenceConfiguration( + confidence_threshold=confidence, + max_detections=max_detections, + class_filter=class_filter, + client_downsizing_disabled=True, + ) + ) + responses = client.infer( + [i.base64_image for i in images], model_id=model_id + ) + responses = responses if isinstance(responses, list) else [responses] + else: + raise ValueError("Unknown native OBB step execution mode") + detections = [native_obb_from_inference(r) for r in responses] + detections = attach_parents_coordinates_to_batch_of_sv_detections( + images=images, predictions=detections + ) + if ENABLE_TENSOR_DATA_REPRESENTATION: + detections = [native_obb_to_tensor(d) for d in detections] + return [{"predictions": d} for d in detections] + + +class NativeOBBVisualizationManifest(WorkflowBlockManifest): + """Draw native footprints and fronts without axis-aligned replacement.""" + + model_config = ConfigDict( + json_schema_extra={ + "name": "Oriented Box Visualization", + "version": "v1", + "block_type": "visualization", + "short_description": "Draw oriented footprints and class-specific front edges.", + "license": "Apache-2.0", + } + ) + type: Literal["roboflow_core/oriented_box_visualization@v1"] + image: Selector(kind=[IMAGE_KIND]) = Field( + description="Image in the same frame as the detections." + ) + predictions: Selector(kind=[ORIENTED_OBJECT_DETECTION_PREDICTION_KIND]) = Field( + description="Native rectangles or ordered quadrilaterals." + ) + thickness: int = Field(default=2, gt=0, description="Outline thickness in pixels.") + + @classmethod + def describe_outputs(cls): + return [OutputDefinition(name="image", kind=[IMAGE_KIND])] + + @classmethod + def get_execution_engine_compatibility(cls): + return ">=1.3.0,<2.0.0" + + def discover_work_operations(self) -> List[WorkOperation]: + """Declare native footprint visualization. + + Returns: + The work performed by this block. + """ + return [WorkOperation.VISUALIZATION] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + """Declare unrestricted execution. + + Args: + ignore_environment_restrictions: Whether to omit host restrictions. + + Returns: + A complete empty restriction declaration. + """ + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + def discover_dependent_resources(self) -> List[DependentResource]: + """Declare that no external resources are required. + + Returns: + A complete empty resource declaration. + """ + return [] + + +class NativeOBBVisualizationBlockV1(WorkflowBlock): + """Render the current ordered corner carrier in either representation.""" + + @classmethod + def get_manifest(cls): + return NativeOBBVisualizationManifest + + def run(self, image, predictions, thickness): + """Draw a copy of the input image and return it. + + Args: + image: Workflow image. + predictions: Native prediction carrier. + thickness: Outline width. + + Returns: + Annotated image output. + """ + from roboflow_workflows.core_steps.common.oriented import ( + _native_obb_to_sv, + annotate_native_obb, + ) + + scene = image.numpy_image.copy() + annotated = annotate_native_obb( + scene, _native_obb_to_sv(predictions), thickness=thickness + ) + return { + "image": WorkflowImageData.copy_and_replace( + origin_image_data=image, numpy_image=annotated + ) + } diff --git a/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1.py b/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1.py index 8dc4c3c411..5068702ff6 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1.py +++ b/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1.py @@ -8,6 +8,7 @@ from roboflow_workflows.execution_engine.entities.types import ( INSTANCE_SEGMENTATION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, Selector, ) from roboflow_workflows.execution_engine.entities.workload import ( @@ -82,6 +83,7 @@ class BlockManifest(WorkflowBlockManifest): prediction_one: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, ] ) = Field( @@ -91,6 +93,7 @@ class BlockManifest(WorkflowBlockManifest): prediction_two: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, ] ) = Field( @@ -105,6 +108,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, ], ), @@ -139,4 +143,27 @@ def run( prediction_one: sv.Detections, prediction_two: sv.Detections, ) -> BlockResult: - return {"predictions": sv.Detections.merge([prediction_one, prediction_two])} + from roboflow_workflows.core_steps.common.oriented import merge_native_semantics + + class_heading = merge_native_semantics(prediction_one, prediction_two) + if class_heading is not None: + from roboflow_workflows.core_steps.common.oriented import ( + _native_obb_in_frame, + _native_obb_to_sv, + ) + + prediction_one = _native_obb_to_sv(prediction_one) + prediction_two = _native_obb_to_sv(prediction_two) + header = {**prediction_one.metadata, "class_heading": class_heading} + result = prediction_one[:0] + if len(prediction_one) or len(prediction_two): + result = sv.Detections.merge( + [ + _native_obb_in_frame(prediction_one, header), + _native_obb_in_frame(prediction_two, header), + ] + ) + result.metadata = header + else: + result = sv.Detections.merge([prediction_one, prediction_two]) + return {"predictions": result} diff --git a/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1_tensor.py b/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1_tensor.py index 5eb291422a..4b01515711 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1_tensor.py +++ b/workflows/roboflow_workflows/core_steps/transformations/detections_combine/v1_tensor.py @@ -15,7 +15,10 @@ TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, ) -from roboflow_workflows.execution_engine.entities.types import Selector +from roboflow_workflows.execution_engine.entities.types import ( + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, + Selector, +) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, RuntimeRestriction, @@ -92,6 +95,7 @@ class BlockManifest(WorkflowBlockManifest): prediction_one: Selector( kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, ] ) = Field( @@ -101,6 +105,7 @@ class BlockManifest(WorkflowBlockManifest): prediction_two: Selector( kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, ] ) = Field( @@ -115,6 +120,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, ], ), @@ -267,6 +273,12 @@ def _combine_image_metadata( if base is None: return None image_metadata = dict(base) + from roboflow_workflows.core_steps.common.oriented import merge_native_semantics + + class_heading = merge_native_semantics(prediction_one, prediction_two) + if class_heading is not None: + image_metadata["class_heading"] = class_heading + image_metadata["native_obb"] = True image_metadata[CLASS_NAMES_KEY] = _merged_class_names( prediction_one=prediction_one, prediction_two=prediction_two, @@ -319,6 +331,15 @@ def run( prediction_one=prediction_one, prediction_two=prediction_two, ) + if (image_metadata or {}).get("native_obb"): + from roboflow_workflows.core_steps.common.oriented import _NATIVE_FRAME_KEYS + + for entry in bboxes_metadata or []: + for key in _NATIVE_FRAME_KEYS: + if key in image_metadata: + entry[key] = image_metadata[key] + elif key != "parent_id": + entry.pop(key, None) one_has_masks = isinstance(prediction_one, InstanceDetections) two_has_masks = isinstance(prediction_two, InstanceDetections) diff --git a/workflows/roboflow_workflows/core_steps/transformations/detections_filter/v1.py b/workflows/roboflow_workflows/core_steps/transformations/detections_filter/v1.py index f202f258e9..49125b3d9d 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/detections_filter/v1.py +++ b/workflows/roboflow_workflows/core_steps/transformations/detections_filter/v1.py @@ -14,6 +14,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, Selector, ) from roboflow_workflows.execution_engine.entities.workload import ( @@ -116,6 +117,7 @@ class BlockManifest(WorkflowBlockManifest): predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ] @@ -155,6 +157,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], diff --git a/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1.py b/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1.py index 4ed1a7bcde..e490419e1f 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1.py +++ b/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1.py @@ -5,6 +5,10 @@ import numpy as np import supervision as sv from pydantic import AliasChoices, ConfigDict, Field +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB +from roboflow_workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_sv_detections, +) from roboflow_workflows.execution_engine.constants import ( DETECTION_ID_KEY, KEYPOINTS_XY_KEY_IN_SV_DETECTIONS, @@ -21,6 +25,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, RGB_COLOR_KIND, STRING_KIND, Selector, @@ -104,6 +109,7 @@ class BlockManifest(WorkflowBlockManifest): predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ] @@ -165,6 +171,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -220,6 +227,21 @@ def crop_image( background_color: Union[str, Tuple[int, int, int]], detection_id_key: str = DETECTION_ID_KEY, ) -> List[Dict[str, any]]: + """Crop each detection and translate its geometry to the extraction origin. + + Args: + image: Source image and lineage. + detections: Predictions defining the crop windows and identities. + mask_opacity: Background-removal strength for segmentation predictions. + background_color: Background color used by mask overlays. + detection_id_key: Metadata key identifying each crop's parent detection. + + Returns: + Crop images and translated predictions, or null entries for empty windows. + + Raises: + ValueError: If detections lack crop identities. + """ if len(detections) == 0: return [] if detection_id_key not in detections.data: @@ -227,10 +249,18 @@ def crop_image( f"sv.Detections object passed to crop step do not fulfill contract - lack of {detection_id_key} key " f"in data dictionary." ) + native = bool(detections.metadata.get(NATIVE_OBB)) or np.any( + detections.data.get(NATIVE_OBB, False) + ) crops = [] for idx, ((x_min, y_min, x_max, y_max), detection_id) in enumerate( zip(detections.xyxy.round().astype(dtype=int), detections[detection_id_key]) ): + if native: + # Clip the extraction window, retaining the complete ordered geometry. + height, width = image.numpy_image.shape[:2] + x_min, x_max = np.clip([x_min, x_max], 0, width) + y_min, y_max = np.clip([y_min, y_max], 0, height) cropped_image = image.numpy_image[y_min:y_max, x_min:x_max] if not cropped_image.size: crops.append({"crops": None, "predictions": None}) @@ -284,6 +314,13 @@ def crop_image( ORIENTED_BOX_COORDINATES ] - np.array([x_min, y_min]) + if translated_detection.metadata.get(NATIVE_OBB): + dimensions = list(result.numpy_image.shape[:2]) + translated_detection.metadata["image_dimensions"] = dimensions + translated_detection.data["image_dimensions"] = np.asarray([dimensions]) + translated_detection = attach_parents_coordinates_to_sv_detections( + detections=translated_detection, image=result + ) crops.append( { "crops": result, diff --git a/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1_tensor.py b/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1_tensor.py index ede567678a..fda7b20b08 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1_tensor.py +++ b/workflows/roboflow_workflows/core_steps/transformations/dynamic_crop/v1_tensor.py @@ -15,8 +15,10 @@ import numpy as np import torch from pydantic import AliasChoices, ConfigDict, Field +from roboflow_workflows.core_steps.common.oriented import _NATIVE_FRAME_KEYS, NATIVE_OBB from roboflow_workflows.core_steps.common.tensor_native import ( HOST_MIRROR_KEYS, + build_native_image_metadata, take_prediction_by_indices, ) from roboflow_workflows.execution_engine.constants import ( @@ -37,6 +39,7 @@ from roboflow_workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, IMAGE_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, RGB_COLOR_KIND, STRING_KIND, Selector, @@ -133,6 +136,7 @@ class BlockManifest(WorkflowBlockManifest): predictions: Selector( kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_KEYPOINT_DETECTION_PREDICTION_KIND, ] @@ -194,6 +198,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -309,6 +314,20 @@ def crop_image( x_max=x_max, y_max=y_max, ) + if (bbox_detections.image_metadata or {}).get(NATIVE_OBB): + translated_prediction.image_metadata.update( + build_native_image_metadata( + cropped, + class_names=translated_prediction.image_metadata.get( + "class_names", {} + ), + prediction_type="oriented-object-detection", + ) + ) + for entry in translated_prediction.bboxes_metadata or []: + for key in _NATIVE_FRAME_KEYS: + if key in translated_prediction.image_metadata: + entry[key] = deepcopy(translated_prediction.image_metadata[key]) crops.append( { "crops": cropped, diff --git a/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1.py b/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1.py index 3a6d9112e9..4befa4d585 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1.py +++ b/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1.py @@ -10,6 +10,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, Selector, ) from roboflow_workflows.execution_engine.entities.workload import ( @@ -80,6 +81,7 @@ class BlockManifest(WorkflowBlockManifest): predictions: Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ] @@ -111,6 +113,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, ], diff --git a/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1_tensor.py b/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1_tensor.py index f2d77c6bc8..4d03d18428 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1_tensor.py +++ b/workflows/roboflow_workflows/core_steps/transformations/per_class_confidence_filter/v1_tensor.py @@ -2,6 +2,7 @@ import torch from pydantic import ConfigDict, Field +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.core_steps.common.tensor_native import ( TensorNativeDetections, TensorNativePrediction, @@ -20,6 +21,7 @@ from roboflow_workflows.execution_engine.entities.types import ( DICTIONARY_KIND, FLOAT_ZERO_TO_ONE_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, Selector, ) from roboflow_workflows.execution_engine.entities.workload import ( @@ -90,6 +92,7 @@ class BlockManifest(WorkflowBlockManifest): predictions: Selector( kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_KEYPOINT_DETECTION_PREDICTION_KIND, ] @@ -121,6 +124,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name="predictions", kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_KEYPOINT_DETECTION_PREDICTION_KIND, ], @@ -209,7 +213,11 @@ def _resolve_class_names( ) -> List[Optional[str]]: """Per-box class name, preferring the ``image_metadata[CLASS_NAMES_KEY]`` (class_id -> name) map and falling back to ``bboxes_metadata[i]["class"]`` - when the map is absent or lacks the id.""" + when the map is absent or lacks the id. Native OBB uses authoritative row + names because numeric class IDs can collide across models.""" + if (detections.image_metadata or {}).get(NATIVE_OBB): + return [entry["class_name"] for entry in detections.bboxes_metadata or []] + number_of_detections = int(detections.confidence.shape[0]) name_by_class_id: Dict[int, str] = {} if detections.image_metadata: diff --git a/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1.py b/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1.py index 914c415bed..3d8b5a233c 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1.py +++ b/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1.py @@ -5,6 +5,7 @@ import numpy as np import supervision as sv from pydantic import AliasChoices, ConfigDict, Field +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.execution_engine.constants import ( KEYPOINTS_XY_KEY_IN_SV_DETECTIONS, ) @@ -20,6 +21,7 @@ INTEGER_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, Selector, ) @@ -131,6 +133,7 @@ class PerspectiveCorrectionManifest(WorkflowBlockManifest): Selector( kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, ] ) @@ -195,6 +198,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name=OUTPUT_DETECTIONS_KEY, kind=[ OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, ], ), @@ -691,6 +695,28 @@ def correct_detections( transformed_rect_width: float, transformed_rect_height: float, ) -> sv.Detections: + """Warp detections into the destination perspective frame. + + Args: + detections: Predictions in the source frame. + perspective_transformer: Source-to-destination homography. + transformed_rect_width: Destination width in pixels. + transformed_rect_height: Destination height in pixels. + + Returns: + Warped predictions; native outputs retain policy and destination dimensions. + """ + if detections.metadata.get(NATIVE_OBB): + detections = detections[:] + dimensions = [ + int(round(transformed_rect_height)), + int(round(transformed_rect_width)), + ] + detections.metadata["image_dimensions"] = dimensions + detections.data["image_dimensions"] = np.tile(dimensions, (len(detections), 1)) + if not len(detections): + return detections + corrected_detections: List[sv.Detections] = [] for i in range(len(detections)): # copy @@ -757,9 +783,17 @@ def correct_detections( ), m=perspective_transformer, ).reshape(-1, 2) - detection[ORIENTED_BOX_COORDINATES] = np.array( - [np.around(corrected_obb).astype(np.float32)] + corrected_corners = ( + corrected_obb.astype(np.float64) + if detection.metadata.get(NATIVE_OBB) + else np.around(corrected_obb).astype(np.float32) ) + detection[ORIENTED_BOX_COORDINATES] = np.array([corrected_corners]) + if detection.metadata.get(NATIVE_OBB): + detection.xyxy = np.concatenate( + (corrected_corners.min(0), corrected_corners.max(0)) + )[None] + corrected_detections.append(detection) return sv.Detections.merge(corrected_detections) diff --git a/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1_tensor.py b/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1_tensor.py index 67790a5701..f0bea4eeb3 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1_tensor.py +++ b/workflows/roboflow_workflows/core_steps/transformations/perspective_correction/v1_tensor.py @@ -45,6 +45,7 @@ import supervision as sv import torch from pydantic import AliasChoices, ConfigDict, Field +from roboflow_workflows.core_steps.common.oriented import NATIVE_OBB from roboflow_workflows.core_steps.common.tensor_native import ( attach_native_detection_metadata, instance_mask_to_numpy, @@ -64,6 +65,7 @@ from roboflow_workflows.execution_engine.entities.tensor_native_types import ( TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, ) from roboflow_workflows.execution_engine.entities.types import ( BOOLEAN_KIND, @@ -193,6 +195,7 @@ class PerspectiveCorrectionManifest(WorkflowBlockManifest): Selector( kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, ] ) @@ -257,6 +260,7 @@ def describe_outputs(cls) -> List[OutputDefinition]: name=OUTPUT_DETECTIONS_KEY, kind=[ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, + TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, TENSOR_NATIVE_INSTANCE_SEGMENTATION_PREDICTION_KIND, ], ), @@ -982,7 +986,11 @@ def _correct_oriented_box_in_metadata( src=np.array([np.asarray(oriented_box)], dtype=np.float32), m=perspective_transformer, ).reshape(-1, 2) - entry[ORIENTED_BOX_COORDINATES] = np.around(corrected_obb).astype(np.float32) + entry[ORIENTED_BOX_COORDINATES] = ( + corrected_obb.astype(np.float64) + if entry.get(NATIVE_OBB) + else np.around(corrected_obb).astype(np.float32) + ) def _correct_native_detections( @@ -1050,6 +1058,9 @@ def _correct_native_detections( _correct_oriented_box_in_metadata( entry=entry, perspective_transformer=perspective_transformer ) + if (detections.image_metadata or {}).get(NATIVE_OBB): + corners = entry[ORIENTED_BOX_COORDINATES] + corrected_xyxy_rows[-1] = np.concatenate((corners.min(0), corners.max(0))) corrected_bboxes_metadata.append(entry) if number_of_detections > 0: @@ -1097,6 +1108,11 @@ def _correct_native_detections( class_names=class_names, prediction_type=prediction_type, ) + if (detections.image_metadata or {}).get(NATIVE_OBB): + corrected.image_metadata.update( + native_obb=True, + class_heading=dict(detections.image_metadata["class_heading"]), + ) # The corrected boxes/masks live in the destination (warped) frame, not the # source image. ``attach_native_detection_metadata`` reports the source image # dimensions, so override IMAGE_DIMENSIONS_KEY with the destination rect size diff --git a/workflows/roboflow_workflows/execution_engine/entities/tensor_native_types.py b/workflows/roboflow_workflows/execution_engine/entities/tensor_native_types.py index e8d8f827fc..8397ea56a8 100644 --- a/workflows/roboflow_workflows/execution_engine/entities/tensor_native_types.py +++ b/workflows/roboflow_workflows/execution_engine/entities/tensor_native_types.py @@ -1,4 +1,7 @@ -from roboflow_workflows.execution_engine.entities.types import Kind +from roboflow_workflows.execution_engine.entities.types import ( + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, + Kind, +) TENSOR_NATIVE_EMBEDDING_KIND_DOCS = """ This kind represents a vector embedding. It is a list of floating point numbers. @@ -740,3 +743,10 @@ serialised_data_type="list", internal_data_type="torch.Tensor", ) + + +TENSOR_NATIVE_ORIENTED_OBJECT_DETECTION_PREDICTION_KIND = ( + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND.model_copy( + update={"internal_data_type": "inference_models.Detections"} + ) +) diff --git a/workflows/roboflow_workflows/execution_engine/entities/types.py b/workflows/roboflow_workflows/execution_engine/entities/types.py index c910307fb0..f2dcc2e6bc 100644 --- a/workflows/roboflow_workflows/execution_engine/entities/types.py +++ b/workflows/roboflow_workflows/execution_engine/entities/types.py @@ -1436,3 +1436,12 @@ def Selector( ), Field(json_schema_extra=json_schema_extra), ] + + +ORIENTED_OBJECT_DETECTION_PREDICTION_KIND = Kind( + name="oriented_object_detection_prediction", + description="Native OBB with ordered float corners and frozen class front semantics.", + docs="Native OBB detections use original-image pixel coordinates and clockwise radians. Inside Workflows ordered corners preserve front through crops and warps. A perspective warp can emit an explicit quadrilateral; it is never silently refitted into a native rectangle.", + serialised_data_type="dict", + internal_data_type="sv.Detections", +) diff --git a/workflows/roboflow_workflows/execution_engine/v1/introspection/kinds_schemas_register.py b/workflows/roboflow_workflows/execution_engine/v1/introspection/kinds_schemas_register.py index d836d6639e..8277c41f5a 100644 --- a/workflows/roboflow_workflows/execution_engine/v1/introspection/kinds_schemas_register.py +++ b/workflows/roboflow_workflows/execution_engine/v1/introspection/kinds_schemas_register.py @@ -1,3 +1,6 @@ +from roboflow_workflows.core_steps.common.oriented_entities import ( + OrientedWorkflowInferenceResponse, +) from roboflow_workflows.execution_engine.entities.base import VideoMetadata from roboflow_workflows.execution_engine.entities.types import ( CLASSIFICATION_PREDICTION_KIND, @@ -6,6 +9,7 @@ INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND, QR_CODE_DETECTION_KIND, VIDEO_METADATA_KIND, ) @@ -21,6 +25,9 @@ ) KIND_TO_SCHEMA_REGISTER = { + ORIENTED_OBJECT_DETECTION_PREDICTION_KIND.name: OrientedWorkflowInferenceResponse.model_json_schema( + by_alias=True + ), IMAGE_KIND.name: ImageSchema.schema(), VIDEO_METADATA_KIND.name: VideoMetadata.schema(), IMAGE_KEYPOINTS_KIND.name: ImageKeyPoints.schema(), diff --git a/workflows/roboflow_workflows/prototypes/models_provider.py b/workflows/roboflow_workflows/prototypes/models_provider.py index 875f2e8105..3f872cc1df 100644 --- a/workflows/roboflow_workflows/prototypes/models_provider.py +++ b/workflows/roboflow_workflows/prototypes/models_provider.py @@ -96,6 +96,16 @@ def run_object_detection( active_learning_target_dataset: Optional[str] = None, ) -> List[dict]: ... + def run_oriented_object_detection( + self, + model_id: str, + images: List[Any], + api_key: Optional[str] = None, + confidence: float = 0.5, + max_detections: int = 300, + class_filter: Optional[List[str]] = None, + ) -> List[dict]: ... + def run_classification( self, model_id: str, diff --git a/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus.py b/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus.py index a8fcd62a7c..3b35405fcf 100644 --- a/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus.py +++ b/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus.py @@ -708,6 +708,8 @@ def test_get_union_mask() -> None: result = get_union_mask(detections=detections) # then + assert result.dtype == bool + np.testing.assert_array_equal(result, np.logical_or(mask1, mask2)) assert np.allclose( sv.mask_to_xyxy(np.array([result])), [[100, 100, 500, 500]], atol=1e-5 ) @@ -759,6 +761,8 @@ def test_get_intersection_mask() -> None: result = get_intersection_mask(detections=detections) # then + assert result.dtype == bool + np.testing.assert_array_equal(result, np.logical_and(mask1, mask2)) assert np.allclose( sv.mask_to_xyxy(np.array([result])), [[180, 150, 300, 300]], atol=1e-5 ) diff --git a/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus_v2.py b/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus_v2.py new file mode 100644 index 0000000000..82b8b39f29 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/fusion/test_detections_consensus_v2.py @@ -0,0 +1,334 @@ +from copy import deepcopy + +import numpy as np +import pytest +import supervision as sv +from pydantic import ValidationError +from roboflow_workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_sv_detections, +) +from roboflow_workflows.core_steps.fusion.detections_consensus.v2 import ( + AggregationMode, + BlockManifest, + DetectionsConsensusBlockV2, + MaskAggregationMode, + calculate_iou, +) +from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, +) +from supervision.config import ORIENTED_BOX_COORDINATES + + +def test_detections_consensus_v2_validation_when_valid_specification_given() -> None: + # given + specification = { + "type": "roboflow_core/detections_consensus@v2", + "name": "some", + "predictions": [ + "$steps.detection.predictions", + "$steps.detection_2.predictions", + ], + "image_metadata": "$steps.detection.image", + "required_votes": 3, + } + + # when + result = BlockManifest.model_validate(specification) + + # then + assert result == BlockManifest( + type="roboflow_core/detections_consensus@v2", + name="some", + predictions_batches=[ + "$steps.detection.predictions", + "$steps.detection_2.predictions", + ], + image_metadata="$steps.detection.image", + required_votes=3, + class_aware=True, + iou_threshold=0.3, + confidence=0.0, + classes_to_consider=None, + required_objects=None, + presence_confidence_aggregation=AggregationMode.MAX, + detections_merge_confidence_aggregation=AggregationMode.AVERAGE, + detections_merge_coordinates_aggregation=AggregationMode.AVERAGE, + ) + + +def test_detections_consensus_v2_validation_rejects_v1_type_alias() -> None: + # given + specification = { + "type": "roboflow_core/detections_consensus@v1", + "name": "some", + "predictions": ["$steps.detection.predictions"], + "image_metadata": "$steps.detection.image", + "required_votes": 1, + } + + # when + with pytest.raises(ValidationError): + _ = BlockManifest.model_validate(specification) + + +def _rotated_rect( + cx: float, cy: float, w: float, h: float, angle_deg: float +) -> np.ndarray: + angle = np.deg2rad(angle_deg) + cos, sin = np.cos(angle), np.sin(angle) + rot = np.array([[cos, -sin], [sin, cos]]) + corners = np.array( + [[-w / 2, -h / 2], [w / 2, -h / 2], [w / 2, h / 2], [-w / 2, h / 2]] + ) + return (corners @ rot.T + [cx, cy]).astype(np.float32) + + +def test_calculate_iou_uses_box_iou_when_no_richer_geometry_present() -> None: + # given - partially overlapping plain boxes, same expectation as the v1 + # behaviour: box A size = box B size = 800, intersection = 200, + # expected result = 200 / 1400 = 1 / 7 + detections = sv.Detections( + xyxy=np.array([[80, 190, 120, 210], [100, 200, 140, 220]], dtype=np.float64), + confidence=np.array([0.5, 0.6], dtype=np.float64), + class_id=np.array([1, 2]), + data={"class_name": np.array(["a", "b"])}, + ) + detection_a = detections[0] + detection_b = detections[1] + + # when + result = calculate_iou( + detection_a=detection_a, + detection_b=detection_b, + ) + + # then + assert abs(result - 1 / 7) < 1e-5 + + +def test_calculate_iou_uses_mask_iou_when_both_have_masks() -> None: + # given - identical AABB, disjoint masks. AABB IoU would be 1.0; + # mask IoU is 0. + mask_a = np.zeros((1, 10, 10), dtype=bool) + mask_a[0, 0:5, 0:5] = True + mask_b = np.zeros((1, 10, 10), dtype=bool) + mask_b[0, 5:10, 5:10] = True + xyxy = np.array([[0, 0, 10, 10]], dtype=np.float32) + + detection_a = sv.Detections(xyxy=xyxy, mask=mask_a) + detection_b = sv.Detections(xyxy=xyxy, mask=mask_b) + + # when + result = calculate_iou( + detection_a=detection_a, + detection_b=detection_b, + ) + + # then + assert result == 0.0 + + +def test_calculate_iou_uses_obb_iou_when_both_have_oriented_boxes() -> None: + # given - crossed thin rectangles at +/-45 deg: near-identical AABBs but + # barely-overlapping oriented bodies. + quad_a = _rotated_rect(50, 50, 100, 10, +45) + quad_b = _rotated_rect(50, 50, 100, 10, -45) + aabb = np.array( + [ + [ + quad_a[:, 0].min(), + quad_a[:, 1].min(), + quad_a[:, 0].max(), + quad_a[:, 1].max(), + ] + ], + dtype=np.float32, + ) + + detection_a = sv.Detections( + xyxy=aabb, data={ORIENTED_BOX_COORDINATES: quad_a[None, ...]} + ) + detection_b = sv.Detections( + xyxy=aabb, data={ORIENTED_BOX_COORDINATES: quad_b[None, ...]} + ) + + # when + result = calculate_iou( + detection_a=detection_a, + detection_b=detection_b, + ) + + # then + assert result < 0.2 + + +def _mask_sources(masks, *, boxes=None, detection_ids=None): + height, width = masks[0].shape[1:] + image = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="image"), + numpy_image=np.zeros((height, width, 3), dtype=np.uint8), + ) + sources = [] + for index, mask in enumerate(masks): + count = len(mask) + source = sv.Detections( + xyxy=np.array( + boxes[index] if boxes is not None else [[0, 0, width, height]] * count, + dtype=float, + ), + mask=mask, + class_id=np.zeros(count, dtype=int), + confidence=np.full(count, 0.9), + data={ + "class_name": np.full(count, "object"), + "detection_id": np.array( + detection_ids[index] + if detection_ids is not None + else [f"{index}:{row}" for row in range(count)] + ), + "image_dimensions": np.tile([height, width], (count, 1)), + }, + ) + attach_parents_coordinates_to_sv_detections(source, image) + sources.append(source) + return sources + + +def _run_mask_consensus(sources, *, block=None, **updates): + kwargs = dict( + predictions_batches=[[source] for source in sources], + required_votes=2, + class_aware=True, + iou_threshold=0.3, + confidence=0.0, + classes_to_consider=None, + required_objects=None, + presence_confidence_aggregation=AggregationMode.MAX, + detections_merge_confidence_aggregation=AggregationMode.AVERAGE, + detections_merge_coordinates_aggregation=AggregationMode.AVERAGE, + detections_merge_mask_aggregation=MaskAggregationMode.UNION, + ) + kwargs.update(updates) + return (block or DetectionsConsensusBlockV2()).run(**kwargs)[0] + + +@pytest.mark.parametrize( + "dtype, on_a, on_b", [(bool, True, True), (np.uint8, 255, 255), (np.uint8, 1, 2)] +) +@pytest.mark.parametrize( + "boxes", [([[0, 0, 2, 2]], [[0, 0, 2, 2]]), ([[0, 0, 2, 2]], [[8, 8, 10, 10]])] +) +def test_mask_pixels_outside_detector_boxes_determine_consensus( + dtype, on_a, on_b, boxes +): + mask_a = np.zeros((1, 10, 10), dtype=dtype) + mask_b = np.zeros((1, 10, 10), dtype=dtype) + mask_a[0, 2:6, 2:6] = on_a + mask_b[0, 2:6, 4:8] = on_b + sources = _mask_sources([mask_a, mask_b], boxes=boxes) + before = deepcopy(sources) + # Each mask has area 16, with eight shared pixels: IoU is 8 / 24. + assert calculate_iou(*sources) == pytest.approx(1 / 3) + for threshold, count in [(0.3, 1), (1 / 3, 0)]: + result = _run_mask_consensus(sources, iou_threshold=threshold) + assert len(result["predictions"]) == count + assert result["object_present"] is bool(count) + if count: + np.testing.assert_array_equal(result["predictions"].xyxy, [[2, 2, 7, 5]]) + expected_mask = np.zeros((1, 10, 10), dtype=bool) + expected_mask[0, 2:6, 2:8] = True + np.testing.assert_array_equal(result["predictions"].mask, expected_mask) + for original, source in zip(before, sources): + np.testing.assert_array_equal(source.xyxy, original.xyxy) + np.testing.assert_array_equal(source.mask, original.mask) + assert source.metadata == original.metadata + assert source.data.keys() == original.data.keys() + for key in original.data: + np.testing.assert_array_equal(source.data[key], original.data[key]) + + +@pytest.mark.parametrize("foreground", [False, True]) +@pytest.mark.parametrize("compact", [False, True]) +def test_mask_consensus_retains_last_canvas_pixel_and_empty_masks(foreground, compact): + mask = np.zeros((1, 10, 10), dtype=bool) + mask[0, 9, 9] = foreground + sources = _mask_sources([mask.copy(), mask.copy()]) + if compact: + for source in sources: + source.mask = sv.CompactMask.from_dense( + source.mask, sv.mask_to_xyxy(source.mask), (10, 10) + ) + result = _run_mask_consensus(sources) + assert len(result["predictions"]) == int(foreground) + if foreground: + np.testing.assert_array_equal(result["predictions"].xyxy, [[9, 9, 9, 9]]) + np.testing.assert_array_equal(result["predictions"].mask, mask) + + +def test_mask_consensus_recomputes_geometry_on_later_calls_with_the_same_ids(): + mask = np.zeros((1, 10, 10), dtype=bool) + mask[0, 2:6, 2:6] = True + sources = _mask_sources([mask.copy(), mask.copy()]) + block = DetectionsConsensusBlockV2() + first = _run_mask_consensus(sources, block=block)["predictions"] + np.testing.assert_array_equal(first.xyxy, [[2, 2, 5, 5]]) + for source in sources: + source.mask[:] = False + source.mask[0, 8:10, 8:10] = True + second = _run_mask_consensus(sources, block=block)["predictions"] + np.testing.assert_array_equal(second.xyxy, [[8, 8, 9, 9]]) + np.testing.assert_array_equal(second.mask, sources[0].mask) + + +@pytest.mark.parametrize("identity", ["duplicate", 7, np.nan]) +def test_aliased_or_legacy_detection_ids_keep_exact_disjoint_mask_votes(identity): + mask_a = np.zeros((2, 10, 10), dtype=bool) + mask_b = np.zeros((2, 10, 10), dtype=bool) + mask_a[0, 0:2, 0:2] = True + mask_a[1, 2:4, 2:4] = True + mask_b[0, 5:7, 5:7] = True + mask_b[1, 7:9, 7:9] = True + sources = _mask_sources([mask_a, mask_b], detection_ids=[[identity] * 2] * 2) + result = _run_mask_consensus(sources) + assert len(result["predictions"]) == 0 + assert result["object_present"] is False + + +class _UnreadMasks(np.ndarray): + """Fail if a no-candidate call starts reducing or comparing mask pixels.""" + + def any(self, *args, **kwargs): + raise AssertionError("Mask pixels were scanned without a candidate") + + def __array_function__(self, *args, **kwargs): + raise AssertionError("Mask pixels were scanned without a candidate") + + def __array_ufunc__(self, *args, **kwargs): + raise AssertionError("Mask pixels were scanned without a candidate") + + +@pytest.mark.parametrize("with_ids", [False, True]) +def test_mask_consensus_does_not_scan_masks_when_class_filter_removes_all_candidates( + with_ids, +): + sources = _mask_sources([np.ones((2, 10, 10), dtype=bool)] * 2) + for source in sources: + source.mask = source.mask.view(_UnreadMasks) + if not with_ids: + del source.data["detection_id"] + result = _run_mask_consensus(sources, classes_to_consider=["absent"]) + assert len(result["predictions"]) == 0 + assert result["object_present"] is False + + +def test_different_mask_canvases_keep_the_existing_numpy_merge_error(): + # NumPy overlap broadcasts the one-row canvas; merging incompatible canvases + # still raises rather than silently replacing that overlap with a zero vote. + mask_a = np.ones((1, 1, 10), dtype=bool) + mask_b = np.zeros((1, 10, 10), dtype=bool) + mask_b[0, 2:6, :] = True + sources = _mask_sources([mask_a, mask_b]) + with pytest.raises(ValueError): + _run_mask_consensus(sources) diff --git a/workflows/tests/unit_tests/core_steps/fusion/test_native_obb_consensus.py b/workflows/tests/unit_tests/core_steps/fusion/test_native_obb_consensus.py new file mode 100644 index 0000000000..86af058b4b --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/fusion/test_native_obb_consensus.py @@ -0,0 +1,693 @@ +import copy +import math + +import numpy as np +import pytest +import supervision as sv +from roboflow_workflows.core_steps.common.oriented import ( + native_obb_from_inference, + native_obb_to_tensor, + serialise_native_obb, +) +from roboflow_workflows.core_steps.fusion.detections_consensus.v2 import ( + AggregationMode, + DetectionsConsensusBlockV2, + MaskAggregationMode, +) + + +def _response( + *, + identity, + confidence=0.8, + angle=0.0, + sides=(12.0, 24.0), + name="plane", + heading=True +): + return { + "task_type": "oriented-object-detection", + "image": {"width": 100, "height": 80}, + "class_heading": {name: heading, "unused": False}, + "predictions": [ + { + "obb": [40.25, 30.75, *sides, angle], + "class": name, + "class_id": 7 if name == "plane" else 9, + "heading": heading, + "confidence": confidence, + "detection_id": identity, + "parent_id": "same-image", + } + ], + } + + +def _run(sources, *, tensor=False, aggregation=AggregationMode.AVERAGE, **updates): + predictions = [ + native_obb_from_inference(source) if isinstance(source, dict) else source + for source in sources + ] + if tensor: + predictions = [native_obb_to_tensor(source) for source in predictions] + kwargs = dict( + predictions_batches=[[source] for source in predictions], + required_votes=len(sources), + class_aware=False, + iou_threshold=0.1, + confidence=0.0, + classes_to_consider=None, + required_objects=None, + presence_confidence_aggregation=AggregationMode.MAX, + detections_merge_confidence_aggregation=aggregation, + detections_merge_coordinates_aggregation=AggregationMode.AVERAGE, + detections_merge_mask_aggregation=MaskAggregationMode.UNION, + ) + kwargs.update(updates) + return DetectionsConsensusBlockV2().run(**kwargs)[0]["predictions"] + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("sides", [(12.0, 24.0), (12.0, 12.0)]) +@pytest.mark.parametrize("aggregation", list(AggregationMode)) +def test_block_retains_opposite_front_contributor_atomically( + tensor, sides, aggregation +): + first = _response(identity="first", confidence=0.6, sides=sides) + winner = _response(identity="winner", confidence=0.9, sides=sides, angle=math.pi) + winner["class_heading"]["unobserved-from-winner"] = True + result = serialise_native_obb( + _run([first, winner], tensor=tensor, aggregation=aggregation) + ) + row = result["predictions"][0] + assert row["obb"] == pytest.approx(winner["predictions"][0]["obb"]) + assert row["detection_id"] == "winner" + assert row["heading"] is True + assert row["class_id"] == 7 + assert row["parent_id"] == "same-image" + assert row["confidence"] == pytest.approx( + { + AggregationMode.AVERAGE: 0.75, + AggregationMode.MAX: 0.9, + AggregationMode.MIN: 0.6, + }[aggregation] + ) + assert result["class_heading"] == { + **first["class_heading"], + **winner["class_heading"], + } + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize( + "aggregation,expected", + [ + (AggregationMode.AVERAGE, "plane-best"), + (AggregationMode.MIN, "plane-best"), + (AggregationMode.MAX, "stone"), + ], +) +def test_winner_is_highest_confidence_in_aggregated_class( + tensor, aggregation, expected +): + sources = [ + _response(identity="plane-first", confidence=0.3), + _response(identity="plane-best", confidence=0.6, angle=math.pi), + _response(identity="stone", confidence=0.99, name="stone", heading=False), + ] + row = serialise_native_obb(_run(sources, tensor=tensor, aggregation=aggregation))[ + "predictions" + ][0] + assert row["detection_id"] == expected + assert row["heading"] is (expected != "stone") + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_ties_keep_source_order_and_inputs_are_unmodified(tensor): + sources = [_response(identity="first"), _response(identity="second", angle=math.pi)] + before = copy.deepcopy(sources) + row = serialise_native_obb(_run(sources, tensor=tensor))["predictions"][0] + assert row["detection_id"] == "first" + assert sources == before + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("missing", ["xyxyxyxy", "heading"]) +def test_block_rejects_missing_native_geometry_before_voting(tensor, missing): + sources = [ + native_obb_from_inference(_response(identity=str(index))) for index in range(2) + ] + del sources[1].data[missing] + with pytest.raises(ValueError, match="corners and heading"): + _run(sources, tensor=tensor, required_votes=3) + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_block_rejects_conflicting_unobserved_frozen_policy(tensor): + sources = [_response(identity="first"), _response(identity="second")] + sources[1]["class_heading"]["unused"] = True + with pytest.raises(ValueError, match="Conflicting native class heading"): + _run(sources, tensor=tensor) + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_empty_consensus_retains_native_task_and_entire_frozen_policy(tensor): + sources = [_response(identity="first"), _response(identity="second")] + result = serialise_native_obb(_run(sources, tensor=tensor, required_votes=3)) + assert result["predictions"] == [] + assert result["class_heading"] == sources[0]["class_heading"] + assert result["task_type"] == "oriented-object-detection" + + +def test_block_rejects_native_and_ordinary_even_when_classes_do_not_match(): + native = native_obb_from_inference(_response(identity="native")) + ordinary = sv.Detections( + xyxy=np.array([[0.0, 0.0, 10.0, 10.0]]), + class_id=np.array([1]), + confidence=np.array([0.9]), + data={"class_name": np.array(["other"])}, + ) + with pytest.raises(ValueError, match="mix ordinary"): + _run([native, ordinary], class_aware=True) + + +def test_v2_explicitly_rejects_ordinary_tensor_carrier(): + import torch + + from inference_models.models.base.object_detection import Detections + + ordinary = Detections( + xyxy=torch.tensor([[0.0, 0.0, 10.0, 10.0]]), + class_id=torch.tensor([1]), + confidence=torch.tensor([0.9]), + ) + with pytest.raises(ValueError, match="ordinary tensor consensus is unsupported"): + _run([ordinary, ordinary]) + + +@pytest.mark.parametrize("sides", [(12.0, 24.0), (12.0, 12.0)]) +def test_serialized_workflow_native_consensus_and_combination_preserve_policy(sides): + from roboflow_workflows.execution_engine.core import ExecutionEngine + + workflow = { + "version": "1.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": name, + "kind": ["oriented_object_detection_prediction"], + } + for name in ("first", "second") + ], + "steps": [ + { + "type": "roboflow_core/detections_consensus@v2", + "name": "consensus", + "predictions_batches": ["$inputs.first", "$inputs.second"], + "required_votes": 2, + }, + { + "type": "roboflow_core/detections_combine@v1", + "name": "combine", + "prediction_one": "$steps.consensus.predictions", + "prediction_two": "$inputs.first", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.consensus.predictions", + }, + { + "type": "JsonField", + "name": "combined", + "selector": "$steps.combine.predictions", + }, + ], + } + first = _response(identity="first", confidence=0.6, sides=sides) + second = _response(identity="second", confidence=0.9, angle=math.pi, sides=sides) + second["class_heading"]["additional"] = True + engine = ExecutionEngine.init(workflow_definition=workflow) + outputs = engine.run( + runtime_parameters={"first": [first], "second": [second]}, + serialize_results=True, + )[0] + result = outputs["result"] + assert len(outputs["combined"]["predictions"]) == 2 + assert outputs["combined"]["class_heading"] == second["class_heading"] + assert result["predictions"][0]["detection_id"] == "second" + assert result["predictions"][0]["obb"] == pytest.approx( + second["predictions"][0]["obb"] + ) + assert result["class_heading"] == second["class_heading"] + engine.close() + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_all_empty_native_sources_keep_serializable_policy(tensor): + sources = [_response(identity="first"), _response(identity="second")] + for source in sources: + source["predictions"] = [] + result = serialise_native_obb(_run(sources, tensor=tensor)) + assert result["predictions"] == [] + assert result["class_heading"] == sources[0]["class_heading"] + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_native_vote_compares_footprints_instead_of_enclosing_bounds(tensor): + sources = [ + _response(identity="first", sides=(50.0, 3.0), angle=math.pi / 4), + _response(identity="second", sides=(50.0, 3.0), angle=-math.pi / 4), + ] + result = serialise_native_obb(_run(sources, tensor=tensor, iou_threshold=0.5)) + assert result["predictions"] == [] + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_native_block_keeps_source_carriers_unchanged(tensor): + sources = [ + native_obb_from_inference(_response(identity="first", confidence=0.6)), + native_obb_from_inference(_response(identity="second", confidence=0.9)), + ] + sources[1].metadata["class_heading"]["additional"] = True + if tensor: + sources = [native_obb_to_tensor(source) for source in sources] + before = [serialise_native_obb(source) for source in sources] + _run(sources) + assert [serialise_native_obb(source) for source in sources] == before + for source in sources: + if isinstance(source, sv.Detections): + assert "_consensus_source_order" not in source.data + else: + assert all( + "_consensus_source_order" not in row for row in source.bboxes_metadata + ) + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_native_carrier_requires_frozen_policy_for_every_observed_class(tensor): + sources = [ + native_obb_from_inference(_response(identity=str(index))) for index in range(2) + ] + del sources[1].metadata["class_heading"]["plane"] + with pytest.raises(ValueError, match="frozen class semantics"): + _run(sources, tensor=tensor) + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("explicit", [False, True]) +def test_native_http_input_initializes_missing_lineage_before_consensus( + monkeypatch, tensor, explicit +): + from roboflow_workflows import environment + from roboflow_workflows.core_steps.common.oriented import ( + _native_obb_to_sv, + deserialize_native_obb, + ) + + payload = _response(identity="http") + row = payload["predictions"][0] + del row["parent_id"] + if explicit: + row.update( + parent_id="crop", + root_parent_id="root", + parent_origin={"offset_x": 3, "offset_y": 5, "width": 100, "height": 80}, + ) + monkeypatch.setattr(environment, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + source = deserialize_native_obb("external", payload) + lineage = _native_obb_to_sv(source).data + assert lineage["parent_id"].tolist() == ["crop" if explicit else "external"] + assert lineage["root_parent_id"].tolist() == ["root" if explicit else "external"] + if explicit: + np.testing.assert_array_equal(lineage["parent_coordinates"], [[3, 5]]) + result = serialise_native_obb(_run([source], required_votes=1)) + assert result["predictions"][0]["parent_id"] == ("crop" if explicit else "external") + + +@pytest.mark.parametrize( + "empty_by", ["confidence", "votes", "per_class", "after_consensus", "after_combine"] +) +def test_compiled_empty_consensus_output_retains_crop_root_frame(empty_by): + from roboflow_workflows.execution_engine.core import ExecutionEngine + + payload = _response(identity="crop-row") + payload["image"] = {"width": 101, "height": 79} + payload["predictions"][0].update( + parent_id="crop", + root_parent_id="root", + parent_origin={"offset_x": 0, "offset_y": 0, "width": 101, "height": 79}, + root_parent_origin={ + "offset_x": 13, + "offset_y": 17, + "width": 300, + "height": 200, + }, + ) + workflow = { + "version": "1.0", + "inputs": [ + { + "type": "WorkflowBatchInput", + "name": "predictions", + "kind": ["oriented_object_detection_prediction"], + } + ], + "steps": [ + { + "type": "roboflow_core/detections_consensus@v2", + "name": "consensus", + "predictions_batches": ["$inputs.predictions"], + "required_votes": 2 if empty_by == "votes" else 1, + "confidence": 0.99 if empty_by == "confidence" else 0, + } + ], + "outputs": [ + { + "type": "JsonField", + "name": "root", + "selector": "$steps.consensus.predictions", + "coordinates_system": "parent", + }, + { + "type": "JsonField", + "name": "own", + "selector": "$steps.consensus.predictions", + "coordinates_system": "own", + }, + ], + } + if empty_by in {"per_class", "after_consensus", "after_combine"}: + selector = "$inputs.predictions" + if empty_by == "per_class": + workflow["steps"] = [] + elif empty_by == "after_combine": + workflow["steps"] = [ + { + "type": "roboflow_core/detections_combine@v1", + "name": "combined", + "prediction_one": selector, + "prediction_two": selector, + } + ] + selector = "$steps.combined.predictions" + else: + selector = "$steps.consensus.predictions" + workflow["steps"].append( + { + "type": "roboflow_core/per_class_confidence_filter@v1", + "name": "filtered", + "predictions": selector, + "class_thresholds": {"plane": 0.99}, + } + ) + for output in workflow["outputs"]: + output["selector"] = "$steps.filtered.predictions" + engine = ExecutionEngine.init(workflow_definition=workflow) + try: + result = engine.run( + runtime_parameters={"predictions": [payload]}, serialize_results=True + )[0] + finally: + engine.close() + assert result["root"]["predictions"] == [] + assert result["root"]["image"] == {"height": 200, "width": 300} + assert result["root"]["class_heading"] == payload["class_heading"] + assert result["own"]["image"] == payload["image"] + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_native_consensus_accepts_identity_perspective_host_caches(tensor): + from roboflow_workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_sv_detections, + ) + from roboflow_workflows.core_steps.transformations.perspective_correction import ( + v1, + v1_tensor, + ) + from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, + ) + + image = WorkflowImageData( + parent_metadata=ImageParentMetadata("same-image"), + numpy_image=np.zeros((80, 100, 3), np.uint8), + ) + source = attach_parents_coordinates_to_sv_detections( + native_obb_from_inference(_response(identity="original")), image + ) + if tensor: + source = native_obb_to_tensor(source) + before = serialise_native_obb(source) + module = v1_tensor if tensor else v1 + warped = module.PerspectiveCorrectionBlockV1().run( + images=[image], + predictions=[source], + perspective_polygons=[[0, 0], [99, 0], [99, 79], [0, 79]], + transformed_rect_width=100, + transformed_rect_height=80, + extend_perspective_polygon_by_detections_anchor="CENTER", + warp_image=False, + )[0]["corrected_coordinates"] + result = _run([source, warped], required_votes=2) + assert len(result) == 1 + assert serialise_native_obb(result)["predictions"][0]["obb"] == pytest.approx( + before["predictions"][0]["obb"] + ) + assert serialise_native_obb(source) == before + + +@pytest.mark.parametrize( + "fusion,model_first,external_state", + [ + ("combine", False, "populated"), + ("combine", True, "populated"), + ("combine", False, "empty"), + ("combine_then_consensus", False, "empty"), + ("consensus", False, "populated"), + ("consensus", True, "populated"), + ("consensus", False, "empty"), + ("consensus", False, "different-parent"), + ], +) +def test_compiled_external_and_model_sources_retain_selected_frame( + fusion, model_first, external_state +): + from roboflow_workflows.core_steps.common.entities import StepExecutionMode + from roboflow_workflows.errors import StepExecutionError + from roboflow_workflows.execution_engine.core import ExecutionEngine + from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + OriginCoordinatesSystem, + WorkflowImageData, + ) + + external = _response(identity="external", confidence=0.6) + external["class_heading"]["external-only"] = False + if external_state == "empty": + external["predictions"] = [] + elif external_state == "different-parent": + external["predictions"][0]["parent_id"] = "other-image" + model_response = _response(identity="model", confidence=0.9, angle=math.pi) + model_response["class_heading"]["model-only"] = True + image = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="same-image"), + workflow_root_ancestor_metadata=ImageParentMetadata( + parent_id="root", + origin_coordinates=OriginCoordinatesSystem( + left_top_x=13, left_top_y=17, origin_width=300, origin_height=200 + ), + ), + numpy_image=np.zeros((80, 100, 3), dtype=np.uint8), + ) + before = copy.deepcopy(external) + image_before = copy.deepcopy(image) + + class Provider: + def add_model(self, **kwargs): + pass + + def run_oriented_object_detection(self, **kwargs): + return [copy.deepcopy(model_response)] + + selectors = ["$inputs.external", "$steps.model.predictions"] + if model_first: + selectors.reverse() + step = ( + { + "type": "roboflow_core/detections_combine@v1", + "name": "fused", + "prediction_one": selectors[0], + "prediction_two": selectors[1], + } + if fusion != "consensus" + else { + "type": "roboflow_core/detections_consensus@v2", + "name": "fused", + "predictions_batches": selectors, + "required_votes": 1 if external_state == "empty" else 2, + "confidence": 0, + } + ) + workflow = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "images"}, + { + "type": "WorkflowBatchInput", + "name": "external", + "kind": ["oriented_object_detection_prediction"], + }, + ], + "steps": [ + { + "type": "roboflow_core/roboflow_oriented_object_detection_model@v1", + "name": "model", + "images": "$inputs.images", + "model_id": "native/1", + }, + step, + ], + "outputs": [ + { + "type": "JsonField", + "name": coordinates, + "selector": "$steps.fused.predictions", + "coordinates_system": coordinates, + } + for coordinates in ("own", "parent") + ] + + [ + { + "type": "JsonField", + "name": "model_root", + "selector": "$steps.model.predictions", + "coordinates_system": "parent", + }, + { + "type": "JsonField", + "name": "external", + "selector": "$inputs.external", + "coordinates_system": "own", + }, + ], + } + if fusion == "combine_then_consensus": + workflow["steps"].append( + { + "type": "roboflow_core/detections_consensus@v2", + "name": "final", + "predictions_batches": ["$steps.fused.predictions"], + "required_votes": 1, + "confidence": 0, + } + ) + for output in workflow["outputs"][:2]: + output["selector"] = "$steps.final.predictions" + engine = ExecutionEngine.init( + workflow_definition=workflow, + init_parameters={ + "workflows_core.model_manager": Provider(), + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + }, + ) + try: + if external_state == "different-parent": + with pytest.raises(StepExecutionError, match="different parent"): + engine.run( + runtime_parameters={"images": [image], "external": [external]}, + serialize_results=True, + ) + return + result = engine.run( + runtime_parameters={"images": [image], "external": [external]}, + serialize_results=True, + )[0] + finally: + engine.close() + + expected_ids = ["model", "external"] if model_first else ["external", "model"] + if fusion == "consensus" or external_state == "empty": + expected_ids = ["model"] + for coordinates in ("own", "parent"): + output = result[coordinates] + translated = model_first and coordinates == "parent" + assert output["image"] == ( + {"width": 300, "height": 200} + if translated + else {"width": 100, "height": 80} + ) + assert output["class_heading"] == { + **external["class_heading"], + **model_response["class_heading"], + } + assert [row["detection_id"] for row in output["predictions"]] == expected_ids + for identity, row in zip(expected_ids, output["predictions"]): + assert row["obb"] == pytest.approx( + [ + 53.25 if translated else 40.25, + 47.75 if translated else 30.75, + 12.0, + 24.0, + math.pi if identity == "model" else 0.0, + ] + ) + assert row["class"] == "plane" + assert row["class_id"] == 7 + assert row["heading"] is True + assert row["parent_id"] == ("root" if translated else "same-image") + expected_confidence = 0.9 if identity == "model" else 0.6 + if fusion == "consensus" and external_state != "empty": + expected_confidence = 0.75 + assert row["confidence"] == pytest.approx(expected_confidence) + assert result["model_root"]["image"] == {"width": 300, "height": 200} + assert result["model_root"]["predictions"][0]["obb"] == pytest.approx( + [53.25, 47.75, 12.0, 24.0, math.pi] + ) + assert result["model_root"]["class_heading"] == model_response["class_heading"] + assert result["external"] == before + assert external == before + assert image.parent_metadata == image_before.parent_metadata + assert ( + image.workflow_root_ancestor_metadata + == image_before.workflow_root_ancestor_metadata + ) + np.testing.assert_array_equal(image.numpy_image, image_before.numpy_image) + + +@pytest.mark.parametrize("tensor_input", [False, True]) +def test_native_consensus_recovers_legacy_row_frame(tensor_input): + from roboflow_workflows.core_steps.common.tensor_native import ( + native_detections_to_root_coordinates, + ) + from roboflow_workflows.core_steps.common.utils import ( + sv_detections_to_root_coordinates, + ) + + source = native_obb_from_inference(_response(identity="legacy")) + frame = { + "root_parent_id": "root", + "root_parent_coordinates": [13, 17], + "root_parent_dimensions": [200, 300], + } + for key, value in frame.items(): + source.data[key] = np.asarray([value] * len(source)) + if tensor_input: + source = native_obb_to_tensor(source) + for key in frame: + del source.image_metadata[key] + consensus = _run([source], required_votes=1) + rooted = ( + native_detections_to_root_coordinates(consensus) + if tensor_input + else sv_detections_to_root_coordinates(consensus) + ) + result = serialise_native_obb(rooted) + assert result["image"] == {"width": 300, "height": 200} + assert result["predictions"][0]["obb"] == pytest.approx( + [53.25, 47.75, 12.0, 24.0, 0.0] + ) + assert result["predictions"][0]["parent_id"] == "root" diff --git a/workflows/tests/unit_tests/core_steps/test_native_obb_transport.py b/workflows/tests/unit_tests/core_steps/test_native_obb_transport.py new file mode 100644 index 0000000000..726e93a122 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/test_native_obb_transport.py @@ -0,0 +1,1110 @@ +import json +import math + +import numpy as np +import pytest +import supervision as sv +from roboflow_workflows.core_steps.common.oriented import ( + annotate_native_obb, + native_obb_from_inference, + native_obb_to_tensor, + serialise_native_obb, +) +from roboflow_workflows.core_steps.common.tensor_native import ( + native_detections_to_root_coordinates, + take_detections_by_indices, +) +from roboflow_workflows.core_steps.common.utils import ( + scale_sv_detections, + sv_detections_to_root_coordinates, +) + + +def response(): + return dict( + task_type="oriented-object-detection", + image=dict(width=101, height=79), + class_heading={"plane": True, "stone": False}, + predictions=[ + dict( + obb=[40.25, 30.75, 12.5, 25.75, -2.9], + confidence=0.9, + class_id=7, + heading=True, + detection_id="plane0", + **{"class": "plane"} + ), + dict( + obb=[70.0, 50.0, 10.0, 10.0, math.pi / 2], + confidence=0.8, + class_id=9, + heading=False, + detection_id="stone1", + **{"class": "stone"} + ), + ], + ) + + +def test_native_serialization_never_drops_heading_or_rounds_geometry(): + original = response() + transport = native_obb_from_inference(original) + result = serialise_native_obb(transport) + for source, row in zip(original["predictions"], result["predictions"]): + assert row["obb"] == pytest.approx(source["obb"]) + assert row["heading"] is source["heading"] + assert row["class_id"] == source["class_id"] + assert row["detection_id"] == source["detection_id"] + from roboflow_workflows.core_steps.common.serializers import serialise_sv_detections + + assert serialise_sv_detections(transport) == result + assert "x" not in result["predictions"][0] + assert json.loads(json.dumps(result))["class_heading"] == original["class_heading"] + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_row_selection_and_combine_keep_front_alignment(tensor): + original = native_obb_from_inference(response()) + if tensor: + from roboflow_workflows.core_steps.transformations.detections_combine.v1_tensor import ( + DetectionsCombineBlockV1, + ) + + original = native_obb_to_tensor(original) + first = take_detections_by_indices(original, [1]) + second = take_detections_by_indices(original, [0]) + combined = DetectionsCombineBlockV1().run(first, second)["predictions"] + else: + from roboflow_workflows.core_steps.transformations.detections_combine.v1 import ( + DetectionsCombineBlockV1, + ) + + first, second = original[[1]], original[[0]] + combined = DetectionsCombineBlockV1().run(first, second)["predictions"] + rows = serialise_native_obb(combined)["predictions"] + assert [r["detection_id"] for r in rows] == ["stone1", "plane0"] + assert [r["heading"] for r in rows] == [False, True] + assert rows[1]["obb"] == pytest.approx(response()["predictions"][0]["obb"]) + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_root_offset_moves_shape_and_front_together_in_float(tensor): + transport = native_obb_from_inference(response()) + for key, value in { + "root_parent_coordinates": [10, 20], + "root_parent_dimensions": [200, 300], + "root_parent_id": "root", + "parent_id": "crop", + }.items(): + transport.data[key] = np.asarray([value] * len(transport)) + transport.metadata[key] = value + if tensor: + transport = native_detections_to_root_coordinates( + native_obb_to_tensor(transport) + ) + else: + transport = sv_detections_to_root_coordinates(transport) + rows = serialise_native_obb(transport) + assert rows["image"] == dict(width=300, height=200) + assert rows["predictions"][0]["obb"] == pytest.approx( + [50.25, 50.75, 12.5, 25.75, -2.9] + ) + + +def test_scale_preserves_order_and_anisotropy_is_explicit_quadrilateral(): + transport = native_obb_from_inference(response()) + scaled = scale_sv_detections(transport, scale=1.7) + row = serialise_native_obb(scaled)["predictions"][0] + assert row["obb"] == pytest.approx( + [40.25 * 1.7, 30.75 * 1.7, 12.5 * 1.7, 25.75 * 1.7, -2.9] + ) + warped = scale_sv_detections( + transport, scale=(1.4, 0.7), update_scaling_metadata=False + ) + row = serialise_native_obb(warped)["predictions"][0] + assert "obb" not in row and row["geometry_type"] == "quadrilateral" + corners = np.asarray(row["corners"]) + before = transport.data["xyxyxyxy"][0] + np.testing.assert_allclose(corners, before * [1.4, 0.7], atol=1e-10) + np.testing.assert_allclose( + corners[1] - corners[0], (before[1] - before[0]) * [1.4, 0.7] + ) + restored = serialise_native_obb( + native_obb_from_inference(dict(warped=1, **serialise_native_obb(warped))) + ) + np.testing.assert_allclose(restored["predictions"][0]["corners"], corners) + + +@pytest.mark.parametrize("field", ["heading", "detection_id", "obb"]) +def test_missing_native_semantics_rejects(field): + malformed = response() + del malformed["predictions"][0][field] + with pytest.raises((ValueError, KeyError)): + native_obb_from_inference(malformed) + + +def test_conflicting_heading_rejects_instead_of_defaulting(): + malformed = response() + malformed["predictions"][0]["heading"] = False + with pytest.raises(ValueError, match="heading"): + native_obb_from_inference(malformed) + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_empty_native_results_retain_frozen_schema(tensor): + value = response() + value["predictions"] = [] + transport = native_obb_from_inference(value) + if tensor: + transport = native_obb_to_tensor(transport) + assert serialise_native_obb(transport) == value + + +def test_native_visualization_uses_rotated_footprint_and_front(): + value = response() + value["predictions"] = value["predictions"][:1] + transport = native_obb_from_inference(value) + image = np.zeros((79, 101, 3), np.uint8) + annotated = annotate_native_obb(image, transport) + assert np.count_nonzero(annotated) > 0 + assert np.any(np.all(annotated == [51, 182, 255], axis=-1)) + # Center stays empty: this is a footprint/front-edge render, not an AABB fill. + assert not annotated[31, 40].any() + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_dynamic_crop_and_root_restore_preserve_original_native_tuple(tensor): + from copy import deepcopy + + from roboflow_workflows.core_steps.common.utils import ( + attach_parents_coordinates_to_batch_of_sv_detections, + ) + from roboflow_workflows.core_steps.transformations.dynamic_crop.v1 import crop_image + from roboflow_workflows.execution_engine.entities.base import ( + Batch, + ImageParentMetadata, + WorkflowImageData, + ) + + original = response() + original["predictions"] = original["predictions"][:1] + transport = native_obb_from_inference(original) + image = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="image0"), + numpy_image=np.zeros((79, 101, 3), np.uint8), + ) + transport = attach_parents_coordinates_to_batch_of_sv_detections( + images=Batch([image], indices=[(0,)]), predictions=[transport] + )[0] + before = deepcopy(transport) + if tensor: + from roboflow_workflows.core_steps.transformations.dynamic_crop.v1_tensor import ( + crop_image as tensor_crop, + ) + + cropped = tensor_crop(image, native_obb_to_tensor(transport), 0, "black")[0] + else: + cropped = crop_image( + image, transport, mask_opacity=0, background_color="black" + )[0] + localized = cropped["predictions"] + assert serialise_native_obb(localized)["image"] == { + "height": cropped["crops"].numpy_image.shape[0], + "width": cropped["crops"].numpy_image.shape[1], + } + restored = ( + native_detections_to_root_coordinates(localized) + if tensor + else sv_detections_to_root_coordinates(localized) + ) + assert serialise_native_obb(restored)["predictions"][0]["obb"] == pytest.approx( + original["predictions"][0]["obb"], abs=1e-9 + ) + + if tensor: + from roboflow_workflows.core_steps.common.tensor_native import ( + take_detections_by_indices, + ) + + empty = take_detections_by_indices(localized, []) + rooted_empty = native_detections_to_root_coordinates(empty) + else: + rooted_empty = sv_detections_to_root_coordinates(localized[:0]) + assert serialise_native_obb(rooted_empty)["image"] == original["image"] + assert serialise_native_obb(transport)["predictions"][0]["obb"] == pytest.approx( + original["predictions"][0]["obb"] + ) + + assert transport.metadata == before.metadata + for key in transport.data: + np.testing.assert_array_equal(transport.data[key], before.data[key]) + + +def test_workflow_schema_accepts_explicit_quadrilateral_result(): + from roboflow_workflows.core_steps.common.oriented_entities import ( + OrientedWorkflowInferenceResponse, + ) + + transport = native_obb_from_inference(response()) + warped = scale_sv_detections( + transport, scale=(1.4, 0.7), update_scaling_metadata=False + ) + parsed = OrientedWorkflowInferenceResponse.model_validate( + serialise_native_obb(warped) + ) + assert parsed.predictions[0].geometry_type == "quadrilateral" + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("empty", ["neither", "first", "second", "both"]) +def test_combination_preserves_full_frozen_maps_and_rejects_conflicts(tensor, empty): + from copy import deepcopy + + from roboflow_workflows.core_steps.common.oriented import ( + _native_obb_to_sv, + merge_native_semantics, + ) + + original = response() + other = response() + original["predictions"] = original["predictions"][:1] + other["predictions"] = other["predictions"][1:] + original["class_heading"] = {"plane": True} + other["class_heading"] = {"stone": False} + if empty in {"first", "both"}: + original["predictions"] = [] + if empty in {"second", "both"}: + other["predictions"] = [] + first = native_obb_from_inference(original) + other["class_heading"]["unused"] = True + second = native_obb_from_inference(other) + if tensor: + from roboflow_workflows.core_steps.transformations.detections_combine.v1_tensor import ( + DetectionsCombineBlockV1, + ) + + else: + from roboflow_workflows.core_steps.transformations.detections_combine.v1 import ( + DetectionsCombineBlockV1, + ) + first, second = native_obb_to_tensor(first), native_obb_to_tensor(second) + if not tensor: + first, second = _native_obb_to_sv(first), _native_obb_to_sv(second) + headers = [ + deepcopy(value.image_metadata if tensor else value.metadata) + for value in (first, second) + ] + before = [serialise_native_obb(value) for value in (first, second)] + combined = DetectionsCombineBlockV1().run(first, second)["predictions"] + assert serialise_native_obb(combined)["class_heading"]["unused"] is True + assert serialise_native_obb(combined)["image"] == original["image"] + assert [row["class"] for row in serialise_native_obb(combined)["predictions"]] == [ + row["class"] for value in before for row in value["predictions"] + ] + assert [serialise_native_obb(value) for value in (first, second)] == before + assert [ + value.image_metadata if tensor else value.metadata for value in (first, second) + ] == headers + metadata = second.image_metadata if tensor else second.metadata + metadata["class_heading"]["plane"] = False + with pytest.raises(ValueError, match="Conflicting"): + merge_native_semantics(first, second) + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("empty", [False, True]) +@pytest.mark.parametrize("tensor_input", [False, True]) +def test_declared_kind_deserialization_uses_current_carrier( + monkeypatch, tensor, empty, tensor_input +): + from roboflow_workflows import environment + from roboflow_workflows.core_steps.common.oriented import deserialize_native_obb + + original = response() + if empty: + original["predictions"] = [] + supplied = ( + native_obb_to_tensor(native_obb_from_inference(original)) + if tensor_input + else original + ) + monkeypatch.setattr(environment, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + value = deserialize_native_obb("input", supplied) + assert isinstance(value, sv.Detections) is not tensor + serialized = serialise_native_obb(value) + assert serialized["image"] == original["image"] + assert serialized["class_heading"] == original["class_heading"] + if empty: + assert serialized["predictions"] == [] + else: + assert serialized["predictions"][0]["obb"] == pytest.approx( + original["predictions"][0]["obb"] + ) + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize( + "operation", + [ + {"type": "DetectionsShift", "shift_x": 30, "shift_y": 20}, + {"type": "DetectionsOffset", "offset_x": 20, "offset_y": 10}, + { + "type": "DetectionsRename", + "class_map": {"plane": "aircraft", "stone": "rock"}, + }, + ], +) +@pytest.mark.parametrize("marker", ["both", "image", "rows", "empty"]) +def test_query_geometry_operations_reject_native_without_mutation( + monkeypatch, tensor, operation, marker +): + """Reject native geometry through the dispatcher before changing either view.""" + from roboflow_workflows.core_steps.common.query_language.errors import ( + InvalidInputTypeError, + ) + from roboflow_workflows.core_steps.common.query_language.operations.core import ( + execute_operations, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detections import ( + base, + ) + + original = response() + if marker == "empty": + original["predictions"] = [] + transport = native_obb_from_inference(original) + if marker == "rows": + del transport.metadata["native_obb"] + elif marker == "image": + del transport.data["native_obb"] + if tensor: + transport = native_obb_to_tensor(transport) + monkeypatch.setattr(base, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + before_bounds = np.asarray(transport.xyxy).copy() + before_serialized = serialise_native_obb(transport) + + with pytest.raises(InvalidInputTypeError, match="native oriented detections"): + execute_operations(transport, operations=[operation]) + + np.testing.assert_array_equal(transport.xyxy, before_bounds) + assert serialise_native_obb(transport) == before_serialized + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize( + "operation, expected", + [ + ( + {"type": "DetectionsShift", "shift_x": 30, "shift_y": 20}, + [[40, 40, 60, 60]], + ), + ( + {"type": "DetectionsOffset", "offset_x": 20, "offset_y": 10}, + [[0, 15, 40, 45]], + ), + ], +) +def test_query_geometry_operations_preserve_ordinary_detection_behavior( + monkeypatch, tensor, operation, expected +): + """Keep ordinary AABB shift/offset results and input-copy behavior intact.""" + from roboflow_workflows.core_steps.common.query_language.operations.core import ( + execute_operations, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detections import ( + base, + ) + + transport = sv.Detections( + xyxy=np.array([[10, 20, 30, 40]], dtype=float), + class_id=np.array([7]), + confidence=np.array([0.9]), + ) + if tensor: + import torch + + from inference_models.models.base.object_detection import Detections + + transport = Detections( + xyxy=torch.as_tensor(transport.xyxy), + class_id=torch.as_tensor(transport.class_id), + confidence=torch.as_tensor(transport.confidence), + ) + monkeypatch.setattr(base, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + + result = execute_operations(transport, operations=[operation]) + + np.testing.assert_array_equal(result.xyxy, expected) + np.testing.assert_array_equal(transport.xyxy, [[10, 20, 30, 40]]) + + +def test_legacy_corner_scaling_keeps_existing_rounded_aabb_contract(): + """Native float geometry does not change unmarked legacy corner carriers.""" + corners = np.array([[[1.1, 2.2], [11.3, 2.2], [11.3, 12.4], [1.1, 12.4]]]) + source = sv.Detections( + xyxy=np.array([[1.1, 2.2, 11.3, 12.4]]), data={"xyxyxyxy": corners} + ) + + result = scale_sv_detections(source, 2, update_scaling_metadata=False) + + np.testing.assert_array_equal(result.xyxy, [[2, 4, 23, 25]]) + np.testing.assert_array_equal(result.data["xyxyxyxy"], corners) + + +def test_legacy_perspective_corners_keep_rounding_and_dtype(): + """The existing oriented carrier retains its float32 rounded corners.""" + from roboflow_workflows.core_steps.transformations.perspective_correction.v1 import ( + correct_detections, + ) + from roboflow_workflows.core_steps.transformations.perspective_correction.v1_tensor import ( + _correct_oriented_box_in_metadata, + ) + + corners = np.array([[[1.1, 2.2], [11.3, 2.2], [11.3, 12.4], [1.1, 12.4]]]) + matrix = np.array([[1.0, 0.0, 0.25], [0.0, 1.0, 0.5], [0.0, 0.0, 1.0]]) + source = sv.Detections( + xyxy=np.array([[1.1, 2.2, 11.3, 12.4]]), data={"xyxyxyxy": corners} + ) + expected = np.around(corners + [0.25, 0.5]).astype(np.float32) + + result = correct_detections(source, matrix, 100, 100) + entry = {"xyxyxyxy": corners[0]} + _correct_oriented_box_in_metadata(entry, matrix) + + np.testing.assert_array_equal(result.data["xyxyxyxy"], expected) + assert result.data["xyxyxyxy"].dtype == np.float32 + np.testing.assert_array_equal(entry["xyxyxyxy"], expected[0]) + assert entry["xyxyxyxy"].dtype == np.float32 + + +@pytest.mark.parametrize("empty", [False, True]) +@pytest.mark.parametrize( + "boundary", + [ + "converter", + "serialized", + "carrier", + "tensor_serialized", + "tensor_carrier", + "tensor_rle_serialized", + "tensor_rle_carrier", + ], +) +def test_ordinary_detection_boundaries_reject_native_payloads(empty, boundary): + from roboflow_workflows.core_steps.common.deserializers import ( + deserialize_detections_kind, + ) + from roboflow_workflows.core_steps.common.utils import ( + convert_inference_detections_batch_to_sv_detections, + ) + from roboflow_workflows.errors import RuntimeInputError + + payload = response() + if empty: + payload["predictions"] = [] + error = ValueError if boundary == "converter" else RuntimeInputError + with pytest.raises(error, match="oriented_object_detection_prediction"): + if boundary == "converter": + convert_inference_detections_batch_to_sv_detections([payload]) + elif boundary.startswith("tensor"): + from roboflow_workflows.core_steps.common.deserializers_tensor import ( + deserialize_detections_kind as deserialize_tensor, + ) + + if "rle" in boundary: + from roboflow_workflows.core_steps.common.deserializers_tensor import ( + deserialize_rle_detections_kind as deserialize_tensor, + ) + value = ( + native_obb_to_tensor(native_obb_from_inference(payload)) + if boundary.endswith("carrier") + else payload + ) + deserialize_tensor("detections", value) + else: + value = ( + native_obb_from_inference(payload) if boundary == "carrier" else payload + ) + deserialize_detections_kind("detections", value) + + +def test_ordinary_detection_boundaries_preserve_aabb_predictions(): + from roboflow_workflows.core_steps.common.deserializers import ( + deserialize_detections_kind, + ) + from roboflow_workflows.core_steps.common.utils import ( + convert_inference_detections_batch_to_sv_detections, + ) + + payload = dict( + image=dict(width=100, height=80), + predictions=[ + dict( + x=20, + y=30, + width=10, + height=20, + confidence=0.9, + class_id=7, + detection_id="ordinary", + **{"class": "plane"} + ) + ], + ) + for value in ( + convert_inference_detections_batch_to_sv_detections([payload])[0], + deserialize_detections_kind("detections", payload), + ): + np.testing.assert_array_equal(value.xyxy, [[15, 20, 25, 40]]) + assert value["detection_id"].tolist() == ["ordinary"] + assert deserialize_detections_kind("detections", value) is value + + from roboflow_workflows.core_steps.common.deserializers_tensor import ( + deserialize_detections_kind as deserialize_tensor, + ) + + tensor_value = deserialize_tensor("detections", payload) + np.testing.assert_array_equal(tensor_value.xyxy.cpu(), [[15, 20, 25, 40]]) + assert tensor_value.bboxes_metadata[0]["detection_id"] == "ordinary" + tensor_value.bboxes_metadata = [None] + assert deserialize_tensor("detections", tensor_value) is tensor_value + + from roboflow_workflows.core_steps.common.deserializers_tensor import ( + deserialize_rle_detections_kind as deserialize_rle_tensor, + ) + + payload["predictions"][0]["rle_mask"] = {"size": [80, 100], "counts": [8000]} + rle_value = deserialize_rle_tensor("detections", payload) + np.testing.assert_array_equal(rle_value.xyxy.cpu(), [[15, 20, 25, 40]]) + assert rle_value.mask.to_coco_rle_masks() == [{"size": [80, 100], "counts": [8000]}] + assert deserialize_rle_tensor("detections", rle_value) is rle_value + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize( + "obb, window", + [ + ([5, 50, 30, 20, 0], (0, 40, 20, 60)), + ([50, 5, 20, 30, 0], (40, 0, 60, 20)), + ([95, 95, 30, 30, 0], (80, 80, 100, 100)), + ([-50, 50, 30, 20, 0], None), + ], +) +def test_native_crop_clips_pixels_and_origin_without_clipping_ordered_front( + tensor, obb, window +): + from roboflow_workflows.core_steps.transformations.dynamic_crop import v1, v1_tensor + from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, + ) + + payload = response() + payload["image"] = dict(width=100, height=100) + payload["predictions"] = payload["predictions"][:1] + payload["predictions"][0]["obb"] = obb + prediction = native_obb_from_inference(payload) + image_data = np.arange(100 * 100 * 3, dtype=np.uint8).reshape(100, 100, 3) + image = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="image0"), numpy_image=image_data + ) + if tensor: + result = v1_tensor.crop_image( + image, native_obb_to_tensor(prediction), 0, "black" + )[0] + else: + result = v1.crop_image(image, prediction, 0, "black")[0] + if window is None: + assert result == {"crops": None, "predictions": None} + return + + x1, y1, x2, y2 = window + crop = result["crops"] + np.testing.assert_array_equal(crop.numpy_image, image_data[y1:y2, x1:x2]) + assert crop.parent_metadata.parent_id == "plane0" + assert crop.parent_metadata.origin_coordinates.left_top_x == x1 + assert crop.parent_metadata.origin_coordinates.left_top_y == y1 + row = serialise_native_obb(result["predictions"])["predictions"][0] + assert row["obb"] == pytest.approx([obb[0] - x1, obb[1] - y1, *obb[2:]]) + assert row["heading"] is True + assert row["detection_id"] == "plane0" + assert ( + serialise_native_obb(result["predictions"])["class_heading"] + == payload["class_heading"] + ) + + +@pytest.mark.parametrize( + "tensor, row_metadata", [(False, None), (True, None), (True, {"class": "plane"})] +) +def test_query_rename_preserves_ordinary_classes_and_ids( + monkeypatch, tensor, row_metadata +): + from roboflow_workflows.core_steps.common.query_language.operations.core import ( + execute_operations, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detections import ( + base, + ) + + transport = sv.Detections( + xyxy=np.array([[10, 20, 30, 40]], dtype=float), + class_id=np.array([7]), + confidence=np.array([0.9]), + data={"class_name": np.array(["plane"])}, + ) + if tensor: + import torch + + from inference_models.models.base.object_detection import Detections + + transport = Detections( + xyxy=torch.as_tensor(transport.xyxy), + class_id=torch.as_tensor(transport.class_id), + confidence=torch.as_tensor(transport.confidence), + bboxes_metadata=[row_metadata], + image_metadata={"class_names": {7: "plane"}}, + ) + monkeypatch.setattr(base, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + result = execute_operations( + transport, + operations=[{"type": "DetectionsRename", "class_map": {"plane": "aircraft"}}], + ) + names = ( + result.image_metadata["class_names"][int(result.class_id[0])] + if tensor + else result["class_name"][0] + ) + assert names == "aircraft" + assert int(result.class_id[0]) == 0 + + +def test_query_rename_declaration_excludes_native_kind(): + from roboflow_workflows.core_steps.common.query_language.entities.operations import ( + DetectionsRename, + ) + + schema = DetectionsRename.model_json_schema() + for kinds in (schema["input_kind"], schema["output_kind"]): + names = [kind.name for kind in kinds] + assert "object_detection_prediction" in names + assert "oriented_object_detection_prediction" not in names + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("fusion", ["consensus", "combine"]) +@pytest.mark.parametrize("consumer", ["per_class", "parent_class", "filter"]) +def test_native_colliding_class_ids_keep_row_names_through_filter( + monkeypatch, tensor, fusion, consumer +): + from roboflow_workflows.core_steps.fusion.detections_consensus.v1 import ( + AggregationMode, + MaskAggregationMode, + ) + from roboflow_workflows.core_steps.fusion.detections_consensus.v2 import ( + DetectionsConsensusBlockV2, + ) + from roboflow_workflows.core_steps.transformations.detections_combine import ( + v1 as combine, + ) + from roboflow_workflows.core_steps.transformations.detections_combine import ( + v1_tensor as combine_tensor, + ) + from roboflow_workflows.core_steps.transformations.per_class_confidence_filter import ( + v1, + v1_tensor, + ) + + value = response() + sources = [] + for row in value["predictions"]: + row["class_id"] = 0 + row["confidence"] = 0.7 + row["parent_id"] = "image" + source = native_obb_from_inference({**value, "predictions": [row]}) + sources.append(native_obb_to_tensor(source) if tensor else source) + if fusion == "combine": + module = combine_tensor if tensor else combine + merged = module.DetectionsCombineBlockV1().run(*sources)["predictions"] + else: + merged = DetectionsConsensusBlockV2().run( + predictions_batches=[[source] for source in sources], + required_votes=1, + class_aware=True, + iou_threshold=0.5, + confidence=0, + classes_to_consider=None, + required_objects=None, + presence_confidence_aggregation=AggregationMode.MAX, + detections_merge_confidence_aggregation=AggregationMode.AVERAGE, + detections_merge_coordinates_aggregation=AggregationMode.AVERAGE, + detections_merge_mask_aggregation=MaskAggregationMode.UNION, + )[0]["predictions"] + from roboflow_workflows.core_steps.common.query_language.entities.operations import ( + DetectionsProperty, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detections import ( + base, + ) + + monkeypatch.setattr(base, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + assert base.extract_detections_property( + merged, DetectionsProperty.CLASS_NAME, "test" + ) == ["plane", "stone"] + if consumer == "per_class": + module = v1_tensor if tensor else v1 + result = module.PerClassConfidenceFilterBlockV1().run( + predictions=[merged], + class_thresholds={"plane": 0.9, "stone": 0.1}, + )[0]["predictions"] + expected = "stone" + else: + from roboflow_workflows.core_steps.common.query_language.operations.core import ( + execute_operations, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detection import ( + base as single_detection, + ) + + monkeypatch.setattr( + single_detection, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor + ) + operation = ( + {"type": "PickDetectionsByParentClass", "parent_class": "plane"} + if consumer == "parent_class" + else { + "type": "DetectionsFilter", + "filter_operation": { + "type": "StatementGroup", + "operator": "and", + "statements": [ + { + "type": "BinaryStatement", + "negate": False, + "left_operand": { + "type": "DynamicOperand", + "operand_name": "_", + "operations": [ + { + "type": "ExtractDetectionProperty", + "property_name": "class_name", + } + ], + }, + "comparator": {"type": "=="}, + "right_operand": { + "type": "StaticOperand", + "value": "plane", + }, + } + ], + }, + } + ) + result = execute_operations(merged, operations=[operation]) + expected = "plane" + rows = serialise_native_obb(result)["predictions"] + assert [(row["class"], row["class_id"]) for row in rows] == [(expected, 0)] + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_native_empty_visualization_block_returns_unchanged_image(tensor): + from roboflow_workflows.core_steps.models.roboflow.oriented_object_detection.v1 import ( + NativeOBBVisualizationBlockV1, + ) + from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, + ) + + value = response() + value["predictions"] = [] + predictions = native_obb_from_inference(value) + if tensor: + predictions = native_obb_to_tensor(predictions) + pixels = np.full((79, 101, 3), 123, dtype=np.uint8) + image = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id="image"), numpy_image=pixels + ) + result = NativeOBBVisualizationBlockV1().run(image, predictions, 2) + np.testing.assert_array_equal(result["image"].numpy_image, pixels) + + +@pytest.mark.parametrize("empty", [False, True]) +def test_compiled_native_perspective_roundtrip_preserves_ordered_subpixel_quad(empty): + from roboflow_workflows.execution_engine.core import ExecutionEngine + + workflow = { + "version": "1.0", + "inputs": [ + {"type": "InferenceImage", "name": "image"}, + { + "type": "WorkflowBatchInput", + "name": "predictions", + "kind": ["oriented_object_detection_prediction"], + }, + ], + "steps": [ + { + "type": "roboflow_core/perspective_correction@v1", + "name": "warp", + "images": "$inputs.image", + "predictions": "$inputs.predictions", + "perspective_polygons": [[0, 0], [90, 7], [100, 78], [2, 75]], + "transformed_rect_width": 103, + "transformed_rect_height": 83, + "extend_perspective_polygon_by_detections_anchor": "CENTER", + "warp_image": False, + }, + { + "type": "roboflow_core/detections_consensus@v2", + "name": "consumer", + "predictions_batches": ["$steps.warp.corrected_coordinates"], + "required_votes": 1, + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "result", + "selector": "$steps.consumer.predictions", + "coordinates_system": "own", + } + ], + } + payload = response() + if empty: + payload["predictions"] = [] + engine = ExecutionEngine.init(workflow_definition=workflow) + try: + result = engine.run( + runtime_parameters={ + "image": np.zeros((79, 101, 3), dtype=np.uint8), + "predictions": [payload], + }, + serialize_results=True, + )[0]["result"] + finally: + engine.close() + assert result["image"] == {"height": 83, "width": 103} + assert result["class_heading"] == payload["class_heading"] + if empty: + assert result["predictions"] == [] + return + + row = result["predictions"][0] + assert row["geometry_type"] == "quadrilateral" + corners = np.asarray(row["corners"]) + assert np.isfinite(corners).all() + assert np.any(np.abs(corners - np.rint(corners)) > 0.01) + restored = serialise_native_obb(native_obb_from_inference(result)) + np.testing.assert_allclose(restored["predictions"][0]["corners"], corners) + assert row["heading"] is True + assert row["detection_id"] == "plane0" + + +@pytest.mark.parametrize("initially_empty", [False, True]) +def test_empty_tensor_bridge_retains_root_frame_for_serialization(initially_empty): + from roboflow_workflows.core_steps.common.oriented import _native_obb_to_sv + + value = response() + if initially_empty: + value["predictions"] = [] + source = native_obb_from_inference(value) + source.metadata.update( + parent_id="crop", + root_parent_id="root", + root_parent_coordinates=[13, 17], + root_parent_dimensions=[200, 300], + ) + bridged = _native_obb_to_sv(native_obb_to_tensor(source)) + restored = sv_detections_to_root_coordinates(bridged[:0]) + assert restored.metadata["parent_id"] == "root" + assert serialise_native_obb(restored)["image"] == {"height": 200, "width": 300} + assert serialise_native_obb(restored)["predictions"] == [] + + +@pytest.mark.parametrize("tensor", [False, True]) +@pytest.mark.parametrize("initially_empty", [False, True]) +def test_parent_class_query_empty_result_remains_native( + monkeypatch, tensor, initially_empty +): + from roboflow_workflows.core_steps.common.query_language.operations.core import ( + execute_operations, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detections import ( + base, + ) + from roboflow_workflows.core_steps.transformations.detections_combine import ( + v1, + v1_tensor, + ) + + payload = response() + if initially_empty: + payload["predictions"] = [] + source = native_obb_from_inference(payload) + if tensor: + source = native_obb_to_tensor(source) + monkeypatch.setattr(base, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + selected = execute_operations( + source, + operations=[{"type": "PickDetectionsByParentClass", "parent_class": "absent"}], + ) + serialized = serialise_native_obb(selected) + assert serialized == {**payload, "predictions": []} + module = v1_tensor if tensor else v1 + combined = module.DetectionsCombineBlockV1().run(selected, selected)["predictions"] + assert serialise_native_obb(combined) == serialized + + +@pytest.mark.parametrize("tensor", [False, True]) +def test_native_identity_perspective_keeps_tight_bounds_and_bbox_filter( + monkeypatch, tensor +): + from roboflow_workflows.core_steps.common.oriented import _native_obb_to_sv + from roboflow_workflows.core_steps.common.query_language.operations.core import ( + execute_operations, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detection import ( + base as single, + ) + from roboflow_workflows.core_steps.common.query_language.operations.detections import ( + base, + ) + from roboflow_workflows.core_steps.transformations.perspective_correction import ( + v1, + v1_tensor, + ) + from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + WorkflowImageData, + ) + + payload = response() + payload["predictions"] = payload["predictions"][:1] + source = native_obb_from_inference(payload) + if tensor: + source = native_obb_to_tensor(source) + before = serialise_native_obb(source) + image = WorkflowImageData( + parent_metadata=ImageParentMetadata("image"), + numpy_image=np.zeros((79, 101, 3), np.uint8), + ) + module = v1_tensor if tensor else v1 + warped = module.PerspectiveCorrectionBlockV1().run( + images=[image], + predictions=[source], + perspective_polygons=[[0, 0], [100, 0], [100, 78], [0, 78]], + transformed_rect_width=101, + transformed_rect_height=79, + extend_perspective_polygon_by_detections_anchor="CENTER", + warp_image=False, + )[0]["corrected_coordinates"] + converted = _native_obb_to_sv(warped) + corners = converted.data["xyxyxyxy"] + np.testing.assert_allclose( + converted.xyxy, + np.concatenate((corners.min(1), corners.max(1)), axis=1), + atol=1e-6, + ) + assert serialise_native_obb(source) == before + if tensor: + from roboflow_workflows.core_steps.common.tensor_native import read_host_mirror + + mirror = read_host_mirror(warped.bboxes_metadata, len(warped)) + assert mirror is not None + np.testing.assert_allclose(mirror[0], converted.xyxy, atol=1e-6) + monkeypatch.setattr(base, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + monkeypatch.setattr(single, "ENABLE_TENSOR_DATA_REPRESENTATION", tensor) + operation = { + "type": "DetectionsFilter", + "filter_operation": { + "type": "StatementGroup", + "operator": "and", + "statements": [ + { + "type": "BinaryStatement", + "negate": False, + "left_operand": { + "type": "DynamicOperand", + "operand_name": "_", + "operations": [ + { + "type": "ExtractDetectionProperty", + "property_name": "x_min", + } + ], + }, + "comparator": {"type": "(Number) >"}, + "right_operand": {"type": "StaticOperand", "value": 31.05}, + } + ], + }, + } + assert len(execute_operations(source, operations=[operation])) == 1 + assert len(execute_operations(warped, operations=[operation])) == 1 + + +@pytest.mark.parametrize("initially_empty", [False, True]) +def test_native_scale_updates_header_before_rows_can_disappear(initially_empty): + payload = response() + if initially_empty: + payload["predictions"] = [] + source = native_obb_from_inference(payload) + for key, value in [ + ("scaling_relative_to_parent", 1.25), + ("scaling_relative_to_root_parent", 0.5), + ]: + source.metadata[key] = value + source.data[key] = np.full(len(source), value) + scaled = scale_sv_detections(source, 2.0, target_size_wh=(203, 159)) + assert scaled.metadata["image_dimensions"] == [159, 203] + assert scaled.metadata["scaling_relative_to_parent"] == 2.5 + assert scaled.metadata["scaling_relative_to_root_parent"] == 1.0 + if len(scaled): + np.testing.assert_array_equal( + scaled.data["image_dimensions"], [[159, 203]] * len(scaled) + ) + np.testing.assert_array_equal( + scaled.data["scaling_relative_to_parent"], [2.5] * len(scaled) + ) + assert serialise_native_obb(native_obb_to_tensor(scaled[:0]))["image"] == { + "height": 159, + "width": 203, + } + assert source.metadata["image_dimensions"] == [79, 101] + assert source.metadata["scaling_relative_to_parent"] == 1.25 + + +def test_native_carrier_header_wins_without_mutating_input(monkeypatch): + from copy import deepcopy + + from roboflow_workflows import environment + from roboflow_workflows.core_steps.common.oriented import deserialize_native_obb + + source = native_obb_from_inference(response()) + source.metadata.update( + root_parent_id="root", + root_parent_coordinates=[13, 17], + root_parent_dimensions=[200, 300], + ) + source.data["image_dimensions"][:] = [333, 444] + source.data["root_parent_id"] = np.array(["stale"] * len(source)) + source.data["root_parent_coordinates"] = np.tile([100, 100], (len(source), 1)) + source.data["root_parent_dimensions"] = np.tile([333, 444], (len(source), 1)) + before = deepcopy(source) + monkeypatch.setattr(environment, "ENABLE_TENSOR_DATA_REPRESENTATION", False) + normalized = deserialize_native_obb("input", source) + assert serialise_native_obb(normalized)["image"] == {"width": 101, "height": 79} + tensor = native_obb_to_tensor(normalized) + assert tensor.image_metadata["image_dimensions"] == [79, 101] + assert tensor.image_metadata["root_parent_dimensions"] == [200, 300] + rooted = sv_detections_to_root_coordinates(normalized) + bridged_empty = take_detections_by_indices(native_obb_to_tensor(rooted), []) + assert serialise_native_obb(bridged_empty)["image"] == {"width": 300, "height": 200} + assert rooted.metadata["root_parent_coordinates"] == [0, 0] + assert rooted.metadata["parent_id"] == "root" + assert source.metadata == before.metadata + for key in source.data: + np.testing.assert_array_equal(source.data[key], before.data[key])