From 5b4d3893758bed5f6a9214297afd094dde870267 Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Thu, 29 Oct 2026 13:25:11 -0500 Subject: [PATCH 1/2] Accept step output images in video-state blocks Embedded inner workflows are inlined before compilation, which replaces the child's `$inputs.image` with the parent binding. When that binding is a step output (e.g. a preprocessing block), blocks typed with WorkflowImageSelector rejected it and the merged definition failed to parse, although each workflow compiled on its own. Use Selector(kind=[IMAGE_KIND]) like the newer tracker blocks so the image can come from an input or a step. Co-Authored-By: Claude --- workflows/CHANGELOG.md | 4 + .../analytics/detection_event_log/v1.py | 4 +- .../detection_event_log/v1_tensor.py | 4 +- .../core_steps/analytics/line_counter/v2.py | 4 +- .../analytics/line_counter/v2_tensor.py | 4 +- .../core_steps/analytics/path_deviation/v2.py | 4 +- .../analytics/path_deviation/v2_tensor.py | 4 +- .../core_steps/analytics/time_in_zone/v2.py | 4 +- .../analytics/time_in_zone/v2_tensor.py | 4 +- .../core_steps/analytics/time_in_zone/v3.py | 4 +- .../analytics/time_in_zone/v3_tensor.py | 4 +- .../core_steps/analytics/velocity/v1.py | 4 +- .../analytics/velocity/v1_tensor.py | 4 +- .../core_steps/cache/cache_get/v1.py | 3 +- .../core_steps/cache/cache_set/v1.py | 3 +- .../flow_control/delta_filter/v1.py | 4 +- .../flow_control/rate_limiter/v1.py | 3 +- .../transformations/byte_tracker/v2.py | 4 +- .../transformations/byte_tracker/v2_tensor.py | 4 +- ...ded_video_blocks_with_step_output_image.py | 293 ++++++++++++++++++ 20 files changed, 330 insertions(+), 36 deletions(-) create mode 100644 workflows/tests/unit_tests/execution_engine/inner_workflow/test_embedded_video_blocks_with_step_output_image.py diff --git a/workflows/CHANGELOG.md b/workflows/CHANGELOG.md index 09ac8d73e7..5fa572b9ae 100644 --- a/workflows/CHANGELOG.md +++ b/workflows/CHANGELOG.md @@ -20,6 +20,10 @@ for contributor and maintainer responsibilities. - Vision Events (`roboflow_core/roboflow_vision_events@v1`) and Vision Event Bundle (`roboflow_core/vision_event_bundle@v1`) blocks gain an optional `value_base_units` input: what the event is worth, in base units of the use case's currency (e.g. cents for USD). The currency always comes from the use case. The value is sent as `valueBaseUnits` to the Vision Events API and in the bundle `payload.json`, and as `value_base_units` to the local event store. A typed value or workflow input must be a whole number (`true`/`false` are rejected). A value from another step is rounded to whole base units; one that is not a finite number or falls outside ±10,000,000,000 is dropped with a warning and the event is still recorded. Left empty, the use case's default value applies. `operator_feedback` events never carry a value, so the blocks omit it for them. +### Fixed + +- Video-state blocks now accept an image from a step output as well as from a Workflow input, so they keep working inside an embedded `roboflow_core/inner_workflow@v1` whose `image` is bound to a parent step (for example a preprocessing block). Inlining replaces the child's `$inputs.image` with that step selector, and the merged definition used to fail with `Could not parse workflow definition` even though each Workflow compiled on its own. Affected blocks: `time_in_zone@v2`/`@v3`, `velocity@v1`, `line_counter@v2`, `path_deviation_analytics@v2`, `detection_event_log@v1`, `byte_tracker@v2`, `cache_get@v1`, `cache_set@v1`, `delta_filter@v1` and `rate_limiter@v1` (`video_reference_image`). The blocks read `video_metadata` from the image, which step outputs built with `WorkflowImageData.copy_and_replace` keep, so per-video state is unchanged. + ## `0.2.14` Bundled execution engine: `1.17.0`. diff --git a/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1.py b/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1.py index 9450f629fb..3872dd3333 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1.py +++ b/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1.py @@ -19,11 +19,11 @@ from roboflow_workflows.execution_engine.entities.types import ( DICTIONARY_KIND, FLOAT_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, Selector, - WorkflowImageSelector, WorkflowParameterSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( @@ -94,7 +94,7 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/detection_event_log@v1"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Reference to the image for video metadata (frame number, timestamp).", examples=["$inputs.image"], ) diff --git a/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1_tensor.py b/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1_tensor.py index 1448069b33..070ec65fb8 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1_tensor.py +++ b/workflows/roboflow_workflows/core_steps/analytics/detection_event_log/v1_tensor.py @@ -22,9 +22,9 @@ from roboflow_workflows.execution_engine.entities.types import ( DICTIONARY_KIND, FLOAT_KIND, + IMAGE_KIND, INTEGER_KIND, Selector, - WorkflowImageSelector, WorkflowParameterSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( @@ -98,7 +98,7 @@ class BlockManifest(WorkflowBlockManifest): ) type: Literal["roboflow_core/detection_event_log@v1"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Reference to the image for video metadata (frame number, timestamp).", examples=["$inputs.image"], ) diff --git a/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2.py b/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2.py index 2fa78e846f..da66db1a02 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2.py +++ b/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2.py @@ -10,13 +10,13 @@ WorkflowImageData, ) from roboflow_workflows.execution_engine.entities.types import ( + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -136,7 +136,7 @@ class LineCounterManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/line_counter@v2"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image with embedded video metadata. The video_metadata contains video_identifier to maintain separate counting state for different videos. Required for persistent counting across frames.", ) detections: Selector( diff --git a/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2_tensor.py b/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2_tensor.py index db0a1486c9..7cef5e72e5 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2_tensor.py +++ b/workflows/roboflow_workflows/core_steps/analytics/line_counter/v2_tensor.py @@ -20,11 +20,11 @@ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, ) from roboflow_workflows.execution_engine.entities.types import ( + IMAGE_KIND, INTEGER_KIND, LIST_OF_VALUES_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -147,7 +147,7 @@ class LineCounterManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/line_counter@v2"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image with embedded video metadata. The video_metadata contains video_identifier to maintain separate counting state for different videos. Required for persistent counting across frames.", ) detections: Selector( diff --git a/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2.py b/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2.py index 72ca2fddfa..b277e98805 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2.py +++ b/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2.py @@ -14,12 +14,12 @@ WorkflowImageData, ) from roboflow_workflows.execution_engine.entities.types import ( + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -134,7 +134,7 @@ class PathDeviationManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/path_deviation_analytics@v2"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image with embedded video metadata. The video_metadata contains video_identifier to maintain separate path tracking state for different videos. Required for persistent path accumulation across frames.", ) detections: Selector( diff --git a/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2_tensor.py b/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2_tensor.py index e08730d4bd..1fd1d8eef6 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2_tensor.py +++ b/workflows/roboflow_workflows/core_steps/analytics/path_deviation/v2_tensor.py @@ -19,10 +19,10 @@ TENSOR_NATIVE_OBJECT_DETECTION_PREDICTION_KIND, ) from roboflow_workflows.execution_engine.entities.types import ( + IMAGE_KIND, LIST_OF_VALUES_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -140,7 +140,7 @@ class PathDeviationManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/path_deviation_analytics@v2"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image with embedded video metadata. The video_metadata contains video_identifier to maintain separate path tracking state for different videos. Required for persistent path accumulation across frames.", ) detections: Selector( diff --git a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2.py b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2.py index 4c3308c67d..c8fee5f2b0 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2.py +++ b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2.py @@ -15,12 +15,12 @@ ) from roboflow_workflows.execution_engine.entities.types import ( BOOLEAN_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -144,7 +144,7 @@ class TimeInZoneManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/time_in_zone@v2"] - image: Union[WorkflowImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Image", description="Input image for the current video frame containing embedded video metadata (fps, frame_number, frame_timestamp, video_identifier, video source) required for time calculation and state management. The block extracts video_metadata from the WorkflowImageData object. The fps and frame_number are used for video files to calculate timestamps (timestamp = frame_number / fps). For streamed video, frame_timestamp is used directly. The video_identifier is used to maintain separate tracking state and zone configurations for different videos. Used for zone visualization and reference. The image dimensions are used to validate zone coordinates. This version simplifies input by embedding metadata in the image object rather than requiring a separate metadata field.", examples=["$inputs.image", "$steps.cropping.crops"], diff --git a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2_tensor.py b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2_tensor.py index ddb601794a..acf150cf33 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2_tensor.py +++ b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v2_tensor.py @@ -24,10 +24,10 @@ ) from roboflow_workflows.execution_engine.entities.types import ( BOOLEAN_KIND, + IMAGE_KIND, LIST_OF_VALUES_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -154,7 +154,7 @@ class TimeInZoneManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/time_in_zone@v2"] - image: Union[WorkflowImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Image", description="Input image for the current video frame containing embedded video metadata (fps, frame_number, frame_timestamp, video_identifier, video source) required for time calculation and state management. The block extracts video_metadata from the WorkflowImageData object. The fps and frame_number are used for video files to calculate timestamps (timestamp = frame_number / fps). For streamed video, frame_timestamp is used directly. The video_identifier is used to maintain separate tracking state and zone configurations for different videos. Used for zone visualization and reference. The image dimensions are used to validate zone coordinates. This version simplifies input by embedding metadata in the image object rather than requiring a separate metadata field.", examples=["$inputs.image", "$steps.cropping.crops"], diff --git a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3.py b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3.py index 3c03370666..c4c72d4938 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3.py +++ b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3.py @@ -17,12 +17,12 @@ ) from roboflow_workflows.execution_engine.entities.types import ( BOOLEAN_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, LIST_OF_VALUES_KIND, OBJECT_DETECTION_PREDICTION_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -167,7 +167,7 @@ class TimeInZoneManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/time_in_zone@v3"] - image: Union[WorkflowImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Image", description="Input image for the current video frame containing embedded video metadata (fps, frame_number, frame_timestamp, video_identifier, video source) required for time calculation and state management. The block extracts video_metadata from the WorkflowImageData object. The fps and frame_number are used for video files to calculate timestamps (timestamp = frame_number / fps). For streamed video, frame_timestamp is used directly. The video_identifier is used to maintain separate tracking state and zone configurations for different videos. Used for zone visualization and reference. The image dimensions are used to validate zone coordinates. This version supports multiple zones per video with efficient zone cache management.", examples=["$inputs.image", "$steps.cropping.crops"], diff --git a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3_tensor.py b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3_tensor.py index abcaa693f4..d29da00a53 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3_tensor.py +++ b/workflows/roboflow_workflows/core_steps/analytics/time_in_zone/v3_tensor.py @@ -26,10 +26,10 @@ ) from roboflow_workflows.execution_engine.entities.types import ( BOOLEAN_KIND, + IMAGE_KIND, LIST_OF_VALUES_KIND, STRING_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -177,7 +177,7 @@ class TimeInZoneManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/time_in_zone@v3"] - image: Union[WorkflowImageSelector] = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( title="Image", description="Input image for the current video frame containing embedded video metadata (fps, frame_number, frame_timestamp, video_identifier, video source) required for time calculation and state management. The block extracts video_metadata from the WorkflowImageData object. The fps and frame_number are used for video files to calculate timestamps (timestamp = frame_number / fps). For streamed video, frame_timestamp is used directly. The video_identifier is used to maintain separate tracking state and zone configurations for different videos. Used for zone visualization and reference. The image dimensions are used to validate zone coordinates. This version supports multiple zones per video with efficient zone cache management.", examples=["$inputs.image", "$steps.cropping.crops"], diff --git a/workflows/roboflow_workflows/core_steps/analytics/velocity/v1.py b/workflows/roboflow_workflows/core_steps/analytics/velocity/v1.py index 0415d727d0..b7d0cf36f0 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/velocity/v1.py +++ b/workflows/roboflow_workflows/core_steps/analytics/velocity/v1.py @@ -18,12 +18,12 @@ ) from roboflow_workflows.execution_engine.entities.types import ( FLOAT_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, KEYPOINT_DETECTION_PREDICTION_KIND, OBJECT_DETECTION_PREDICTION_KIND, Selector, StepOutputSelector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -134,7 +134,7 @@ class VelocityManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/velocity@v1"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image with embedded video metadata. The video_metadata contains fps, frame_number, frame_timestamp, and video_identifier. Required for calculating time deltas and maintaining separate velocity tracking state for different videos.", ) detections: StepOutputSelector( diff --git a/workflows/roboflow_workflows/core_steps/analytics/velocity/v1_tensor.py b/workflows/roboflow_workflows/core_steps/analytics/velocity/v1_tensor.py index dd9251a3c6..319143b54c 100644 --- a/workflows/roboflow_workflows/core_steps/analytics/velocity/v1_tensor.py +++ b/workflows/roboflow_workflows/core_steps/analytics/velocity/v1_tensor.py @@ -26,9 +26,9 @@ ) from roboflow_workflows.execution_engine.entities.types import ( FLOAT_KIND, + IMAGE_KIND, Selector, StepOutputSelector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -143,7 +143,7 @@ class VelocityManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/velocity@v1"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Image with embedded video metadata. The video_metadata contains fps, frame_number, frame_timestamp, and video_identifier. Required for calculating time deltas and maintaining separate velocity tracking state for different videos.", ) detections: StepOutputSelector( diff --git a/workflows/roboflow_workflows/core_steps/cache/cache_get/v1.py b/workflows/roboflow_workflows/core_steps/cache/cache_get/v1.py index 96dead28bf..37765de826 100644 --- a/workflows/roboflow_workflows/core_steps/cache/cache_get/v1.py +++ b/workflows/roboflow_workflows/core_steps/cache/cache_get/v1.py @@ -15,7 +15,6 @@ STRING_KIND, WILDCARD_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -99,7 +98,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/cache_get@v1"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Input image used to determine the cache namespace. The block extracts the video identifier from the image's video metadata and uses it as the cache namespace. If no video identifier is present, the block uses 'default' as the namespace. The namespace isolates cache entries so different videos or streams have separate cache storage. Use the same image (with the same video identifier) for both Cache Set and Cache Get blocks to access the same cache namespace.", examples=["$inputs.image", "$steps.input.output"], ) diff --git a/workflows/roboflow_workflows/core_steps/cache/cache_set/v1.py b/workflows/roboflow_workflows/core_steps/cache/cache_set/v1.py index d6d37d0c8c..dcc0a5d1b3 100644 --- a/workflows/roboflow_workflows/core_steps/cache/cache_set/v1.py +++ b/workflows/roboflow_workflows/core_steps/cache/cache_set/v1.py @@ -16,7 +16,6 @@ STRING_KIND, WILDCARD_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -100,7 +99,7 @@ class BlockManifest(WorkflowBlockManifest): } ) type: Literal["roboflow_core/cache_set@v1"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Input image used to determine the cache namespace. The block extracts the video identifier from the image's video metadata and uses it as the cache namespace. If no video identifier is present, the block uses 'default' as the namespace. The namespace isolates cache entries so different videos or streams have separate cache storage. Use the same image (with the same video identifier) for both Cache Set and Cache Get blocks to access the same cache namespace.", examples=["$inputs.image", "$steps.input.output"], ) diff --git a/workflows/roboflow_workflows/core_steps/flow_control/delta_filter/v1.py b/workflows/roboflow_workflows/core_steps/flow_control/delta_filter/v1.py index 49e0b17fe7..d169ad572e 100644 --- a/workflows/roboflow_workflows/core_steps/flow_control/delta_filter/v1.py +++ b/workflows/roboflow_workflows/core_steps/flow_control/delta_filter/v1.py @@ -9,10 +9,10 @@ WorkflowImageData, ) from roboflow_workflows.execution_engine.entities.types import ( + IMAGE_KIND, WILDCARD_KIND, Selector, StepSelector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -92,7 +92,7 @@ class DeltaFilterManifest(WorkflowBlockManifest): }, } ) - image: WorkflowImageSelector + image: Selector(kind=[IMAGE_KIND]) value: Selector(kind=[WILDCARD_KIND]) = Field( title="Input Value", description="Value to monitor for changes. Can be any data type (numbers, strings, detection counts, metrics, etc.) from workflow inputs or step outputs. The workflow branch continues to next_steps only when this value differs from the previously cached value for the current video stream. If the value remains the same, the branch terminates to avoid redundant processing. Example: Monitor a line counter count ($steps.line_counter.count_in) and trigger actions only when the count changes.", diff --git a/workflows/roboflow_workflows/core_steps/flow_control/rate_limiter/v1.py b/workflows/roboflow_workflows/core_steps/flow_control/rate_limiter/v1.py index ec925f7055..8eb1f6f82d 100644 --- a/workflows/roboflow_workflows/core_steps/flow_control/rate_limiter/v1.py +++ b/workflows/roboflow_workflows/core_steps/flow_control/rate_limiter/v1.py @@ -13,7 +13,6 @@ IMAGE_KIND, Selector, StepSelector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -113,7 +112,7 @@ class RateLimiterManifest(WorkflowBlockManifest): description="List of workflow steps to execute if the rate limit allows (i.e., sufficient time has passed since the last execution). These steps receive control flow only when the cooldown period has elapsed, enabling throttled downstream processing. If the cooldown period hasn't elapsed, these steps will not execute as the branch terminates. Each step selector references a block in the workflow that should execute when rate limiting permits.", examples=[["$steps.upload"]], ) - video_reference_image: Optional[WorkflowImageSelector] = Field( + video_reference_image: Optional[Selector(kind=[IMAGE_KIND])] = Field( description="Optional reference to a video frame image to use for video-time-based timestamp generation. When provided, the rate limiter calculates timestamps based on video metadata (frame number and FPS) rather than system clock time. This is useful when processing video faster than real-time, ensuring rate limiting works correctly relative to video playback time rather than processing speed. If not provided (None), the block uses system clock time (datetime.now()). Only applicable for video processing workflows.", examples=["$inputs.image"], default=None, diff --git a/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2.py b/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2.py index c7527f62b6..0ee6b3bab0 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2.py +++ b/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2.py @@ -13,11 +13,11 @@ ) from roboflow_workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INSTANCE_SEGMENTATION_PREDICTION_KIND, INTEGER_KIND, OBJECT_DETECTION_PREDICTION_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -136,7 +136,7 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): protected_namespaces=(), ) type: Literal["roboflow_core/byte_tracker@v2"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Input image containing embedded video metadata (fps and video_identifier) required for ByteTrack initialization and tracking state management. The block extracts video_metadata from the WorkflowImageData object. The fps value is used to configure the tracker, and the video_identifier is used to maintain separate tracking state for different videos. If fps is missing or invalid, the block defaults to 0 and logs a warning but continues operation. If processing multiple videos, each video should have a unique video_identifier in its metadata to maintain separate tracking states. The block maintains persistent trackers across frames for each video using the video_identifier. This version simplifies input by embedding metadata in the image object rather than requiring a separate metadata field.", ) detections: Selector( diff --git a/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2_tensor.py b/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2_tensor.py index e0730e8049..fa55736efb 100644 --- a/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2_tensor.py +++ b/workflows/roboflow_workflows/core_steps/transformations/byte_tracker/v2_tensor.py @@ -22,9 +22,9 @@ ) from roboflow_workflows.execution_engine.entities.types import ( FLOAT_ZERO_TO_ONE_KIND, + IMAGE_KIND, INTEGER_KIND, Selector, - WorkflowImageSelector, ) from roboflow_workflows.execution_engine.entities.workload import ( Discovery, @@ -203,7 +203,7 @@ class ByteTrackerBlockManifest(WorkflowBlockManifest): protected_namespaces=(), ) type: Literal["roboflow_core/byte_tracker@v2"] - image: WorkflowImageSelector = Field( + image: Selector(kind=[IMAGE_KIND]) = Field( description="Input image containing embedded video metadata (fps and video_identifier) required for ByteTrack initialization and tracking state management. The block extracts video_metadata from the WorkflowImageData object. The fps value is used to configure the tracker, and the video_identifier is used to maintain separate tracking state for different videos. If fps is missing or invalid, the block defaults to 0 and logs a warning but continues operation. If processing multiple videos, each video should have a unique video_identifier in its metadata to maintain separate tracking states. The block maintains persistent trackers across frames for each video using the video_identifier. This version simplifies input by embedding metadata in the image object rather than requiring a separate metadata field.", ) detections: Selector( diff --git a/workflows/tests/unit_tests/execution_engine/inner_workflow/test_embedded_video_blocks_with_step_output_image.py b/workflows/tests/unit_tests/execution_engine/inner_workflow/test_embedded_video_blocks_with_step_output_image.py new file mode 100644 index 0000000000..7d45701ab6 --- /dev/null +++ b/workflows/tests/unit_tests/execution_engine/inner_workflow/test_embedded_video_blocks_with_step_output_image.py @@ -0,0 +1,293 @@ +""" +Embedded workflows whose ``image`` input is bound to a parent step output. + +Inlining replaces the child's ``$inputs.image`` with the parent binding, so a child +that compiles on its own must still compile once its image comes from a step such as +an image preprocessing block. Video-state blocks (trackers, analytics, cache, flow +control) used to accept only ``$inputs.*`` image selectors, which made the merged +definition fail to parse. +""" + +import datetime +from typing import Any, Dict + +import numpy as np +from roboflow_workflows.core_steps.cache.memory_cache import WorkflowMemoryCache +from roboflow_workflows.execution_engine.core import ExecutionEngine +from roboflow_workflows.execution_engine.entities.base import ( + ImageParentMetadata, + VideoMetadata, + WorkflowImageData, +) + +ZONE = [[0, 0], [100, 0], [100, 100], [0, 100]] + +VIDEO_STATE_CHILD_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + { + "type": "WorkflowBatchInput", + "name": "predictions", + "kind": ["object_detection_prediction"], + }, + ], + "steps": [ + { + "type": "roboflow_core/byte_tracker@v2", + "name": "tracker", + "image": "$inputs.image", + "detections": "$inputs.predictions", + }, + { + "type": "roboflow_core/time_in_zone@v2", + "name": "time_in_zone_v2", + "image": "$inputs.image", + "detections": "$steps.tracker.tracked_detections", + "zone": ZONE, + }, + { + "type": "roboflow_core/time_in_zone@v3", + "name": "time_in_zone_v3", + "image": "$inputs.image", + "detections": "$steps.tracker.tracked_detections", + "zone": ZONE, + }, + { + "type": "roboflow_core/velocity@v1", + "name": "velocity", + "image": "$inputs.image", + "detections": "$steps.tracker.tracked_detections", + }, + { + "type": "roboflow_core/line_counter@v2", + "name": "line_counter", + "image": "$inputs.image", + "detections": "$steps.tracker.tracked_detections", + "line_segment": [[0, 50], [100, 50]], + }, + { + "type": "roboflow_core/path_deviation_analytics@v2", + "name": "path_deviation", + "image": "$inputs.image", + "detections": "$steps.tracker.tracked_detections", + "reference_path": [[0, 0], [100, 100]], + }, + { + "type": "roboflow_core/detection_event_log@v1", + "name": "event_log", + "image": "$inputs.image", + "detections": "$steps.tracker.tracked_detections", + }, + { + "type": "roboflow_core/cache_get@v1", + "name": "before", + "image": "$inputs.image", + "key": "k", + }, + { + "type": "roboflow_core/rate_limiter@v1", + "name": "rate_limiter", + "depends_on": "$steps.before.output", + "next_steps": ["$steps.as_text"], + "video_reference_image": "$inputs.image", + }, + { + "type": "roboflow_core/property_definition@v1", + "name": "as_text", + "data": "$steps.before.output", + "operations": [{"type": "ToString"}], + }, + { + "type": "roboflow_core/delta_filter@v1", + "name": "delta_filter", + "image": "$inputs.image", + "value": "$steps.as_text.output", + "next_steps": ["$steps.store"], + }, + { + "type": "roboflow_core/cache_set@v1", + "name": "store", + "image": "$inputs.image", + "key": "k", + "value": "$steps.as_text.output", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "time_in_zone", + "selector": "$steps.time_in_zone_v3.timed_detections", + }, + { + "type": "JsonField", + "name": "velocity", + "selector": "$steps.velocity.velocity_detections", + }, + ], +} + +CACHE_CHILD_WORKFLOW = { + "version": "1.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}], + "steps": [ + { + "type": "roboflow_core/cache_get@v1", + "name": "before", + "image": "$inputs.image", + "key": "k", + }, + { + "type": "roboflow_core/property_definition@v1", + "name": "as_text", + "data": "$steps.before.output", + "operations": [{"type": "ToString"}], + }, + { + "type": "roboflow_core/cache_set@v1", + "name": "store", + "image": "$inputs.image", + "key": "k", + "value": "$steps.as_text.output", + }, + ], + "outputs": [ + {"type": "JsonField", "name": "before", "selector": "$steps.before.output"}, + {"type": "JsonField", "name": "stored", "selector": "$steps.store.output"}, + ], +} + + +def _parent_workflow( + child: Dict[str, Any], + *, + extra_inputs: list, + extra_bindings: Dict[str, str], + outputs: list, +) -> Dict[str, Any]: + return { + "version": "1.0", + "inputs": [{"type": "WorkflowImage", "name": "image"}, *extra_inputs], + "steps": [ + { + "type": "roboflow_core/convert_grayscale@v1", + "name": "standard_image", + "image": "$inputs.image", + }, + { + "type": "roboflow_core/inner_workflow@v1", + "name": "nested", + "workflow_definition": child, + "parameter_bindings": { + "image": "$steps.standard_image.image", + **extra_bindings, + }, + }, + ], + "outputs": outputs, + } + + +def _video_frame(*, video_id: str, frame_number: int) -> WorkflowImageData: + video_metadata = VideoMetadata( + video_identifier=video_id, + frame_number=frame_number, + frame_timestamp=datetime.datetime.fromtimestamp( + 1726570875 + frame_number, tz=datetime.timezone.utc + ), + fps=1, + ) + frame = WorkflowImageData( + parent_metadata=ImageParentMetadata(parent_id=video_id), + numpy_image=np.zeros((32, 32, 3), dtype=np.uint8), + video_metadata=video_metadata, + ) + + return frame + + +def test_video_state_blocks_compile_when_embedded_image_comes_from_parent_step() -> ( + None +): + # given + workflow_definition = _parent_workflow( + VIDEO_STATE_CHILD_WORKFLOW, + extra_inputs=[ + { + "type": "WorkflowBatchInput", + "name": "predictions", + "kind": ["object_detection_prediction"], + } + ], + extra_bindings={"predictions": "$inputs.predictions"}, + outputs=[ + { + "type": "JsonField", + "name": "time_in_zone", + "selector": "$steps.nested.time_in_zone", + }, + { + "type": "JsonField", + "name": "velocity", + "selector": "$steps.nested.velocity", + }, + ], + ) + + # when + execution_engine = ExecutionEngine.init( + workflow_definition=workflow_definition, + init_parameters={}, + max_concurrent_steps=1, + ) + + # then + assert execution_engine is not None + + +def test_embedded_cache_keeps_video_namespace_when_image_comes_from_parent_step() -> ( + None +): + # given + WorkflowMemoryCache.cache.clear() + WorkflowMemoryCache._retain_counts.clear() + workflow_definition = _parent_workflow( + CACHE_CHILD_WORKFLOW, + extra_inputs=[], + extra_bindings={}, + outputs=[ + { + "type": "JsonField", + "name": "before", + "selector": "$steps.nested.before", + }, + { + "type": "JsonField", + "name": "stored", + "selector": "$steps.nested.stored", + }, + ], + ) + execution_engine = ExecutionEngine.init( + workflow_definition=workflow_definition, + init_parameters={}, + max_concurrent_steps=1, + ) + + # when + first = execution_engine.run( + runtime_parameters={"image": _video_frame(video_id="cam-a", frame_number=1)} + ) + second = execution_engine.run( + runtime_parameters={"image": _video_frame(video_id="cam-a", frame_number=2)} + ) + other_video = execution_engine.run( + runtime_parameters={"image": _video_frame(video_id="cam-b", frame_number=1)} + ) + + # then + assert first[0] == {"before": False, "stored": "False"} + assert second[0] == {"before": "False", "stored": "False"} + assert other_video[0] == {"before": False, "stored": "False"} + assert "cam-a" in WorkflowMemoryCache.cache + assert "cam-b" in WorkflowMemoryCache.cache From 978e5c248040d1a4381712e5fbf9bf921436eb40 Mon Sep 17 00:00:00 2001 From: Thomas Hansen Date: Fri, 30 Oct 2026 09:50:48 -0500 Subject: [PATCH 2/2] Keep video identity stable through static crops and camera calibration Static crops appended the per-frame crop UUID to video_identifier, so stateful blocks fed by a crop saw a new video every frame. They now pass a stable region identifier via create_crop's new video_crop_identifier, keeping a unique parent_id. Camera Calibration rebuilt its output without video metadata, which pinned the rate limiter's video time at zero; it now uses copy_and_replace. Co-Authored-By: Claude --- workflows/CHANGELOG.md | 2 + .../absolute_static_crop/v1.py | 1 + .../absolute_static_crop/v1_tensor.py | 2 + .../transformations/camera_calibration/v1.py | 8 +- .../relative_static_crop/v1.py | 1 + .../execution_engine/entities/base.py | 14 +- .../execution_engine/entities/test_base.py | 32 +++ ...ded_video_blocks_with_step_output_image.py | 196 +++++++++++++++--- 8 files changed, 221 insertions(+), 35 deletions(-) diff --git a/workflows/CHANGELOG.md b/workflows/CHANGELOG.md index 8e49266f30..d31575868b 100644 --- a/workflows/CHANGELOG.md +++ b/workflows/CHANGELOG.md @@ -53,6 +53,8 @@ for contributor and maintainer responsibilities. - Root-coordinate recovery (`sv_detections_to_root_coordinates`, applied to workflow outputs by default) re-encodes `rle_mask` from the re-anchored dense masks. A block that ran on a crop and returns RLE with dense masks, such as SAM 3 v3 with `output_format: rle`, used to serialize root-coordinate boxes next to an RLE still sized and placed in the crop. RLE-only predictions (`mask=None`) are unchanged, as in `scale_sv_detections`. - Video-state blocks now accept an image from a step output as well as from a Workflow input, so they keep working inside an embedded `roboflow_core/inner_workflow@v1` whose `image` is bound to a parent step (for example a preprocessing block). Inlining replaces the child's `$inputs.image` with that step selector, and the merged definition used to fail with `Could not parse workflow definition` even though each Workflow compiled on its own. Affected blocks: `time_in_zone@v2`/`@v3`, `velocity@v1`, `line_counter@v2`, `path_deviation_analytics@v2`, `detection_event_log@v1`, `byte_tracker@v2`, `cache_get@v1`, `cache_set@v1`, `delta_filter@v1` and `rate_limiter@v1` (`video_reference_image`). The blocks read `video_metadata` from the image, which step outputs built with `WorkflowImageData.copy_and_replace` keep, so per-video state is unchanged. +- Absolute and relative static crops (`absolute_static_crop@v1`, `relative_statoic_crop@v1`) now give a crop the same `video_identifier` on every frame (`