From fff542ff88e71ec16fb3eabf8a5a7c110fb90e66 Mon Sep 17 00:00:00 2001 From: Shayanide Date: Thu, 20 Aug 2026 19:01:35 +0000 Subject: [PATCH 1/2] Support rating reference media --- CHANGELOG.md | 5 +++++ README.md | 15 +++++++++++++++ src/humanevals/scorers.py | 20 +++++++++++++------- src/humanevals/types.py | 5 +++++ tests/test_rating.py | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec843d..f94ecda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Allow `HumanRating` media items to include one non-selectable reference + through `RatingItem.reference`. + ### Fixed - Re-upload local media when its contents change at the same path, keeping diff --git a/README.md b/README.md index 40bcfac..8f9e37a 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,21 @@ scorer = he.HumanRating( ) scores = scorer.eval_batch([resp.text for resp in responses]) +# Media rating against a non-selectable reference +identity = he.HumanRating( + "Do people, animals, or objects maintain their original identity and features after the edit?", + scale=(1, 5), + labels={1: "Not preserved", 5: "Fully preserved"}, +) +scores = identity.eval_batch( + [ + he.RatingItem( + subject=he.Media("shoe2.png"), + reference=he.Media("shoe1.png"), + ) + ] +) + # Multiple choice scorer = he.HumanMultipleChoice("Answer based only on the screenshot.") scores = scorer.eval_batch( diff --git a/src/humanevals/scorers.py b/src/humanevals/scorers.py index e58a85d..049ec23 100644 --- a/src/humanevals/scorers.py +++ b/src/humanevals/scorers.py @@ -570,7 +570,8 @@ class HumanRating(HumanScorer): labels: Optional anchor labels, e.g. ``{1: "Poor", 5: "Excellent"}``. Items are :class:`~humanevals.RatingItem` objects, or bare ``str`` / - :class:`~humanevals.Media` subjects. + :class:`~humanevals.Media` subjects. Use ``RatingItem.reference`` to show + non-selectable reference media alongside a media subject. """ @@ -615,6 +616,8 @@ def _coerce_item(self, item: Any) -> RatingItem: def _validate_batch(self, items: list[RatingItem]) -> None: for item in items: if isinstance(item.subject, str): + if item.reference is not None: + raise ValueError("Text rating subjects cannot carry a media reference.") if "{context}" not in self.instruction: raise ValueError( "Text subjects are shown through the instruction's {context} " @@ -650,13 +653,16 @@ def _datapoint( if isinstance(item.subject, str): # Text-only rating: the API requires an explicit empty media dict. return {"media": {}, "context": item.subject} - datapoint: dict[str, Any] = { - "media": { - "subject": [ - self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) - ] - } + media = { + "subject": [ + self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) + ] } + if item.reference is not None: + media["reference"] = [ + self._media_item(item.reference, for_naming=for_naming, media_hashes=media_hashes) + ] + datapoint: dict[str, Any] = {"media": media} if item.context is not None: datapoint["context"] = item.context return datapoint diff --git a/src/humanevals/types.py b/src/humanevals/types.py index 1da0a75..2dbade7 100644 --- a/src/humanevals/types.py +++ b/src/humanevals/types.py @@ -161,11 +161,16 @@ class RatingItem: also setting ``context``. context: Optional extra context for a *media* subject. Shown only if the instruction contains ``{context}``. + reference: Optional non-selectable media shown alongside the subject + (for example, the original image for an edited-image rating). + References require a media subject; text-only ratings cannot + carry one. """ subject: str | Media context: str | None = None + reference: Media | None = None @dataclass(frozen=True) diff --git a/tests/test_rating.py b/tests/test_rating.py index b2a39d9..06baad6 100644 --- a/tests/test_rating.py +++ b/tests/test_rating.py @@ -68,6 +68,26 @@ def test_media_subject_with_context(client: he.Client, api: FakeAPI): assert api.body()["datapoints"][0]["context"] == "a red fox" +def test_media_subject_with_reference(client: he.Client, api: FakeAPI): + api.add("POST", "/jobs", create_job_response()) + scorer = he.HumanRating("How well does the edit preserve identity?", client=client) + scorer.submit( + [ + he.RatingItem( + subject=he.Media("dp://edited123/shoe.png"), + reference=he.Media("dp://original456/shoe.png"), + ) + ] + ) + + assert api.body()["datapoints"][0] == { + "media": { + "subject": [{"url": "dp://edited123/shoe.png", "type": "image"}], + "reference": [{"url": "dp://original456/shoe.png", "type": "image"}], + } + } + + # -- validation -------------------------------------------------------------- @@ -82,6 +102,18 @@ def test_text_subject_with_extra_context_rejected(client: he.Client): make_scorer(client).submit([he.RatingItem(subject="text", context="extra")]) +def test_text_subject_with_reference_rejected(client: he.Client): + with pytest.raises(ValueError, match="cannot carry a media reference"): + make_scorer(client).submit( + [ + he.RatingItem( + subject="text", + reference=he.Media("dp://original456/shoe.png"), + ) + ] + ) + + def test_wrong_item_type_rejected(client: he.Client): with pytest.raises(TypeError, match="HumanRating items"): make_scorer(client).submit([42]) From 755d0a9d9cba3b5ce8f7f0be0a1c6111f492660f Mon Sep 17 00:00:00 2001 From: Shayanide Date: Thu, 20 Aug 2026 19:42:31 +0000 Subject: [PATCH 2/2] Support text ratings with reference media --- CHANGELOG.md | 4 ++-- README.md | 15 +++++++++++++++ src/humanevals/scorers.py | 16 +++++++++++----- src/humanevals/types.py | 5 ++--- tests/test_rating.py | 26 ++++++++++++++++---------- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f94ecda..013f2fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Allow `HumanRating` media items to include one non-selectable reference - through `RatingItem.reference`. +- Allow `HumanRating` media and text items to include one non-selectable media + reference through `RatingItem.reference`. ### Fixed diff --git a/README.md b/README.md index 8f9e37a..dedc089 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,21 @@ scores = identity.eval_batch( ] ) +# Text description rating against reference media +description_accuracy = he.HumanRating( + "How accurately does this description match the reference image?\n\nDescription: {context}", + scale=(1, 5), + labels={1: "Not accurate", 5: "Fully accurate"}, +) +scores = description_accuracy.eval_batch( + [ + he.RatingItem( + subject="A red shoe with white laces.", + reference=he.Media("shoe1.png"), + ) + ] +) + # Multiple choice scorer = he.HumanMultipleChoice("Answer based only on the screenshot.") scores = scorer.eval_batch( diff --git a/src/humanevals/scorers.py b/src/humanevals/scorers.py index 049ec23..8c76ac6 100644 --- a/src/humanevals/scorers.py +++ b/src/humanevals/scorers.py @@ -571,7 +571,7 @@ class HumanRating(HumanScorer): Items are :class:`~humanevals.RatingItem` objects, or bare ``str`` / :class:`~humanevals.Media` subjects. Use ``RatingItem.reference`` to show - non-selectable reference media alongside a media subject. + non-selectable reference media alongside either kind of subject. """ @@ -616,8 +616,6 @@ def _coerce_item(self, item: Any) -> RatingItem: def _validate_batch(self, items: list[RatingItem]) -> None: for item in items: if isinstance(item.subject, str): - if item.reference is not None: - raise ValueError("Text rating subjects cannot carry a media reference.") if "{context}" not in self.instruction: raise ValueError( "Text subjects are shown through the instruction's {context} " @@ -651,8 +649,16 @@ def _datapoint( self, item: RatingItem, *, for_naming: bool, media_hashes: _MediaHashes ) -> dict[str, Any]: if isinstance(item.subject, str): - # Text-only rating: the API requires an explicit empty media dict. - return {"media": {}, "context": item.subject} + media: dict[str, list[dict[str, Any]]] = {} + if item.reference is not None: + media["reference"] = [ + self._media_item( + item.reference, + for_naming=for_naming, + media_hashes=media_hashes, + ) + ] + return {"media": media, "context": item.subject} media = { "subject": [ self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) diff --git a/src/humanevals/types.py b/src/humanevals/types.py index 2dbade7..0e9dcac 100644 --- a/src/humanevals/types.py +++ b/src/humanevals/types.py @@ -162,9 +162,8 @@ class RatingItem: context: Optional extra context for a *media* subject. Shown only if the instruction contains ``{context}``. reference: Optional non-selectable media shown alongside the subject - (for example, the original image for an edited-image rating). - References require a media subject; text-only ratings cannot - carry one. + (for example, the original image for an edited-image rating, or + an image whose text description is being rated). """ diff --git a/tests/test_rating.py b/tests/test_rating.py index 06baad6..4020ece 100644 --- a/tests/test_rating.py +++ b/tests/test_rating.py @@ -102,16 +102,22 @@ def test_text_subject_with_extra_context_rejected(client: he.Client): make_scorer(client).submit([he.RatingItem(subject="text", context="extra")]) -def test_text_subject_with_reference_rejected(client: he.Client): - with pytest.raises(ValueError, match="cannot carry a media reference"): - make_scorer(client).submit( - [ - he.RatingItem( - subject="text", - reference=he.Media("dp://original456/shoe.png"), - ) - ] - ) +def test_text_subject_with_reference(client: he.Client, api: FakeAPI): + api.add("POST", "/jobs", create_job_response()) + + make_scorer(client).submit( + [ + he.RatingItem( + subject="A red shoe with white laces.", + reference=he.Media("dp://original456/shoe.png"), + ) + ] + ) + + assert api.body()["datapoints"][0] == { + "media": {"reference": [{"url": "dp://original456/shoe.png", "type": "image"}]}, + "context": "A red shoe with white laces.", + } def test_wrong_item_type_rejected(client: he.Client):