diff --git a/CHANGELOG.md b/CHANGELOG.md index aec843d..013f2fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Allow `HumanRating` media and text items to include one non-selectable media + reference through `RatingItem.reference`. + ### Fixed - Re-upload local media when its contents change at the same path, keeping diff --git a/README.md b/README.md index 40bcfac..dedc089 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,36 @@ scorer = he.HumanRating( ) scores = scorer.eval_batch([resp.text for resp in responses]) +# Media rating against a non-selectable reference +identity = he.HumanRating( + "Do people, animals, or objects maintain their original identity and features after the edit?", + scale=(1, 5), + labels={1: "Not preserved", 5: "Fully preserved"}, +) +scores = identity.eval_batch( + [ + he.RatingItem( + subject=he.Media("shoe2.png"), + reference=he.Media("shoe1.png"), + ) + ] +) + +# Text description rating against reference media +description_accuracy = he.HumanRating( + "How accurately does this description match the reference image?\n\nDescription: {context}", + scale=(1, 5), + labels={1: "Not accurate", 5: "Fully accurate"}, +) +scores = description_accuracy.eval_batch( + [ + he.RatingItem( + subject="A red shoe with white laces.", + reference=he.Media("shoe1.png"), + ) + ] +) + # Multiple choice scorer = he.HumanMultipleChoice("Answer based only on the screenshot.") scores = scorer.eval_batch( diff --git a/src/humanevals/scorers.py b/src/humanevals/scorers.py index e58a85d..8c76ac6 100644 --- a/src/humanevals/scorers.py +++ b/src/humanevals/scorers.py @@ -570,7 +570,8 @@ class HumanRating(HumanScorer): labels: Optional anchor labels, e.g. ``{1: "Poor", 5: "Excellent"}``. Items are :class:`~humanevals.RatingItem` objects, or bare ``str`` / - :class:`~humanevals.Media` subjects. + :class:`~humanevals.Media` subjects. Use ``RatingItem.reference`` to show + non-selectable reference media alongside either kind of subject. """ @@ -648,15 +649,26 @@ def _datapoint( self, item: RatingItem, *, for_naming: bool, media_hashes: _MediaHashes ) -> dict[str, Any]: if isinstance(item.subject, str): - # Text-only rating: the API requires an explicit empty media dict. - return {"media": {}, "context": item.subject} - datapoint: dict[str, Any] = { - "media": { - "subject": [ - self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) + media: dict[str, list[dict[str, Any]]] = {} + if item.reference is not None: + media["reference"] = [ + self._media_item( + item.reference, + for_naming=for_naming, + media_hashes=media_hashes, + ) ] - } + return {"media": media, "context": item.subject} + media = { + "subject": [ + self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes) + ] } + if item.reference is not None: + media["reference"] = [ + self._media_item(item.reference, for_naming=for_naming, media_hashes=media_hashes) + ] + datapoint: dict[str, Any] = {"media": media} if item.context is not None: datapoint["context"] = item.context return datapoint diff --git a/src/humanevals/types.py b/src/humanevals/types.py index 1da0a75..0e9dcac 100644 --- a/src/humanevals/types.py +++ b/src/humanevals/types.py @@ -161,11 +161,15 @@ class RatingItem: also setting ``context``. context: Optional extra context for a *media* subject. Shown only if the instruction contains ``{context}``. + reference: Optional non-selectable media shown alongside the subject + (for example, the original image for an edited-image rating, or + an image whose text description is being rated). """ subject: str | Media context: str | None = None + reference: Media | None = None @dataclass(frozen=True) diff --git a/tests/test_rating.py b/tests/test_rating.py index b2a39d9..4020ece 100644 --- a/tests/test_rating.py +++ b/tests/test_rating.py @@ -68,6 +68,26 @@ def test_media_subject_with_context(client: he.Client, api: FakeAPI): assert api.body()["datapoints"][0]["context"] == "a red fox" +def test_media_subject_with_reference(client: he.Client, api: FakeAPI): + api.add("POST", "/jobs", create_job_response()) + scorer = he.HumanRating("How well does the edit preserve identity?", client=client) + scorer.submit( + [ + he.RatingItem( + subject=he.Media("dp://edited123/shoe.png"), + reference=he.Media("dp://original456/shoe.png"), + ) + ] + ) + + assert api.body()["datapoints"][0] == { + "media": { + "subject": [{"url": "dp://edited123/shoe.png", "type": "image"}], + "reference": [{"url": "dp://original456/shoe.png", "type": "image"}], + } + } + + # -- validation -------------------------------------------------------------- @@ -82,6 +102,24 @@ def test_text_subject_with_extra_context_rejected(client: he.Client): make_scorer(client).submit([he.RatingItem(subject="text", context="extra")]) +def test_text_subject_with_reference(client: he.Client, api: FakeAPI): + api.add("POST", "/jobs", create_job_response()) + + make_scorer(client).submit( + [ + he.RatingItem( + subject="A red shoe with white laces.", + reference=he.Media("dp://original456/shoe.png"), + ) + ] + ) + + assert api.body()["datapoints"][0] == { + "media": {"reference": [{"url": "dp://original456/shoe.png", "type": "image"}]}, + "context": "A red shoe with white laces.", + } + + def test_wrong_item_type_rejected(client: he.Client): with pytest.raises(TypeError, match="HumanRating items"): make_scorer(client).submit([42])