Add rating reference media to HumanRating

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Allow `HumanRating` media and text items to include one non-selectable media
+  reference through `RatingItem.reference`.
+
 ### Fixed
 
 - Re-upload local media when its contents change at the same path, keeping

README.md

@@ -91,6 +91,36 @@ scorer = he.HumanRating(
 )
 scores = scorer.eval_batch([resp.text for resp in responses])
 
+# Media rating against a non-selectable reference
+identity = he.HumanRating(
+    "Do people, animals, or objects maintain their original identity and features after the edit?",
+    scale=(1, 5),
+    labels={1: "Not preserved", 5: "Fully preserved"},
+)
+scores = identity.eval_batch(
+    [
+        he.RatingItem(
+            subject=he.Media("shoe2.png"),
+            reference=he.Media("shoe1.png"),
+        )
+    ]
+)
+
+# Text description rating against reference media
+description_accuracy = he.HumanRating(
+    "How accurately does this description match the reference image?\n\nDescription: {context}",
+    scale=(1, 5),
+    labels={1: "Not accurate", 5: "Fully accurate"},
+)
+scores = description_accuracy.eval_batch(
+    [
+        he.RatingItem(
+            subject="A red shoe with white laces.",
+            reference=he.Media("shoe1.png"),
+        )
+    ]
+)
+
 # Multiple choice
 scorer = he.HumanMultipleChoice("Answer based only on the screenshot.")
 scores = scorer.eval_batch(

src/humanevals/scorers.py

@@ -570,7 +570,8 @@ class HumanRating(HumanScorer):
         labels: Optional anchor labels, e.g. ``{1: "Poor", 5: "Excellent"}``.
 
     Items are :class:`~humanevals.RatingItem` objects, or bare ``str`` /
-    :class:`~humanevals.Media` subjects.
+    :class:`~humanevals.Media` subjects. Use ``RatingItem.reference`` to show
+    non-selectable reference media alongside either kind of subject.
 
     """
 
@@ -648,15 +649,26 @@ def _datapoint(
         self, item: RatingItem, *, for_naming: bool, media_hashes: _MediaHashes
     ) -> dict[str, Any]:
         if isinstance(item.subject, str):
-            # Text-only rating: the API requires an explicit empty media dict.
-            return {"media": {}, "context": item.subject}
-        datapoint: dict[str, Any] = {
-            "media": {
-                "subject": [
-                    self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes)
+            media: dict[str, list[dict[str, Any]]] = {}
+            if item.reference is not None:
+                media["reference"] = [
+                    self._media_item(
+                        item.reference,
+                        for_naming=for_naming,
+                        media_hashes=media_hashes,
+                    )
                 ]
-            }
+            return {"media": media, "context": item.subject}
+        media = {
+            "subject": [
+                self._media_item(item.subject, for_naming=for_naming, media_hashes=media_hashes)
+            ]
         }
+        if item.reference is not None:
+            media["reference"] = [
+                self._media_item(item.reference, for_naming=for_naming, media_hashes=media_hashes)
+            ]
+        datapoint: dict[str, Any] = {"media": media}
         if item.context is not None:
             datapoint["context"] = item.context
         return datapoint

src/humanevals/types.py

@@ -161,11 +161,15 @@ class RatingItem:
             also setting ``context``.
         context: Optional extra context for a *media* subject. Shown only if
             the instruction contains ``{context}``.
+        reference: Optional non-selectable media shown alongside the subject
+            (for example, the original image for an edited-image rating, or
+            an image whose text description is being rated).
 
     """
 
     subject: str | Media
     context: str | None = None
+    reference: Media | None = None
 
 
 @dataclass(frozen=True)

tests/test_rating.py

@@ -68,6 +68,26 @@ def test_media_subject_with_context(client: he.Client, api: FakeAPI):
     assert api.body()["datapoints"][0]["context"] == "a red fox"
 
 
+def test_media_subject_with_reference(client: he.Client, api: FakeAPI):
+    api.add("POST", "/jobs", create_job_response())
+    scorer = he.HumanRating("How well does the edit preserve identity?", client=client)
+    scorer.submit(
+        [
+            he.RatingItem(
+                subject=he.Media("dp://edited123/shoe.png"),
+                reference=he.Media("dp://original456/shoe.png"),
+            )
+        ]
+    )
+
+    assert api.body()["datapoints"][0] == {
+        "media": {
+            "subject": [{"url": "dp://edited123/shoe.png", "type": "image"}],
+            "reference": [{"url": "dp://original456/shoe.png", "type": "image"}],
+        }
+    }
+
+
 # -- validation --------------------------------------------------------------
 
 
@@ -82,6 +102,24 @@ def test_text_subject_with_extra_context_rejected(client: he.Client):
         make_scorer(client).submit([he.RatingItem(subject="text", context="extra")])
 
 
+def test_text_subject_with_reference(client: he.Client, api: FakeAPI):
+    api.add("POST", "/jobs", create_job_response())
+
+    make_scorer(client).submit(
+        [
+            he.RatingItem(
+                subject="A red shoe with white laces.",
+                reference=he.Media("dp://original456/shoe.png"),
+            )
+        ]
+    )
+
+    assert api.body()["datapoints"][0] == {
+        "media": {"reference": [{"url": "dp://original456/shoe.png", "type": "image"}]},
+        "context": "A red shoe with white laces.",
+    }
+
+
 def test_wrong_item_type_rejected(client: he.Client):
     with pytest.raises(TypeError, match="HumanRating items"):
         make_scorer(client).submit([42])