From babd438662fad092a83fdaeecb00050b51787e6c Mon Sep 17 00:00:00 2001 From: wiliyam <36193746+wiliyam@users.noreply.github.com> Date: Tue, 11 Aug 2026 05:17:19 +0000 Subject: [PATCH 1/2] feat: add StringCheckGrader support for OpenAI Evals backend Port the grader onto the current OpenAI eval backend, including label-model support added since the original review. Use a static reference in the example as requested and cover validation, criteria, item shape, and no-golden-set behavior. Co-authored-by: OpenAI Codex --- README.md | 2 +- docs/custom-evaluators.md | 18 +++++++++- examples/custom_evaluators/eval_config.yaml | 7 ++++ src/agentevals/config.py | 14 +++++++- src/agentevals/openai_eval_backend.py | 17 ++++++++-- tests/test_openai_eval_backend.py | 37 +++++++++++++++++++++ 6 files changed, 90 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9082cb4..6925c17 100644 --- a/README.md +++ b/README.md @@ -240,7 +240,7 @@ evaluators: threshold: 0.7 ``` -Evaluators with a `requirements.txt` get automatic virtual environment management. You can also use `type: remote` for community evaluators from GitHub, or `type: openai_eval` to delegate grading to the [OpenAI Evals API](https://developers.openai.com/api/reference/resources/evals/methods/create) (requires `pip install "agentevals-cli[openai]"`). +Evaluators with a `requirements.txt` get automatic virtual environment management. You can also use `type: remote` for community evaluators from GitHub, or `type: openai_eval` with text-similarity, string-check, and label-model graders to delegate grading to the [OpenAI Evals API](https://developers.openai.com/api/reference/resources/evals/methods/create) (requires `pip install "agentevals-cli[openai]"`). See the [Custom Evaluators guide](docs/custom-evaluators.md) for the full protocol reference, SDK helpers, and how to contribute evaluators. diff --git a/docs/custom-evaluators.md b/docs/custom-evaluators.md index fe6bb81..0ec41a7 100644 --- a/docs/custom-evaluators.md +++ b/docs/custom-evaluators.md @@ -337,9 +337,25 @@ evaluators: The `threshold` field is not used for `label_model`. A response passes if its assigned label is in `passing_labels`. +### String Check Grader + +Compares each response with a fixed reference string. It does not require an eval set. + +```yaml +evaluators: + - name: city_name_check + type: openai_eval + grader: + type: string_check + operation: eq + reference: Paris +``` + +Supported operations are `eq`, `ne`, `like`, and `ilike`. The `threshold` field is not used for `string_check`; each comparison returns either 0 or 1. + ### How it works -Under the hood, agentevals creates an ephemeral eval on OpenAI, submits the actual and expected responses as JSONL items, polls for results, and cleans up. The agent's response and the golden reference are both placed in the `item` namespace (with `include_sample_schema: false`), so OpenAI only grades the provided text without generating any model outputs. +Under the hood, agentevals creates an ephemeral eval on OpenAI, submits response data as JSONL items, polls for results, and cleans up. Text-similarity graders receive actual and expected responses; string-check and label-model graders only receive the actual response. With `include_sample_schema: false`, OpenAI only grades the provided text without generating model outputs. ### Configuring the GitHub source diff --git a/examples/custom_evaluators/eval_config.yaml b/examples/custom_evaluators/eval_config.yaml index 59171b6..9fa5dff 100644 --- a/examples/custom_evaluators/eval_config.yaml +++ b/examples/custom_evaluators/eval_config.yaml @@ -33,3 +33,10 @@ evaluators: threshold: 0.110 executor: local + # OpenAI Evals API grader (requires OPENAI_API_KEY) + - name: city_name_check + type: openai_eval + grader: + type: string_check + operation: eq + reference: Paris diff --git a/src/agentevals/config.py b/src/agentevals/config.py index cf3c194..e69a7a4 100644 --- a/src/agentevals/config.py +++ b/src/agentevals/config.py @@ -94,6 +94,8 @@ class RemoteEvaluatorDef(BaseEvaluatorDef): } ) +_VALID_STRING_CHECK_OPERATIONS = frozenset({"eq", "ne", "like", "ilike"}) + class OpenAIEvalDef(BaseModel): """An evaluator that delegates grading to the OpenAI Evals API.""" @@ -121,8 +123,18 @@ def _validate_grader(cls, v: dict[str, Any]) -> dict[str, Any]: invalid = [lbl for lbl in v["passing_labels"] if lbl not in v["labels"]] if invalid: raise ValueError(f"passing_labels contains labels not declared in labels: {invalid}") + elif grader_type == "string_check": + operation = v.get("operation") + if not operation: + raise ValueError("'operation' is required for string_check grader") + if operation not in _VALID_STRING_CHECK_OPERATIONS: + raise ValueError(f"Unknown operation '{operation}'. Valid: {sorted(_VALID_STRING_CHECK_OPERATIONS)}") + if not v.get("reference"): + raise ValueError("'reference' is required for string_check grader") else: - raise ValueError(f"Unsupported grader type: '{grader_type}'. Supported: label_model, text_similarity") + raise ValueError( + f"Unsupported grader type: '{grader_type}'. Supported: label_model, string_check, text_similarity" + ) return v diff --git a/src/agentevals/openai_eval_backend.py b/src/agentevals/openai_eval_backend.py index e3c5cab..9172654 100644 --- a/src/agentevals/openai_eval_backend.py +++ b/src/agentevals/openai_eval_backend.py @@ -67,6 +67,15 @@ def _build_testing_criteria(evaluator_def: OpenAIEvalDef) -> dict[str, Any]: "passing_labels": grader["passing_labels"], } + if grader_type == "string_check": + return { + "type": "string_check", + "name": evaluator_def.name, + "input": "{{ item.actual_response }}", + "reference": grader["reference"], + "operation": grader["operation"], + } + raise ValueError(f"Unsupported grader type: {grader_type}") @@ -131,7 +140,9 @@ async def evaluate_openai_eval( ) items = _build_jsonl_items( - actual_invocations, expected_invocations or [], include_expected=(grader_type != "label_model") + actual_invocations, + expected_invocations or [], + include_expected=(grader_type == "text_similarity"), ) if not items: return MetricResult( @@ -145,7 +156,7 @@ async def evaluate_openai_eval( try: client = await asyncio.to_thread(_get_openai_client) - item_schema = _ACTUAL_ONLY_SCHEMA if grader_type == "label_model" else _TEXT_PAIR_SCHEMA + item_schema = _TEXT_PAIR_SCHEMA if grader_type == "text_similarity" else _ACTUAL_ONLY_SCHEMA eval_obj = await asyncio.to_thread( client.evals.create, name=f"agentevals-openai-{evaluator_def.name}", @@ -252,6 +263,8 @@ async def _collect_results(client: Any, eval_id: str, run_id: str, run: Any, eva elif grader["type"] == "label_model": details["model"] = grader.get("model") details["passing_labels"] = grader.get("passing_labels") + elif grader["type"] == "string_check": + details["operation"] = grader.get("operation") per_criteria = getattr(run, "per_testing_criteria_results", None) if per_criteria: details["per_testing_criteria"] = [ diff --git a/tests/test_openai_eval_backend.py b/tests/test_openai_eval_backend.py index 7ea0f2a..69b2855 100644 --- a/tests/test_openai_eval_backend.py +++ b/tests/test_openai_eval_backend.py @@ -56,6 +56,21 @@ def test_label_model_passing_labels_not_in_labels(self): with pytest.raises(Exception, match="passing_labels"): OpenAIEvalDef(name="lm", grader=grader) + def test_string_check_valid(self): + d = OpenAIEvalDef(name="check", grader={"type": "string_check", "operation": "eq", "reference": "Paris"}) + assert d.grader["type"] == "string_check" + + @pytest.mark.parametrize("field", ["operation", "reference"]) + def test_string_check_missing_required_field(self, field): + grader = {"type": "string_check", "operation": "eq", "reference": "Paris"} + grader.pop(field) + with pytest.raises(Exception, match=field): + OpenAIEvalDef(name="check", grader=grader) + + def test_string_check_invalid_operation(self): + with pytest.raises(Exception, match="Unknown operation"): + OpenAIEvalDef(name="check", grader={"type": "string_check", "operation": "contains", "reference": "Paris"}) + def test_unsupported_grader_type(self): with pytest.raises(Exception, match="Unsupported grader type"): OpenAIEvalDef(name="x", grader={"type": "unknown"}) @@ -81,6 +96,17 @@ def test_label_model_shape(self): assert c["passing_labels"] == ["good"] assert c["input"] == grader["input"] + def test_string_check_shape(self): + d = OpenAIEvalDef(name="check", grader={"type": "string_check", "operation": "ilike", "reference": "Paris"}) + c = _build_testing_criteria(d) + assert c == { + "type": "string_check", + "name": "check", + "input": "{{ item.actual_response }}", + "reference": "Paris", + "operation": "ilike", + } + class TestBuildJsonlItems: def test_text_similarity_includes_expected(self): @@ -91,6 +117,10 @@ def test_label_model_excludes_expected(self): items = _build_jsonl_items([_invocation("hello")], [], include_expected=False) assert "expected_response" not in items[0]["item"] + def test_string_check_excludes_expected(self): + items = _build_jsonl_items([_invocation("Paris")], [], include_expected=False) + assert items == [{"item": {"actual_response": "Paris"}}] + def test_missing_expected_falls_back_to_empty(self): items = _build_jsonl_items([_invocation("hello")], [], include_expected=True) assert items[0]["item"]["expected_response"] == "" @@ -115,3 +145,10 @@ async def test_label_model_does_not_require_expected(self, monkeypatch): d = OpenAIEvalDef(name="lm", grader=_label_grader()) result = await evaluate_openai_eval(d, [_invocation("hi")], None) assert "expected invocations" not in (result.error or "") + + async def test_string_check_does_not_require_expected(self, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setattr("agentevals.openai_eval_backend._get_openai_client", lambda: None) + d = OpenAIEvalDef(name="check", grader={"type": "string_check", "operation": "eq", "reference": "Paris"}) + result = await evaluate_openai_eval(d, [_invocation("Paris")], None) + assert "expected invocations" not in (result.error or "") From 68f1252513ecf86ee29801866057f7092eb00f02 Mon Sep 17 00:00:00 2001 From: wiliyam <36193746+wiliyam@users.noreply.github.com> Date: Sat, 15 Aug 2026 03:20:30 +0000 Subject: [PATCH 2/2] fix: address string check review feedback --- examples/custom_evaluators/eval_config.yaml | 8 -------- .../eval_config_openai_eval.yaml | 7 +++++++ src/agentevals/config.py | 10 +++++----- src/agentevals/openai_eval_backend.py | 9 +++++++-- tests/test_openai_eval_backend.py | 19 +++++++++++++++---- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/examples/custom_evaluators/eval_config.yaml b/examples/custom_evaluators/eval_config.yaml index 9fa5dff..d3bd261 100644 --- a/examples/custom_evaluators/eval_config.yaml +++ b/examples/custom_evaluators/eval_config.yaml @@ -32,11 +32,3 @@ evaluators: ref: evaluators/random_evaluator/random_evaluator.py threshold: 0.110 executor: local - - # OpenAI Evals API grader (requires OPENAI_API_KEY) - - name: city_name_check - type: openai_eval - grader: - type: string_check - operation: eq - reference: Paris diff --git a/examples/custom_evaluators/eval_config_openai_eval.yaml b/examples/custom_evaluators/eval_config_openai_eval.yaml index fb04802..34ebd30 100644 --- a/examples/custom_evaluators/eval_config_openai_eval.yaml +++ b/examples/custom_evaluators/eval_config_openai_eval.yaml @@ -16,3 +16,10 @@ evaluators: content: "Rate this response: {{ item.actual_response }}" labels: [good, bad] passing_labels: [good] + + - name: helm_release_check + type: openai_eval + grader: + type: string_check + operation: ilike + reference: "*two Helm releases*" diff --git a/src/agentevals/config.py b/src/agentevals/config.py index e69a7a4..90ec457 100644 --- a/src/agentevals/config.py +++ b/src/agentevals/config.py @@ -95,6 +95,7 @@ class RemoteEvaluatorDef(BaseEvaluatorDef): ) _VALID_STRING_CHECK_OPERATIONS = frozenset({"eq", "ne", "like", "ilike"}) +_SUPPORTED_GRADER_TYPES = frozenset({"label_model", "string_check", "text_similarity"}) class OpenAIEvalDef(BaseModel): @@ -129,12 +130,11 @@ def _validate_grader(cls, v: dict[str, Any]) -> dict[str, Any]: raise ValueError("'operation' is required for string_check grader") if operation not in _VALID_STRING_CHECK_OPERATIONS: raise ValueError(f"Unknown operation '{operation}'. Valid: {sorted(_VALID_STRING_CHECK_OPERATIONS)}") - if not v.get("reference"): - raise ValueError("'reference' is required for string_check grader") + reference = v.get("reference") + if not isinstance(reference, str) or not reference: + raise ValueError("'reference' must be a non-empty string for string_check grader") else: - raise ValueError( - f"Unsupported grader type: '{grader_type}'. Supported: label_model, string_check, text_similarity" - ) + raise ValueError(f"Unsupported grader type: '{grader_type}'. Supported: {sorted(_SUPPORTED_GRADER_TYPES)}") return v diff --git a/src/agentevals/openai_eval_backend.py b/src/agentevals/openai_eval_backend.py index 9172654..d9cb50a 100644 --- a/src/agentevals/openai_eval_backend.py +++ b/src/agentevals/openai_eval_backend.py @@ -38,6 +38,11 @@ } +def _get_item_schema(grader_type: str) -> dict[str, Any]: + """Return the data-source item schema required by a grader type.""" + return _TEXT_PAIR_SCHEMA if grader_type == "text_similarity" else _ACTUAL_ONLY_SCHEMA + + def _build_testing_criteria(evaluator_def: OpenAIEvalDef) -> dict[str, Any]: """Build the OpenAI testing_criteria dict from the evaluator config. @@ -139,10 +144,11 @@ async def evaluate_openai_eval( error="OpenAI text_similarity grader requires expected invocations (golden eval set).", ) + item_schema = _get_item_schema(grader_type) items = _build_jsonl_items( actual_invocations, expected_invocations or [], - include_expected=(grader_type == "text_similarity"), + include_expected="expected_response" in item_schema["required"], ) if not items: return MetricResult( @@ -156,7 +162,6 @@ async def evaluate_openai_eval( try: client = await asyncio.to_thread(_get_openai_client) - item_schema = _TEXT_PAIR_SCHEMA if grader_type == "text_similarity" else _ACTUAL_ONLY_SCHEMA eval_obj = await asyncio.to_thread( client.evals.create, name=f"agentevals-openai-{evaluator_def.name}", diff --git a/tests/test_openai_eval_backend.py b/tests/test_openai_eval_backend.py index 69b2855..c45c85a 100644 --- a/tests/test_openai_eval_backend.py +++ b/tests/test_openai_eval_backend.py @@ -6,6 +6,7 @@ from agentevals.openai_eval_backend import ( _build_jsonl_items, _build_testing_criteria, + _get_item_schema, evaluate_openai_eval, ) @@ -71,6 +72,11 @@ def test_string_check_invalid_operation(self): with pytest.raises(Exception, match="Unknown operation"): OpenAIEvalDef(name="check", grader={"type": "string_check", "operation": "contains", "reference": "Paris"}) + @pytest.mark.parametrize("reference", [None, "", 123, ["Paris"]]) + def test_string_check_reference_must_be_non_empty_string(self, reference): + with pytest.raises(Exception, match="reference"): + OpenAIEvalDef(name="check", grader={"type": "string_check", "operation": "eq", "reference": reference}) + def test_unsupported_grader_type(self): with pytest.raises(Exception, match="Unsupported grader type"): OpenAIEvalDef(name="x", grader={"type": "unknown"}) @@ -117,15 +123,20 @@ def test_label_model_excludes_expected(self): items = _build_jsonl_items([_invocation("hello")], [], include_expected=False) assert "expected_response" not in items[0]["item"] - def test_string_check_excludes_expected(self): - items = _build_jsonl_items([_invocation("Paris")], [], include_expected=False) - assert items == [{"item": {"actual_response": "Paris"}}] - def test_missing_expected_falls_back_to_empty(self): items = _build_jsonl_items([_invocation("hello")], [], include_expected=True) assert items[0]["item"]["expected_response"] == "" +class TestGetItemSchema: + def test_text_similarity_requires_expected_response(self): + assert "expected_response" in _get_item_schema("text_similarity")["required"] + + @pytest.mark.parametrize("grader_type", ["label_model", "string_check"]) + def test_other_graders_only_require_actual_response(self, grader_type): + assert _get_item_schema(grader_type)["required"] == ["actual_response"] + + class TestEvaluateOpenAIEval: async def test_no_api_key_returns_error(self, monkeypatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False)