🐛 Fix(evaluation): require a selected evaluator and remove Jiuwen scoring fallback

backend/contracts/errors/codes.py

@@ -271,6 +271,7 @@ class ErrorCode(Enum):
     AGENT_EVALUATION_AGENT_NOT_FOUND = "160206"
     AGENT_EVALUATION_JUDGE_MODEL_REQUIRED = "160207"
     AGENT_EVALUATION_ONLY_CREATOR_CAN_DELETE = "160208"
+    AGENT_EVALUATION_EVALUATOR_REQUIRED = "160209"
     # 03 - AI generation
     AGENT_EVALUATION_QUERY_GENERATION_FAILED = "160301"
     AGENT_EVALUATION_QUERY_GENERATION_FORMAT = "160302"
@@ -403,6 +404,7 @@ class ErrorCode(Enum):
     ErrorCode.AGENT_EVALUATION_AGENT_NOT_FOUND: 404,
     ErrorCode.AGENT_EVALUATION_JUDGE_MODEL_REQUIRED: 400,
     ErrorCode.AGENT_EVALUATION_ONLY_CREATOR_CAN_DELETE: 403,
+    ErrorCode.AGENT_EVALUATION_EVALUATOR_REQUIRED: 400,
     ErrorCode.AGENT_EVALUATION_QUERY_GENERATION_FAILED: 500,
     ErrorCode.AGENT_EVALUATION_QUERY_GENERATION_FORMAT: 500,
     ErrorCode.AGENT_EVALUATION_QUERY_GENERATION_EMPTY: 400,

backend/contracts/errors/messages.py

@@ -166,6 +166,7 @@ class ErrorMessage:
         ErrorCode.AGENT_EVALUATION_AGENT_NOT_FOUND: "Agent not found.",
         ErrorCode.AGENT_EVALUATION_JUDGE_MODEL_REQUIRED: "Judge model ID is required.",
         ErrorCode.AGENT_EVALUATION_ONLY_CREATOR_CAN_DELETE: "Only the creator or a tenant administrator can delete this evaluation run.",
+        ErrorCode.AGENT_EVALUATION_EVALUATOR_REQUIRED: "At least one published evaluator is required.",
         ErrorCode.AGENT_EVALUATION_QUERY_GENERATION_FAILED: "Failed to generate test queries.",
         ErrorCode.AGENT_EVALUATION_QUERY_GENERATION_FORMAT: "AI returned invalid format for test queries.",
         ErrorCode.AGENT_EVALUATION_QUERY_GENERATION_EMPTY: "AI generated no valid test queries.",

backend/management/api/evaluation/agent_evaluation.py

@@ -6,11 +6,15 @@
 from typing import Any
 
 from fastapi import APIRouter, Body, Header, Query, Request
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi.responses import JSONResponse, Response, StreamingResponse
 from pydantic import BaseModel, Field, StrictInt, TypeAdapter, ValidationError
 
 from contracts.errors.codes import ErrorCode
-from contracts.errors.exceptions import AppException, UnauthorizedError
+from contracts.errors.exceptions import (
+    AppException,
+    RuntimeUpstreamError,
+    UnauthorizedError,
+)
 
 # AppException is caught by global middleware (maps ErrorCode to HTTP status)
 from management.db.evaluation.runs import update_annotation_schema_ids
@@ -552,6 +556,14 @@ async def trial_run_api(
 
     except UnauthorizedError:
         raise AppException(ErrorCode.COMMON_UNAUTHORIZED, _AUTH_REQUIRED_MSG)
+    except RuntimeUpstreamError as exc:
+        # Preserve the runtime's structured error (code/message/status) instead
+        # of collapsing a validation failure into a generic 500.
+        return Response(
+            content=exc.content,
+            status_code=exc.status_code,
+            media_type="application/json",
+        )
     except Exception as exc:
         logger.exception(
             "trial_run_api ERROR: tenant=%s user=%s agent_id=%s version=%s "

backend/management/services/evaluation/execution.py

@@ -8,18 +8,13 @@
 from contracts.errors.exceptions import AppException
 from contracts.agent import AgentRequest
 
-try:
-    from adapters.jiuwen_sdk_adapter import JiuwenSDKAdapter
-except ModuleNotFoundError:
-    JiuwenSDKAdapter = None
 import asyncio
 import json
 import uuid
 from collections import defaultdict
 from datetime import datetime
 from statistics import mean
 
-from adapters.exception import JiuwenSDKUnavailableError
 from contracts.constants.constants import MAX_TURNS_PER_SESSION
 from management.db.evaluation.runs import create_agent_evaluation_cases, get_agent_evaluation, list_agent_evaluation_cases, update_agent_evaluation_case_result, update_agent_evaluation_status
 from management.db.client import get_db_session
@@ -246,7 +241,6 @@ def _execute_single_case(
     case: dict[str, Any],
     run: dict[str, Any],
     judge_model_id: int,
-    adapter: Any,
     evaluators: dict[int, dict[str, Any]],
     judge_system_prompt: str,
     context_window: int = 4096,
@@ -304,7 +298,6 @@ def _execute_single_case(
                 agent_version_no=agent_version_no,
                 query=query,
                 judge_model_id=judge_model_id,
-                adapter=adapter,
                 evaluators=evaluators,
                 judge_system_prompt=judge_system_prompt,
                 language=run.get("language", "zh"),
@@ -559,12 +552,14 @@ def execute_agent_evaluation_run(
             raise AppException(ErrorCode.AGENT_EVALUATION_JUDGE_MODEL_REQUIRED)
         judge_model_id = int(judge_model_id)
 
-        if JiuwenSDKAdapter is None:
-            raise JiuwenSDKUnavailableError("Jiuwen SDK adapter is unavailable.")
-        adapter = JiuwenSDKAdapter(model_id=judge_model_id, tenant_id=tenant_id)
-
         # Preload evaluators and judge template (loaded once, reused for all cases)
         evaluators = _preload_evaluators_for_run(run, tenant_id)
+        if not evaluators:
+            raise AppException(
+                ErrorCode.AGENT_EVALUATION_EVALUATOR_REQUIRED,
+                "No usable evaluator is configured for this run. "
+                "Select at least one published evaluator and retry.",
+            )
         judge_system_prompt = compose_auxiliary_prompt(
             run.get("language", "zh"), "evaluation_judge"
         ).system
@@ -600,7 +595,6 @@ def execute_agent_evaluation_run(
                     case=c,
                     run=run,
                     judge_model_id=judge_model_id,
-                    adapter=adapter,
                     evaluators=evaluators,
                     judge_system_prompt=judge_system_prompt,
                     context_window=context_window,
@@ -672,7 +666,6 @@ async def _evaluate_query(
     agent_version_no: int,
     query: str,
     judge_model_id: int,
-    adapter: Any,
     evaluators: dict[int, dict[str, Any]],
     judge_system_prompt: str,
     runtime_events: list[dict] | None = None,
@@ -683,6 +676,11 @@ async def _evaluate_query(
     conversation_id: int | None = None,
 ) -> tuple[str, list[dict] | None, dict, dict]:
     """Run agent + score with evaluators. Returns (answer, events, scores, reasons)."""
+    if not evaluators:
+        raise AppException(
+            ErrorCode.AGENT_EVALUATION_EVALUATOR_REQUIRED,
+            "At least one published evaluator is required to score an evaluation.",
+        )
     answer_text, events = await _run_agent_to_final_answer(
         agent_id=agent_id,
         tenant_id=tenant_id,
@@ -692,27 +690,18 @@ async def _evaluate_query(
         history=history,
         conversation_id=conversation_id,
     )
-    if evaluators:
-        score, reason = _score_with_evaluators(
-            evaluators=evaluators,
-            judge_system_prompt=judge_system_prompt,
-            tenant_id=tenant_id,
-            query=query,
-            expected=expected,
-            actual=answer_text,
-            judge_model_id=judge_model_id,
-            runtime_events=runtime_events or events,
-            context_window=context_window,
-            conversation_history=history,
-        )
-    else:
-        score, reason = adapter.evaluate_semantic_consistency(
-            question=query,
-            expected_answer="",
-            model_answer=answer_text,
-        )
-        score = {"semantic_consistency": score}
-        reason = {"semantic_consistency": reason}
+    score, reason = _score_with_evaluators(
+        evaluators=evaluators,
+        judge_system_prompt=judge_system_prompt,
+        tenant_id=tenant_id,
+        query=query,
+        expected=expected,
+        actual=answer_text,
+        judge_model_id=judge_model_id,
+        runtime_events=runtime_events or events,
+        context_window=context_window,
+        conversation_history=history,
+    )
     return answer_text, events, score, reason
 
 

backend/management/services/evaluation/runs.py

@@ -10,10 +10,6 @@
 from contracts.agent import AgentRequest
 from nexent.core.agents.prompt.auxiliary import AuxiliaryPrompt
 
-try:
-    from adapters.jiuwen_sdk_adapter import JiuwenSDKAdapter
-except ModuleNotFoundError:
-    JiuwenSDKAdapter = None
 import ast
 import asyncio
 import json
@@ -26,7 +22,6 @@
 from math import isfinite
 from statistics import mean
 
-from adapters.exception import JiuwenSDKUnavailableError
 from contracts.constants.constants import MAX_CONCURRENT_RUNS, MAX_EVALUATORS_PER_RUN, MAX_TOTAL_RUNS, MAX_TURNS_PER_SESSION
 from contracts.constants.constants import MAX_FAILURE_EXAMPLES
 from management.db.evaluation.runs import count_active_runs, count_total_runs, create_agent_evaluation, create_agent_evaluation_cases, get_agent_evaluation, get_evaluation_case_scores, hard_delete_agent_evaluation, list_agent_evaluation_cases, list_agent_evaluations_by_agent, update_agent_evaluation_analysis_report, update_agent_evaluation_case_result, update_agent_evaluation_status
@@ -286,20 +281,20 @@ async def trial_run_evaluator_impl(
             ev = get_evaluator(eid, tenant_id)
             if ev and ev.get("status") == "PUBLISHED":
                 evaluators[eid] = ev
+    if not evaluators:
+        raise AppException(
+            ErrorCode.AGENT_EVALUATION_EVALUATOR_REQUIRED,
+            "At least one published evaluator is required for a trial run.",
+        )
     judge_system_prompt = compose_auxiliary_prompt(language, "evaluation_judge").system
 
-    if JiuwenSDKAdapter is None:
-        raise JiuwenSDKUnavailableError("Jiuwen SDK adapter is unavailable")
-    adapter = JiuwenSDKAdapter(model_id=judge_model_id, tenant_id=tenant_id)
-
     answer_text, _runtime_events, score, reason = await _evaluate_query(
         tenant_id=tenant_id,
         user_id=user_id,
         agent_id=agent_id,
         agent_version_no=agent_version_no,
         query=query,
         judge_model_id=judge_model_id,
-        adapter=adapter,
         evaluators=evaluators,
         judge_system_prompt=judge_system_prompt,
         language=language,

frontend/app/[locale]/evaluation/page.tsx

@@ -852,8 +852,17 @@ function RunsTab() {
                         }
                       );
                       const d = await r.json();
+                      if (!r.ok) {
+                        message.error(
+                          d?.message ||
+                            d?.detail ||
+                            t("agentEvaluation.trialFailed")
+                        );
+                        return;
+                      }
                       setTrialResult(d.data || d);
                     } catch {
+                      message.error(t("agentEvaluation.trialFailed"));
                     } finally {
                       setTrialRunning(false);
                     }

frontend/public/locales/en/common.json

@@ -5254,6 +5254,7 @@
   "agentEvaluation.trialResult": "Trial Result",
   "agentEvaluation.trialRun": "Trial Run",
   "agentEvaluation.trialRunning": "Running trial, please wait...",
+  "agentEvaluation.trialFailed": "Trial run failed. Please check the configuration and try again.",
   "agentEvaluation.versionRequired": "Version *",
   "agentEvaluation.scoreMin": "Score Min",
   "agentEvaluation.scoreMax": "Score Max (≤100)",

frontend/public/locales/zh/common.json

@@ -5192,6 +5192,7 @@
   "agentEvaluation.trialResult": "试运行结果",
   "agentEvaluation.trialRun": "试运行",
   "agentEvaluation.trialRunning": "试运行中，请稍候...",
+  "agentEvaluation.trialFailed": "试运行失败，请检查配置后重试",
   "agentEvaluation.versionRequired": "版本 *",
   "agentEvaluation.scoreMin": "评分下限",
   "agentEvaluation.scoreMax": "评分上限 (≤100)",

test/backend/app/test_agent_evaluation_app.py

@@ -658,6 +658,25 @@ def test_app_exception_propagates(self, client):
         )
         assert response.status_code == 404
 
+    def test_runtime_upstream_error_is_forwarded(self, client):
+        from contracts.errors.exceptions import RuntimeUpstreamError
+
+        _mock_impls(
+            forward_agent_evaluation_trial_run=AsyncMock(
+                side_effect=RuntimeUpstreamError(
+                    status_code=400,
+                    content=b'{"code":"160209","message":"At least one published evaluator is required.","details":null}',
+                    headers={"content-type": "application/json"},
+                )
+            )
+        )
+        response = client.post(
+            "/agent-evaluations/trial-run",
+            json={"agent_id": 1, "query": "q", "judge_model_id": 2},
+        )
+        assert response.status_code == 400
+        assert response.json()["code"] == "160209"
+
 
 # ---------------------------------------------------------------------------
 # safe-extract helpers