feat(studio): persist evaluation datasets and feedback in TOS

.github/workflows/harness-sidecar-release-gate.yaml

@@ -136,7 +136,7 @@ jobs:
               tests/cli/test_generated_agent_mcp.py \
               tests/cli/test_generated_agent_backend_codegen_extended.py::test_generated_debug_applies_published_mcp_credential_contract_before_discovery \
               tests/cli/test_studio_rbac.py::test_deployment_status_recovers_completed_update_from_fresh_instance \
-              tests/cli/test_studio_rbac.py::test_new_deployment_only_updates_non_default_instance_range \
+              tests/cli/test_studio_rbac.py::test_new_deployment_creates_requested_instance_range_without_republishing \
               tests/cli/test_studio_rbac.py::test_sidecar_deployment_uses_agentkit_cli_structured_release \
               tests/cli/test_studio_rbac.py::test_source_preserving_disabled_sidecar_contract_filters_harness_env \
               tests/cli/test_studio_rbac.py::test_source_preserving_output_repository_migrates_only_tagless_runtimes \

frontend/README.md

@@ -1565,3 +1565,36 @@ Tokens 使用带展开提示的轻量文字按钮，可通过悬浮、键盘聚
 `usage` 事件仍保留兼容；已有数据库只增加 `run_turns.metrics` 列。回滚到旧代码时应
 使用新的短期数据库路径，因为旧版本按固定列数写入 `run_turns`。当前单实例云部署
 在实例替换后丢失短期记录的约定不变。
+
+### Runtime evaluation storage
+
+Studio stores evaluation sets and samples in the configured private TOS bucket
+(`VEADK_STUDIO_TOS_BUCKET` and `VEADK_STUDIO_TOS_REGION`). Volcengine and BytePlus
+use their respective server credentials. No AgentKit evaluation APIs are used.
+
+Objects live under `veadk-studio/v1/evaluation/<runtime-id>/`:
+
+- `sets/good.json` and `sets/bad.json` are the two default sets
+- `sets/<id>.json` stores a user-created set and its editable name and description
+- `samples/<id>.json` stores an individual sample, its set ID and `user` or `auto` source
+
+App names, projects and cloud regions are not part of the storage identity.
+Use separate buckets for local and production installations. Existing AgentKit
+sets are not imported.
+
+New Runtime deployments apply the requested instance limits at creation and
+wait for the configured instances to become ready before reporting success.
+This avoids starting the first conversation on a temporary startup instance.
+Runtime sessions still use the Agent's configured short-term memory backend;
+use database-backed memory when sessions must survive instance replacement.
+
+The existing evaluation page and conversation feedback controls use the same
+TOS records. Runtime-scoped APIs support custom set CRUD, sample editing and
+moving, source filtering, search and pagination. Updates use TOS ETag conditions;
+a stale edit returns a conflict. Deletion removes
+sample content and retains a small marker so automatic retries cannot recreate
+it. Deleting a default set stops feedback writes to that set and prevents background recreation; renaming it keeps feedback routing intact.
+
+List filtering and counts currently scan only the selected Runtime's objects,
+with bounded parallel reads. This is intended for the initial dataset sizes;
+large collections will need a separate rebuildable query index.

frontend/server/evaluation/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Studio-owned evaluation storage and management"""
+
+from .repository import EvaluationStorage
+
+__all__ = ["EvaluationStorage"]

frontend/server/evaluation/automatic.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Adapt automatic evaluation output to the shared Runtime sample store"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from frontend.server.evaluation_automation.models import AutoEvaluationCase
+
+from .models import Sample
+from .repository import TosEvaluationRepository, sample_id
+
+
+class TosAutomaticCases:
+    def __init__(self, repository: TosEvaluationRepository) -> None:
+        self.repository = repository
+
+    async def upsert(self, case: AutoEvaluationCase) -> AutoEvaluationCase:
+        identifier = sample_id(
+            "auto",
+            case.user_id,
+            case.session_id,
+            case.message_id,
+            case.evaluator_version,
+        )
+        saved = await self.repository.save_automatic(
+            Sample(
+                id=identifier,
+                evaluationSetId=case.kind,
+                source="auto",
+                kind=case.kind,
+                input=case.input,
+                output=case.output,
+                referenceOutput=case.reference_output,
+                comment=case.comment,
+                userId=case.user_id,
+                sessionId=case.session_id,
+                messageId=case.message_id,
+                invocationId=case.invocation_id,
+                score=case.score,
+                reason=case.reason,
+                evaluatorVersion=case.evaluator_version,
+            )
+        )
+        return case.model_copy(
+            update={
+                "id": identifier,
+                "item_key": identifier,
+                "evaluation_set_id": saved.evaluation_set_id if saved else case.kind,
+            }
+        )
+
+    async def list_cases(
+        self, *, agent_name: str, page_size: int = 100
+    ) -> list[AutoEvaluationCase]:
+        from frontend.server.evaluation_automation.models import AutoEvaluationCase
+
+        rows = await self.repository.list_samples()
+        return [
+            AutoEvaluationCase(
+                id=row.id,
+                itemKey=row.id,
+                kind=row.kind,
+                input=row.input,
+                output=row.output,
+                referenceOutput=row.reference_output,
+                comment=row.comment,
+                agentName=agent_name,
+                sessionId=row.session_id,
+                messageId=row.message_id,
+                runtimeId=self.repository.runtime_id,
+                invocationId=row.invocation_id,
+                userId=row.user_id,
+                createdAt=row.created_at,
+                evaluationSetId=row.evaluation_set_id,
+                score=row.score,
+                reason=row.reason,
+                evaluatorVersion=row.evaluator_version,
+            )
+            for row in rows
+            if row.source == "auto" and row.kind is not None and row.score is not None
+        ][:page_size]

frontend/server/evaluation/models.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Runtime-owned evaluation collections and samples"""
+
+from datetime import datetime, timezone
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+def now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class Record(BaseModel):
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+    schema_version: Literal[1] = Field(default=1, alias="schemaVersion")
+    id: str
+    created_at: str = Field(default_factory=now, alias="createdAt")
+    updated_at: str = Field(default_factory=now, alias="updatedAt")
+    deleted_at: str | None = Field(default=None, alias="deletedAt")
+    revision: str = Field(default="", exclude=True)
+
+
+class EvaluationSet(Record):
+    name: str = Field(min_length=1, max_length=100)
+    description: str = Field(default="", max_length=4000)
+    default_kind: Literal["good", "bad"] | None = Field(
+        default=None, alias="defaultKind"
+    )
+    created_by: str = Field(default="", alias="createdBy")
+
+
+class Sample(Record):
+    evaluation_set_id: str = Field(alias="evaluationSetId", min_length=1)
+    source: Literal["user", "auto"]
+    kind: Literal["good", "bad"] | None = None
+    input: str = Field(max_length=200_000)
+    output: str = Field(max_length=200_000)
+    reference_output: str = Field(
+        default="", alias="referenceOutput", max_length=200_000
+    )
+    comment: str = Field(default="", max_length=20_000)
+    user_id: str = Field(default="", alias="userId")
+    session_id: str = Field(default="", alias="sessionId")
+    message_id: str = Field(default="", alias="messageId")
+    invocation_id: str = Field(default="", alias="invocationId")
+    score: float | None = Field(default=None, ge=0, le=1)
+    reason: str = Field(default="", max_length=20_000)
+    evaluator_version: str = Field(default="", alias="evaluatorVersion")
+
+
+def public_record(record: Record) -> dict:
+    return {
+        **record.model_dump(mode="json", by_alias=True),
+        "revision": record.revision,
+    }