diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 950b97e..3de9b48 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -1,5 +1,6 @@ import UnoCSS from 'unocss/vite' +import { transformerNotationWordHighlight } from '@shikijs/transformers' import { extendConfig } from '@voidzero-dev/vitepress-theme/config' import { defineConfig } from 'vitepress' @@ -87,6 +88,9 @@ export default extendConfig(defineConfig({ }, }, markdown: { + codeTransformers: [ + transformerNotationWordHighlight(), + ], theme: { dark: 'github-dark', light: 'github-light', @@ -156,11 +160,26 @@ export default extendConfig(defineConfig({ ], '/en/guide/': [ { + collapsed: false, items: [ - { link: '/en/guide/', text: 'Introduction' }, + { link: '/en/guide/', text: 'Guide Overview' }, + { link: '/en/guide/why', text: 'Why Vieval' }, { link: '/en/guide/getting-started', text: 'Getting Started' }, + { link: '/en/guide/core-concepts', text: 'Core Concepts' }, + ], + text: 'Introduction', + }, + { + collapsed: false, + items: [ + { link: '/en/guide/learn/tasks-cases-and-inputs', text: 'Tasks, Cases, and Inputs' }, + { link: '/en/guide/learn/assertions-scores-and-metrics', text: 'Assertions, Scores, and Metrics' }, + { link: '/en/guide/learn/models-and-inference-executors', text: 'Models and Inference Executors' }, + { link: '/en/guide/learn/matrices-and-datasets', text: 'Matrices and Datasets' }, + { link: '/en/guide/learn/reliable-execution', text: 'Reliable Execution' }, + { link: '/en/guide/learn/reports-and-comparisons', text: 'Reports and Comparisons' }, ], - text: 'Guide', + text: 'Learn', }, ], '/zh-hans/api/': [ @@ -181,11 +200,26 @@ export default extendConfig(defineConfig({ ], '/zh-hans/guide/': [ { + collapsed: false, items: [ - { link: '/zh-hans/guide/', text: '介绍' }, + { link: '/zh-hans/guide/', text: '指南概览' }, + { link: '/zh-hans/guide/why', text: '为什么选择 Vieval' }, { link: '/zh-hans/guide/getting-started', text: '快速开始' }, + { link: '/zh-hans/guide/core-concepts', text: '核心概念' }, + ], + text: '介绍', + }, + { + collapsed: false, + items: [ + { link: '/zh-hans/guide/learn/tasks-cases-and-inputs', text: '任务、用例与输入' }, + { link: '/zh-hans/guide/learn/assertions-scores-and-metrics', text: '断言、分数与指标' }, + { link: '/zh-hans/guide/learn/models-and-inference-executors', text: '模型与推理执行器' }, + { link: '/zh-hans/guide/learn/matrices-and-datasets', text: '矩阵与数据集' }, + { link: '/zh-hans/guide/learn/reliable-execution', text: '可靠执行' }, + { link: '/zh-hans/guide/learn/reports-and-comparisons', text: '报告与比较' }, ], - text: '指南', + text: '学习', }, ], }, diff --git a/docs/.vitepress/theme/styles.css b/docs/.vitepress/theme/styles.css index 67e3f4a..b103779 100644 --- a/docs/.vitepress/theme/styles.css +++ b/docs/.vitepress/theme/styles.css @@ -2,6 +2,13 @@ @source "./**/*.vue"; +.vp-doc [class*='language-'] code .highlighted-word { + padding: 1px 2px; + margin: -1px -2px; + background-color: var(--vp-code-line-highlight-color); + border-radius: 4px; +} + /* Brand color: black in light mode, white in dark mode */ /* :root, :root[data-variant="voidzero"] { diff --git a/docs/content/en/api/index.md b/docs/content/en/api/index.md index 55d34e4..124e8f7 100644 --- a/docs/content/en/api/index.md +++ b/docs/content/en/api/index.md @@ -1,14 +1,44 @@ # API -This section will document Vieval package exports and runtime entrypoints. - -Start with the public package entrypoints: - -- `vieval` -- `vieval/config` -- `vieval/core/runner` -- `vieval/core/assertions` -- `vieval/core/inference-executors` -- `vieval/plugins/chat-models` -- `vieval/expect` -- `vieval/testing/expect-extensions` +Vieval publishes focused entrypoints so application code can import the narrowest supported surface for its job. The package's `.` export is imported as `vieval`; every other current export is listed below by its published import path. + +## Common authoring + +| Import | Import when | +| --- | --- | +| `vieval` | Author eval files or config with the task DSL, config helpers, and the preconfigured assertion API; representative imports are `describeTask`, `defineConfig`, and `expect`. | +| `vieval/core/assertions` | Build and evaluate structured assertion pipelines and convert their outcomes into runner scores with exports such as `evaluateAssertions`, `expectRubric`, and `toRunScores`. | +| `vieval/expect` | Import the standalone `expect` with Vieval matchers pre-installed without importing the rest of the main authoring surface. | + +Start with [Tasks, Cases, and Inputs](/en/guide/learn/tasks-cases-and-inputs) and [Assertions, Scores, and Metrics](/en/guide/learn/assertions-scores-and-metrics). The task DSL and the structured core assertion pipeline are both public, but outcomes from `vieval/core/assertions` are not connected to task cases automatically. + +## Config and integration + +| Import | Import when | +| --- | --- | +| `vieval/config` | Define typed eval or task objects and work with config-owned contracts through exports such as `defineEval`, `defineTask`, and `resolveModelByName`; the CLI-facing `defineConfig` and `loadEnv` helpers remain on `vieval`. | +| `vieval/plugins/chat-models` | Register chat models and resolve matrix selections with exports such as `chatModelFrom`, `ChatModels`, and `modelFromRun`. | + +Use the [Config map](/en/config/) for file ownership and scope, and [Models and Inference Executors](/en/guide/learn/models-and-inference-executors) for the boundary between registration, selection, and an actual model call. + +## Advanced execution and processing + +These are public integration surfaces for hosts that need to assemble or drive execution directly. Most eval authors do not need them. + +| Import | Import when | +| --- | --- | +| `vieval/cli` | Embed the import-safe top-level CLI parser and command dispatcher through `parseTopLevelCliArguments` and `runTopLevelCli`. | +| `vieval/core/runner` | Assemble collection, scheduling, and execution directly with exports such as `collectEvalEntries`, `createRunnerSchedule`, and `runScheduledTasks`. | +| `vieval/core/scheduler` | Construct scoped concurrency coordination without the runner's collection APIs by using `createSchedulerRuntime` and `getActiveScopes`. | +| `vieval/core/processors/results` | Apply built-in result policies and inspect the gate through `processRunResults` and the `ResultGateDecision` type. | +| `vieval/core/inference-executors` | Build custom inference integrations with exports such as `createProviderAdapter`, `createRetryPolicy`, and `createOpenAIFromEnv`. | + +Read [Reliable Execution](/en/guide/learn/reliable-execution) before composing runner, scheduler, retry, or cache behavior. For aggregation, gating, and persisted evidence, continue with [Reports and Comparisons](/en/guide/learn/reports-and-comparisons). + +## Testing support + +| Import | Import when | +| --- | --- | +| `vieval/testing/expect-extensions` | Explicitly install Vieval's custom matchers or consume their supporting types through `installVievalExpectMatchers`, `KeywordMatcherOptions`, and `ToolCallContainer`. | + +`expect` from the main `vieval` entrypoint and `expect` from `vieval/expect` both arrive with Vieval's matchers installed; the main entrypoint is convenient beside the task DSL, while the subpath is the focused assertion import. `vieval/testing/expect-extensions` exports the installer rather than an `expect` value, so call `installVievalExpectMatchers()` only when explicit installation is the integration you need. diff --git a/docs/content/en/config/index.md b/docs/content/en/config/index.md index ab5a2ac..b811cdb 100644 --- a/docs/content/en/config/index.md +++ b/docs/content/en/config/index.md @@ -1,5 +1,28 @@ # Config -Vieval configuration lives close to the project that owns the evaluations. +Vieval configuration belongs to the repository or package that owns the evaluations. `vieval run` starts at the command's working directory and uses the nearest readable `vieval.config.ts`, `.mts`, `.cts`, `.js`, `.mjs`, `.cjs`, or `.json`, searching upward through parent directories. Pass `--config ` to select a file explicitly; a relative path is resolved from the command's working directory. -This page will document config loading, model provider settings, runner behavior, reporters, and output options. +The config file's directory becomes the base for relative project roots. Each project's `include` and `exclude` patterns are then evaluated within that project root. This keeps discovery ownership explicit when one config describes more than one project. + +## Configuration domains + +| Domain | What belongs here | Learn more | +| --- | --- | --- | +| Loading and environment | Config discovery or `--config` selection, `defineConfig`, and top-level `env`. Vieval does not load `.env*` files automatically; when those values are needed, call `loadEnv` explicitly from the config file. | [Models and Inference Executors](/en/guide/learn/models-and-inference-executors) | +| Projects and modes | Project names, roots, discovery patterns, executors, and project-scoped settings. Top-level `projects`, `workspaces`, and `comparisons` are mutually exclusive: `run` accepts project mode and currently maps each workspace `{ id, root }` to a project boundary, while `compare` owns comparison mode. Workspace mode does not add another nested orchestration layer. | [Core Concepts](/en/guide/core-concepts) | +| Plugins and models | Top-level or project-local config plugins, model registrations, provider metadata, aliases, and project model overrides. A registration makes a model available to task code; it does not make an inference request by itself. | [Models and Inference Executors](/en/guide/learn/models-and-inference-executors) | +| Run and eval matrices | Project `runMatrix` and `evalMatrix` definitions, plus how eval- and task-level layers extend, override, or disable axes. Matrix expansion schedules variants; task code decides what each selected value does. | [Matrices and Datasets](/en/guide/learn/matrices-and-datasets) | +| Execution policy and concurrency | Project scheduling caps and the task/case attempt, retry, timeout, and concurrency controls that bound execution. The `vieval run` concurrency flags are runtime overrides or caps at their documented scopes; `--attempt` labels report artifacts and does not configure automatic attempts. | [Reliable Execution](/en/guide/learn/reliable-execution) | +| Cache | The task cache is injected through `TaskRunContext`, not declared as a general top-level cache block. CLI runs place project cache data under `.vieval/cache` inside the project root; comparison mode can select a shared benchmark cache namespace. | [Reliable Execution](/en/guide/learn/reliable-execution) | +| Reporters and artifacts | Top-level reporters are inherited by projects unless a project supplies its own list. Local report artifacts are enabled at run time with `--report-out`; `--json` changes standard output and does not create artifacts. | [Reports and Comparisons](/en/guide/learn/reports-and-comparisons) | +| Telemetry | `reporting.openTelemetry` controls runtime OpenTelemetry spans and an end-of-run lifecycle callback. Local reporter events and the OTLP-shaped files produced by `--report-out` do not depend on that switch. | [Reports and Comparisons](/en/guide/learn/reports-and-comparisons) | + +## Scope and precedence + +Configuration only inherits where the owning types and runtime define inheritance. Top-level `concurrency.workspace` controls workspace scheduling only. For `project`, `task`, `attempt`, and `case`, each top-level concurrency value becomes the project default; a project can override each field independently, and any field it omits continues to inherit the top-level value. Models and reporters behave differently: their top-level arrays feed project defaults, but a project-provided array replaces the complete inherited list. Matrix layers have their own project → eval → task resolution rules. Plugins may transform top-level config, while project-local plugins run against a project-scoped config view. These rules do not imply that every field is available at every scope. + +The run command can select projects with `--project`, select a config with `--config`, cap or override the documented concurrency scopes, label report organization with `--workspace`, `--experiment`, and `--attempt`, and choose output behavior with `--json` and `--report-out`. Treat these as command-specific runtime inputs rather than a generic override mechanism for arbitrary config fields. + +Each documented field records **Type → Default → Scope → CLI override → Behavior → Example → Interactions**. When a step does not apply to that field, the reference states that explicitly instead of inferring a value or an override. + +For the exported config types and helpers, see the [API map](/en/api/). diff --git a/docs/content/en/guide/core-concepts.md b/docs/content/en/guide/core-concepts.md new file mode 100644 index 0000000..afe79f1 --- /dev/null +++ b/docs/content/en/guide/core-concepts.md @@ -0,0 +1,42 @@ +--- +title: Core Concepts +prev: + text: Getting Started + link: /en/guide/getting-started +next: + text: Tasks, Cases, and Inputs + link: /en/guide/learn/tasks-cases-and-inputs +--- + +# Core Concepts + +Vieval organizes evaluation work into a small set of nested concepts. The following lifecycle is the map for the rest of the guide: + +```text +project + -> task <-> run matrix + eval matrix + -> case -> assertion / score / metric evidence + +task -> attempt -> retry + +run --json -> machine-readable stdout +run --report-out -> report artifacts -> analyze / compare +``` + +A **project** defines a discovery boundary and project-level configuration. A discovered eval definition may carry a **task**, while DSL definitions such as `describeTask` register tasks directly. Matrix rows expand the registered tasks into concrete scheduled work. A task contains one or more **cases**, where assertions, normalized scores, and custom metrics provide execution evidence. + +Execution reliability has two scopes. An **attempt** reruns a full task and retains the evidence from every attempt. A **retry** lets a failed case run again within an attempt. Output is explicit: `--json` prints a machine-readable result to standard output, while `--report-out ` writes the report artifacts consumed by analysis and comparison commands. + +| Concept | Responsibility | Typical scope | +| --- | --- | --- | +| Project | Defines discovery, configuration, executors, and matrix defaults | One product area or evaluation suite | +| Task | Describes evaluation behavior and connects run/eval matrix selections | One behavior evaluated across variants | +| Case | Executes one input or scenario and emits assertion, score, and metric evidence | One check within a task | +| Run matrix | Expands execution-side choices such as models or scenarios | Project, eval, or task definition | +| Eval matrix | Expands evaluation-side choices such as rubrics | Project, eval, or task definition | +| Attempt | Repeats the complete task to measure reliability | One scheduled task | +| Retry | Re-executes a failed case inside an attempt | One case execution | +| Report artifacts | Preserve run summaries, events, and case evidence when `--report-out` is used | One run and its configured report directory | +| Analysis and comparison | Read artifacts to inspect results or compare runs and methods | One or more report artifact sets | + +Next, learn how to author [Tasks, Cases, and Inputs](/en/guide/learn/tasks-cases-and-inputs). diff --git a/docs/content/en/guide/getting-started.md b/docs/content/en/guide/getting-started.md index 58129cd..92adf66 100644 --- a/docs/content/en/guide/getting-started.md +++ b/docs/content/en/guide/getting-started.md @@ -1,15 +1,121 @@ +--- +title: Getting Started +prev: + text: Why Vieval + link: /en/guide/why +next: + text: Core Concepts + link: /en/guide/core-concepts +--- + # Getting Started -Install the package: +This first evaluation is deterministic and runs entirely in your local Node.js process. + +## Install Vieval + +::: code-group -```sh +```sh [pnpm] pnpm add -D vieval ``` -Create evaluation tasks with the Vieval DSL, configure providers, then run the CLI from your project. +```sh [npm] +npm install --save-dev vieval +``` + +::: + +## Configure a project + +Create a config at the root of your project: + +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' + +export default defineConfig({ + projects: [ + { + include: ['evals/*.eval.ts'], + inferenceExecutors: [ + { + id: 'local', + }, + ], + models: [ + { + aliases: [], + id: 'local:deterministic', + inferenceExecutor: 'local', + inferenceExecutorId: 'local', + model: 'deterministic', + }, + ], + name: 'getting-started', + root: '.', + }, + ], +}) +``` + +The project tells Vieval where to discover evaluation files. The `root` is resolved as the project boundary, and `include` selects matching files beneath it. The model entry is local registry metadata used to enable automatic execution of discovered DSL tasks; this evaluation never calls it as a provider. + +## Write an evaluation + +Create the matching evaluation file: + +```ts [evals/arithmetic.eval.ts] +import { caseOf, describeTask, expect } from 'vieval' + +describeTask('arithmetic', () => { + caseOf('adds two numbers', () => { + expect(20 + 22).toBe(42) + }) +}) +``` + +`describeTask` registers the task, `caseOf` registers one case inside it, and `expect` executes an assertion. If its matcher fails, it throws and the case fails. + +## Run it + +::: code-group + +```sh [pnpm] +pnpm vieval run --config ./vieval.config.ts +``` + +```sh [npm] +npx vieval run --config ./vieval.config.ts +``` + +::: + +The stable result is one discovered project, one scheduled task, and one passed case. The terminal presentation may change, so this guide does not depend on exact output text or timing. + +::: info No credentials required +Automatic DSL execution currently requires a registered model target. The `local:deterministic` entry only satisfies that execution gate: this task makes no provider call, uses no credentials, and incurs no model cost. +::: + +## Optional: How to read later examples + +Guide examples use an error highlight to identify the line that failed: + +```ts [evals/arithmetic.eval.ts] +expect(20 + 22).toBe(41) // [!code error] +``` + +Removed and added line highlights then show the correction without hiding the original mistake: + +```ts [evals/arithmetic.eval.ts] +expect(20 + 22).toBe(41) // [!code --] +expect(20 + 22).toBe(42) // [!code ++] +``` + +When the edit itself is the subject, the same change may use a diff fence: -```sh -pnpm vieval run +```diff [evals/arithmetic.eval.ts] +- expect(20 + 22).toBe(41) ++ expect(20 + 22).toBe(42) ``` -The detailed setup guide will be filled in as the public API and examples settle. +You now have two project files: `vieval.config.ts` defines discovery, while `evals/arithmetic.eval.ts` defines executable evaluation behavior. Continue to [Core Concepts](/en/guide/core-concepts) to see how projects, tasks, cases, attempts, and reports fit together. diff --git a/docs/content/en/guide/index.md b/docs/content/en/guide/index.md index 46eb888..93c200a 100644 --- a/docs/content/en/guide/index.md +++ b/docs/content/en/guide/index.md @@ -1,12 +1,20 @@ -# Introduction +# Vieval Guide -Vieval is a Vitest-based evaluation framework for agents, models, and model-backed workflows. +Vieval helps you define repeatable evaluations next to product code, expand them across models and scenarios, and retain both human-readable output and machine-readable evidence. -This documentation site is intentionally minimal for now. It gives the project a VitePress structure, navigation, and theme setup so the guide can grow without changing the docs foundation again. +## Start here -## What belongs here +- [Why Vieval](/en/guide/why) explains the problems Vieval handles and where its boundaries are. +- [Getting Started](/en/guide/getting-started) runs a deterministic evaluation with no provider call, credentials, or model cost. +- [Core Concepts](/en/guide/core-concepts) introduces the objects and lifecycle used throughout the guide. -- Concepts and terminology for evaluation flows. -- Step-by-step usage guides. -- Examples for tasks, assertions, model executors, and reporters. -- Links to reference pages generated or maintained from package APIs. +## Learn by following an evaluation + +1. [Tasks, Cases, and Inputs](/en/guide/learn/tasks-cases-and-inputs) +2. [Assertions, Scores, and Metrics](/en/guide/learn/assertions-scores-and-metrics) +3. [Models and Inference Executors](/en/guide/learn/models-and-inference-executors) +4. [Matrices and Datasets](/en/guide/learn/matrices-and-datasets) +5. [Reliable Execution](/en/guide/learn/reliable-execution) +6. [Reports and Comparisons](/en/guide/learn/reports-and-comparisons) + +For option details and exported interfaces, continue to the [Config reference](/en/config/) and [API reference](/en/api/). diff --git a/docs/content/en/guide/learn/assertions-scores-and-metrics.md b/docs/content/en/guide/learn/assertions-scores-and-metrics.md new file mode 100644 index 0000000..81a4e31 --- /dev/null +++ b/docs/content/en/guide/learn/assertions-scores-and-metrics.md @@ -0,0 +1,103 @@ +--- +title: Assertions, Scores, and Metrics +prev: + text: Tasks, Cases, and Inputs + link: /en/guide/learn/tasks-cases-and-inputs +next: + text: Models and Inference Executors + link: /en/guide/learn/models-and-inference-executors +--- + +# Assertions, Scores, and Metrics + +A case becomes useful evidence when it makes success observable. Start with deterministic assertions, then add normalized scores or report metadata only when the evaluation needs them. + +## Start with a deterministic assertion + +Vieval exports a Vitest-compatible `expect`. A failed matcher throws inside the case, so the case fails; a case that completes without a custom score contributes an `exact` score of `1`, while a failed case contributes `0`. + +```ts [evals/normalized-answer.eval.ts] +import { describeTask, expect } from 'vieval' + +describeTask('normalized answer', ({ caseOf }) => { + caseOf('removes surrounding whitespace', () => { + const answer = ' forty-two '.trim() + expect(answer).toBe('forty-two') + }) +}) +``` + +This is the clearest starting point for exact values, schemas, required fields, and other rules that code can decide reliably. + +## Separate the evidence types + +| Term | Meaning in Vieval | Effect on a task case | +| --- | --- | --- | +| Assertion | A check that produces a pass/fail decision. `expect` controls the case by throwing on failure; `vieval/core/assertions` also exposes assertion functions that return structured outcomes. | A thrown matcher fails the case. Core assertion outcomes are not connected to the task DSL automatically. | +| Score | A normalized number in `0..1`, recorded with `context.score(value, kind)`. Public score kinds are `exact` and `judge`; `exact` is the default. | Contributes to task-result aggregation when the case passes. Its emitted event may also be persisted in a report case record. | +| Metric | Named benchmark metadata recorded with `context.metric(name, value)`. Values may be strings, numbers, booleans, `null`, or arrays of those telemetry values. | Appears in report events and case records, but does not change pass/fail state or score aggregation. | +| Rubric | An assertion whose `judge` callback returns a reason and normalized score, then compares that score with `minScore` (default `0.7`). | Produces an `AssertionOutcome` with score kind `judge`; the author decides how to bridge that outcome into a task case. | + +The `score` and `metric` methods are available on the case callback context received by `caseOf` and `casesFromInputs`. + +## Record scores and metrics from a case + +Use `score` for a value that should be averaged. Use `metric` for dimensions or observations that reports should retain without treating them as the result itself. + +```ts [evals/retrieval.eval.ts] +import { describeTask, expect } from 'vieval' + +describeTask('retrieval', ({ caseOf }) => { + caseOf('finds the expected documents', ({ metric, score }) => { + const expected = new Set(['doc-a', 'doc-b']) + const retrieved = ['doc-a', 'doc-c'] + const matches = retrieved.filter(id => expected.has(id)).length + const recall = matches / expected.size + + score(recall, 'exact') + metric('benchmark.case.id', 'retrieval-basic') + metric('retrieved.count', retrieved.length) + + expect(recall).toBeGreaterThan(0) + }) +}) +``` + +Scores must be finite and inside `0..1`. Within one case, another `score` call with the same kind replaces the earlier value; use one final value per score family. + +Task-result aggregation and persisted report records have different timing. If a case later fails or times out, task-result aggregation ignores its custom score contributions and adds exact failure evidence. Each `score` call also emits an event immediately, however, so a score emitted before the failure may remain in the persisted case record. The case-end event supplies an exact `0` only when that record does not already have an exact score. When diagnosing a report, read the case `state` together with its recorded scores. + +Metrics are evidence for filtering, grouping, and diagnosing reports. A metric alone never makes a case pass, fail, or receive a higher aggregate score. + +## Use rubric assertions deliberately + +The public `vieval/core/assertions` entrypoint includes `expectRubric` and `evaluateAssertions`. `expectRubric` creates an assertion. When evaluated, for example by `evaluateAssertions`, that assertion calls the `judge` function you provide, clamps its returned score to `0..1`, applies `minScore`, and produces a structured `AssertionOutcome`: + +```ts [rubric.ts] +import { evaluateAssertions, expectRubric } from 'vieval/core/assertions' + +const [outcome] = await evaluateAssertions([ + expectRubric({ + id: 'concise-answer', + judge: async ({ text }) => ({ + reason: text.length <= 80 ? 'Answer is concise.' : 'Answer is too long.', + score: text.length <= 80 ? 1 : 0, + }), + minScore: 0.8, + }), +], { + text: 'A short answer.', +}) +``` + +This example is deterministic and makes no model call. `expectRubric` does not select a model or invoke a provider by itself. A `judge` callback may use local logic, a human result, or model inference. + +The core assertion pipeline and the task DSL are currently separate public surfaces. `evaluateAssertions` returns outcomes; it does not automatically call the case callback context's `score`, `metric`, or `expect`. If you bridge an outcome manually, decide explicitly whether you are recording its numeric score, failing the case from `outcome.pass`, retaining its reason as a metric, or some combination. Throwing to fail a case makes the task result treat it as an exact failure; score events emitted before the throw may still appear in the persisted case record. + +::: warning Model-backed judging has runtime consequences +If your `judge` callback invokes a model, it needs the relevant provider configuration and credentials and adds latency and cost. Case concurrency can fan out judge requests, so apply limits that match provider capacity. Review the provider's data policy and choose deliberately which case inputs, outputs, judge prompts, reasons, metrics, or events you retain in report artifacts. +::: + +A common boundary error is treating a metric as a score or assuming a rubric outcome is wired into the case automatically. Keep the decision, numeric evidence, and diagnostic metadata separate in the code. + +Next, learn how model registrations and runtime adapters fit this flow in [Models and Inference Executors](/en/guide/learn/models-and-inference-executors). The assertion primitives are listed under the public [`vieval/core/assertions` entrypoint](/en/api/). diff --git a/docs/content/en/guide/learn/matrices-and-datasets.md b/docs/content/en/guide/learn/matrices-and-datasets.md new file mode 100644 index 0000000..0aab051 --- /dev/null +++ b/docs/content/en/guide/learn/matrices-and-datasets.md @@ -0,0 +1,192 @@ +--- +title: Matrices and Datasets +prev: + text: Models and Inference Executors + link: /en/guide/learn/models-and-inference-executors +next: + text: Reliable Execution + link: /en/guide/learn/reliable-execution +--- + +# Matrices and Datasets + +Matrices repeat an eval task under selected configuration values. Input arrays repeat its cases. Keeping those two kinds of expansion separate makes the size and meaning of a run easier to predict. + +## Start with one model axis + +Add a `model` axis to the project `runMatrix`. With one registered alias, it produces one run row: + +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' + +export default defineConfig({ + projects: [ + { + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + runMatrix: { + extend: { + model: ['assistant-default'], + }, + }, + }, + ], +}) +``` + +Now add a scenario axis. Axis values form a cartesian product, so one model and two scenarios produce two run rows: + +```ts [vieval.config.ts] +export default defineConfig({ + projects: [ + { + name: 'chat-evals', + runMatrix: { + extend: { + model: ['assistant-default'], + scenario: ['baseline', 'stress'], // [!code ++] + }, + }, + }, + ], +}) +``` + +Task code receives one selected value per axis under `context.task.matrix.run`. Axis names have no built-in behavior: adding `scenario` schedules rows, but your task must read the value and change behavior if the scenarios should differ. + +## Separate run and eval matrices + +Use `runMatrix` for variants of the system under evaluation, such as its model, prompt language, or scenario. Use `evalMatrix` for variants of evaluation behavior, such as a rubric name or judge-model selector. + +```ts [vieval.config.ts] +export default defineConfig({ + projects: [ + { + evalMatrix: { + extend: { + rubric: ['strict', 'lenient'], + }, + }, + name: 'chat-evals', + runMatrix: { + extend: { + model: ['assistant-default'], + scenario: ['baseline', 'stress'], + }, + }, + }, + ], +}) +``` + +This definition has two run rows and two eval rows, yielding four row pairs for each discovered entry and scheduler inference executor. The selections appear separately as `context.task.matrix.run` and `context.task.matrix.eval`. + +The distinction is organizational, not an automatic call pipeline. A `model` axis does not invoke that model, and a `rubric` or judge-model axis does not run a judge. Task or assertion code must read the selected values and implement those actions. + +## Layer project, eval, and task matrices + +Matrix layers resolve from outer to inner: + +1. Project `runMatrix` and `evalMatrix` from `vieval.config.*`. +2. Eval-local `matrix` from `defineEval`. +3. Task-local `matrix` from `defineTask`. + +Within every layer, Vieval applies controls in this order: + +1. `disable` removes the named axes inherited so far. Its value is an array of axis names. +2. `extend` adds new axes and appends deduplicated values to inherited axes. +3. `override` replaces an axis with exactly the values at that layer. + +```ts [evals/layered.eval.ts] +import { defineEval, defineTask } from 'vieval/config' + +export default defineEval({ + description: 'Shows matrix layering.', + matrix: { + runMatrix: { + disable: ['scenario'], + extend: { + promptLanguage: ['en', 'zh'], + }, + override: { + model: ['assistant-default'], + }, + }, + }, + name: 'layered', + task: defineTask({ + id: 'layered', + matrix: { + evalMatrix: { + override: { + rubric: ['strict'], + }, + }, + }, + run(context) { + return { + scores: [{ + kind: 'exact', + score: context.task.matrix.eval.rubric === 'strict' ? 1 : 0, + }], + } + }, + }), +}) +``` + +Against the previous project config, the eval layer removes `scenario`, adds two `promptLanguage` values, and fixes the model axis. The task layer replaces the two inherited rubric values with `strict`. A flat matrix object is still accepted and normalized as `extend`, but the layered form makes inheritance explicit. Do not mix control keys such as `disable` with axis keys in the same object. + +## Add input cases without changing matrix rows + +`casesFromInputs` accepts an array that your code already loaded or constructed. It registers one case per item and exposes that item as `matrix.inputs` inside the callback: + +```ts [evals/dataset.eval.ts] +import { describeTask, expect } from 'vieval' + +const inputs = [ + { expected: 4, left: 2, right: 2 }, + { expected: 7, left: 3, right: 4 }, + { expected: 12, left: 5, right: 7 }, +] + +describeTask('arithmetic dataset', ({ casesFromInputs }) => { + casesFromInputs('addition', inputs, ({ matrix }) => { + const result = matrix.inputs.left + matrix.inputs.right + expect(result).toBe(matrix.inputs.expected) + }) +}) +``` + +This API does not discover or load a generic external dataset. If rows live in JSON, a database, or another source, load and validate them in your own code, then pass the resulting array to `casesFromInputs`. + +The runtime shape is: + +```text +Project + -> discovered Eval entry and its Task + -> inference executor × run row × eval row + -> one scheduled task execution + -> explicit caseOf cases + one case per casesFromInputs item +``` + +Inputs therefore do not create more scheduler rows. They create cases inside every scheduled task whose task definition registers them. + +## Calculate the expansion before running + +Before attempts and retries, scheduler cardinality is: + +```text +discovered entries × project inference executors × run rows × eval rows +``` + +If a task registers only one `casesFromInputs` group of `N` items, its baseline case callback count before retries or additional attempts is that task's scheduled executions multiplied by `N`. Other `caseOf` calls or input groups add their own cases. + +For example, one entry, the default single scheduler inference executor, two run rows, two eval rows, and three inputs produce four scheduled task executions and twelve case callback executions. + +::: warning Watch for combinatorial growth and rate limits +Adding values on any axis multiplies scheduled executions; adding inputs multiplies case work within them. This increases runtime and report records. It increases provider calls, cost, and rate-limit exposure only when the repeated task or case code actually invokes a provider, and one case may make more than one call. Estimate that call count separately and set concurrency to match provider capacity. +::: + +Next, control retries, attempts, timeouts, and concurrency in [Reliable Execution](/en/guide/learn/reliable-execution). The [Config reference](/en/config/) and [API reference](/en/api/) list the underlying matrix and task types. diff --git a/docs/content/en/guide/learn/models-and-inference-executors.md b/docs/content/en/guide/learn/models-and-inference-executors.md new file mode 100644 index 0000000..eadbbdf --- /dev/null +++ b/docs/content/en/guide/learn/models-and-inference-executors.md @@ -0,0 +1,116 @@ +--- +title: Models and Inference Executors +prev: + text: Assertions, Scores, and Metrics + link: /en/guide/learn/assertions-scores-and-metrics +next: + text: Matrices and Datasets + link: /en/guide/learn/matrices-and-datasets +--- + +# Models and Inference Executors + +A model registration gives eval code a stable name for runtime configuration. It does not send a request. This lesson follows that boundary from a model alias to the code that can make a remote or local inference call. + +## Register a provider-backed model + +The built-in chat-model plugin turns each `chatModelFrom` result into a configured `ModelDefinition`. The following registration reads `.env` files through `loadEnv` and requires `OPENAI_API_KEY` while the plugin resolves the model: + +```ts [vieval.config.ts] +import { cwd } from 'node:process' + +import { defineConfig, loadEnv, requiredEnvFrom } from 'vieval' // [!code ++] +import { chatModelFrom, ChatModels } from 'vieval/plugins/chat-models' // [!code ++] + +export default defineConfig({ + env: loadEnv('test', cwd(), ''), // [!code ++] + plugins: [ + ChatModels({ // [!code ++] + models: [ + chatModelFrom({ + aliases: ['assistant-default'], + apiKey: config => requiredEnvFrom(config.env, { // [!code ++] + name: 'OPENAI_API_KEY', // [!code ++] + type: 'string', // [!code ++] + }), // [!code ++] + inferenceExecutor: 'openai', + model: 'gpt-4.1-mini', + }), + ], + }), + ], + projects: [ + { + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + runMatrix: { + extend: { + model: ['assistant-default'], + }, + }, + }, + ], +}) +``` + +`chatModelFrom` derives the model id `openai:gpt-4.1-mini` unless you provide `id`. It preserves the concrete provider model name and adds `assistant-default` as an alias. `ChatModels` then appends the resolved definition to the config's `models` collection. + +::: warning Credentials +Keep `OPENAI_API_KEY` outside source control. `requiredEnvFrom` rejects a missing value; it does not obtain, rotate, or redact the credential for you. Be careful not to emit the resolved model parameters into metrics, logs, or report events. +::: + +::: tip Prefer stable aliases in matrix axes +`modelFromRun` can match a registered model by id, concrete model name, or alias. A role-oriented alias such as `assistant-default` lets eval selection stay stable when the concrete model changes. +::: + +## Keep the two executor concepts separate + +Several similarly named fields serve different boundaries: + +| Value | Responsibility | +| --- | --- | +| `model.inferenceExecutor` | Executor metadata owned by the model plugin. For the built-in chat model it may be `'openai'`, `'openrouter'`, `'ollama'`, or a compatible executor object. Core config treats it as opaque. | +| `model.inferenceExecutorId` | Runtime-config discriminator for that model definition. `chatModelFrom` derives it from a string executor; runtime config helpers use it to select the OpenAI, OpenRouter, or Ollama shape. Report results instead take their `inferenceExecutorId` from the scheduled `context.task.inferenceExecutor.id`. | +| `projects[].inferenceExecutors` | An independent list of scheduler targets. Every discovered entry is expanded once per item; its default is `[{ id: 'default' }]`. It is not derived from `models`. | +| `context.task.inferenceExecutor` | The scheduler target selected from `projects[].inferenceExecutors` for this scheduled task. It need not equal the executor metadata of a matrix-selected model. | + +The model path is therefore: + +```text +matrix alias + -> registered ModelDefinition + -> model.inferenceExecutorId selects a runtime-config shape + -> eval or agent code creates and calls the remote/local runtime +``` + +Scheduler fan-out is a separate path through `projects[].inferenceExecutors`. Give the two ids the same value only when your own executor design requires that relationship; Vieval does not join them automatically. + +## Resolve the selected model in eval code + +`modelFromRun` reads the named axis from `context.task.matrix.run`, then resolves its selected value against `context.models`. `modelFromEval` does the same under `context.task.matrix.eval`, which is useful when an eval axis selects a judge or evaluator model. + +```ts [evals/model-selection.eval.ts] +import { describeTask, expect } from 'vieval' +import { modelFromRun, openaiFromRunContext } from 'vieval/plugins/chat-models' + +describeTask('model selection', ({ caseOf }) => { + caseOf('resolves the run model', (context) => { + const selectedModel = modelFromRun(context, { axis: 'model' }) + const runtimeConfig = openaiFromRunContext(selectedModel) + + expect(selectedModel.aliases).toContain('assistant-default') + expect(runtimeConfig.model).toBe('gpt-4.1-mini') + }) +}) +``` + +The axis name is explicit: `{ axis: 'model' }` means “read `context.task.matrix.run.model`.” For an eval-matrix axis such as `judgeModel`, use `modelFromEval(context, { axis: 'judgeModel' })`. + +`openaiFromRunContext` validates and returns provider options such as `apiKey` and `model`. It still does not create a client or call OpenAI. The task or imported agent must pass that configuration to its chosen runtime and invoke it. Likewise, `ChatModels` registers and resolves metadata; it is not a provider client. + +::: warning Cost and data +Registration and resolution are local configuration work. Cost, latency, rate-limit use, and data transfer begin only when your task, agent, or judge invokes a remote provider. Review which prompts, inputs, outputs, telemetry, and report artifacts may contain sensitive data before enabling such calls. Only a fully local deployment without an external proxy or telemetry avoids sending inference input to a remote model provider; local compute, storage, and other networking policies still apply. +::: + +Next, use the alias across controlled combinations in [Matrices and Datasets](/en/guide/learn/matrices-and-datasets). See the [Config reference](/en/config/) for configuration structure and the [API reference](/en/api/) for exported helpers. diff --git a/docs/content/en/guide/learn/reliable-execution.md b/docs/content/en/guide/learn/reliable-execution.md new file mode 100644 index 0000000..b4d6be0 --- /dev/null +++ b/docs/content/en/guide/learn/reliable-execution.md @@ -0,0 +1,165 @@ +--- +title: Reliable Execution +prev: + text: Matrices and Datasets + link: /en/guide/learn/matrices-and-datasets +next: + text: Reports and Comparisons + link: /en/guide/learn/reports-and-comparisons +--- + +# Reliable Execution + +Reliable evaluation requires two different controls: repeat enough work to measure nondeterminism, and bound enough work to protect the service being evaluated. Vieval exposes attempts, retries, timeouts, concurrency limits, and a task cache for those purposes. + +## Read the execution hierarchy + +A scheduled task contains registered cases. When `autoAttempt` is enabled, Vieval runs the complete case set as one attempt, waits for every case to settle, and starts another attempt only if an eligible case still failed or timed out. Each case execution may make an initial try followed by `autoRetry` retries. + +```text +scheduled task + -> attempt 0 + -> case A: initial try -> retry 1 -> retry 2 + -> case B: initial try + -> attempt 1, only if an eligible case still failed + -> case A: initial try ... + -> case B: initial try ... +``` + +Cases within an attempt may run concurrently. Attempts created by `autoAttempt` are currently sequential, and a later attempt reruns every registered case, including cases that passed previously. The next attempt begins only after the current case set has settled. + +The `--attempt` CLI flag is related but distinct: it assigns an `attemptId` to the whole run for report organization. It does not set `autoAttempt` or the number of task attempts. + +## Set concurrency at the scope that owns the work + +`vieval run` accepts five concurrency names, but the current runtime does not use all of them as parallel queues in the same way. + +| Scope | What it limits today | Where it can be set | +| --- | --- | --- | +| Workspace | Projects admitted concurrently within one CLI run. The default effective cap is `1`. | Top-level `concurrency.workspace`; `--workspace-concurrency` | +| Project | An upper bound combined with the task cap when scheduled tasks run inside a project. | Top-level or project `concurrency.project`; `--project-concurrency` | +| Task | Concurrent scheduled tasks inside a project. The default effective cap is `1`. | Top-level or project `concurrency.task`; `--task-concurrency` | +| Attempt | Attempt concurrency metadata on config, projects, and tasks. `autoAttempt` execution is sequential today, so this does not make automatic attempts overlap. | Top-level, project, or task `concurrency.attempt`; `--attempt-concurrency` | +| Case | Concurrent DSL cases. A `casesFromInputs` group can have its own queue; otherwise cases share the task queue. | Top-level, project, or task `concurrency.case`; `casesFromInputs(..., { concurrency })`; `--case-concurrency` | + +For workspace, project, and task scheduling, a CLI value acts as a cap: it cannot raise a lower configured value. For attempt and case settings, the CLI runtime value takes precedence over task and project values. Project config cannot declare `workspace`, and task config contains only `attempt` and `case`. + +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' + +export default defineConfig({ + concurrency: { + workspace: 1, + }, + projects: [ + { + concurrency: { + case: 4, + project: 2, + task: 2, + }, + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + }, + ], +}) +``` + +A task can narrow case concurrency, and one generated case group can narrow it again: + +```ts [evals/retrieval.eval.ts] +import { describeTask } from 'vieval' + +describeTask('retrieval', ({ casesFromInputs }) => { + casesFromInputs('query', ['alpha', 'beta', 'gamma'], async ({ matrix }) => { + await evaluateQuery(matrix.inputs) + }, { + concurrency: 2, + }) +}, { + concurrency: { + case: 4, + }, +}) +``` + +Here the group limit of `2` applies unless `--case-concurrency` supplies a runtime override. + +::: warning Concurrency multiplies real calls only when task code makes them +Matrices, inference-executor registrations, and concurrency settings do not call a provider by themselves. If cases do invoke a model or another metered service, higher task or case concurrency can increase rate-limit failures and create a sudden cost spike. Choose limits from provider capacity and the number of calls each case actually makes. +::: + +## Choose attempts, retries, and timeouts deliberately + +Execution policies can be placed on `describeTask`, `caseOf`, or `casesFromInputs`. A case-level value overrides the task-level value. + +```ts [evals/provider-health.eval.ts] +import { caseOf, describeTask, expect } from 'vieval' + +describeTask('provider health', () => { + caseOf('returns a usable answer', async ({ signal }) => { + const answer = await requestAnswer({ signal }) + expect(answer.length).toBeGreaterThan(0) + }, { + autoAttempt: 1, + autoRetry: 2, + input: 'health-check', + timeout: 10_000, + }) +}) +``` + +`autoRetry: 2` permits two additional case tries after the initial failure. Retries stop at the first pass and remain part of the same attempt. The default delays are exponential: 500 ms before retry 1, then 1,000 ms before retry 2. `autoRetryDelay` can instead be a fixed non-negative number or a function of the retry index. + +`autoAttempt: 1` permits one additional full task attempt. It starts only when a case still fails after its retries and that case has remaining attempts. Because the full registered case set runs again, attempt evidence includes both the earlier and later outcomes. For example, one failed attempt followed by one passing attempt contributes `0.5`, not a replacement pass of `1`. + +::: tip Use repetition for the question it answers +Attempts measure nondeterministic reliability because every completed attempt contributes evidence. Retries recover transient case failures inside one attempt; a retry that eventually passes makes that attempt pass. Retries therefore hide the earlier transient failure from task scoring, although lifecycle events still record the tries. Use retries only when recovery, rather than failure frequency, is the intended measurement. +::: + +`timeout` is measured separately for each case try. When it expires, Vieval marks that try as `timeout`, aborts the case's `signal`, and may proceed to a configured retry or later attempt. Cancellation is cooperative: pass the signal to downstream operations. Code that ignores it may continue external side effects after Vieval has stopped accepting its scores and metrics. + +## Cache deterministic setup artifacts + +Task callbacks receive a filesystem-backed cache through `context.cache`. It is an explicit file API, not automatic memoization: + +```ts [evals/dataset.eval.ts] +import { describeTask } from 'vieval' + +describeTask('dataset-backed eval', ({ caseOf }) => { + caseOf('loads prepared cases', async ({ cache, signal }) => { + const file = cache.namespace('dataset-v1').file({ + ext: 'json', + key: ['prepared', 'source-sha256'], + }) + + if (!await file.exists()) { + await file.writeJson(await prepareDataset({ signal })) + } + + const cases = await file.readJson>() + await evaluatePreparedCases(cases) + }) +}) +``` + +During a CLI run, the path is derived from the project root's `.vieval/cache`, then workspace ID, project name, namespace, key segments, and extension. The runtime sanitizes path segments and writes text, JSON, and buffers atomically. It does not hash inputs, expire entries, or decide whether cached content is still valid. Put every identity that changes the artifact into the namespace or key. + +The same cache root and stable workspace/project/namespace/key resolve to the same file across task attempts and later runs. Top-level comparison runs deliberately share the project identity segment through their configured benchmark cache namespace, but methods under different project roots still use different physical `.vieval/cache` directories. Reusing one cached file also requires the same cache root. + +## Know which evidence survives each layer + +Task aggregation and report artifacts answer related but different questions: + +- A failed or timed-out case adds exact score evidence of `0`; a passing case without a custom score adds `1`. +- A passing case contributes its custom `exact` and `judge` scores. A failed case's custom score is not used by task aggregation. +- Every completed auto attempt contributes another set of case outcomes to the task's aggregated score. +- Reporter lifecycle, score, and metric events are emitted as execution happens. With `--report-out`, `events.jsonl` retains those events. +- `cases.jsonl` is a final normalized projection keyed by task and case, so repeated retries or automatic attempts do not become independent case rows. Read raw events when the sequence matters, and read the task aggregate in `run-summary.json` when attempt-weighted scoring matters. + +As described in [Assertions, Scores, and Metrics](/en/guide/learn/assertions-scores-and-metrics), a score event emitted before a later failure can remain in a persisted case record even though task aggregation counts that outcome as a failure. Diagnose state and score together. + +A common failure is using `autoRetry` to estimate reliability: the recovered case appears as one passing attempt. Use `autoAttempt` when earlier failures should remain in the aggregate, and remember that it reruns cases that already passed. Also avoid treating `--attempt` or `--attempt-concurrency` as a request for more automatic attempts. + +Next, retain and inspect this evidence in [Reports and Comparisons](/en/guide/learn/reports-and-comparisons). diff --git a/docs/content/en/guide/learn/reports-and-comparisons.md b/docs/content/en/guide/learn/reports-and-comparisons.md new file mode 100644 index 0000000..6cd5f52 --- /dev/null +++ b/docs/content/en/guide/learn/reports-and-comparisons.md @@ -0,0 +1,141 @@ +--- +title: Reports and Comparisons +prev: + text: Reliable Execution + link: /en/guide/learn/reliable-execution +--- + +# Reports and Comparisons + +The default CLI summary answers what happened in the active terminal. Report artifacts retain the evidence needed to inspect cases, analyze several runs, and compare a baseline with a candidate later. + +## Move from terminal output to retained artifacts + +`vieval run` prints a human-readable summary by default. `--json` changes stdout to the machine-readable run output; it does not create files. Add `--report-out` when the run should also persist a report directory. + +```bash [Terminal] +vieval run \ + --config ./vieval.config.ts \ + --workspace local \ + --experiment baseline \ + --attempt attempt-a \ + --report-out .vieval/reports +``` + +The command still prints its human-readable summary of the active run and its project, task, and case state. Add `--json` when a script needs the machine-readable run output, including the resolved `reportDirectory`, on stdout. The report directory created by `--report-out` remains available after the terminal session even though the human-readable summary does not print its path. + +Without `--report-out`, `reportDirectory` is `null` and the `vieval report ...` commands have no new artifacts from that run to read. + +## Know what a run writes + +Vieval places each run under the report root by workspace, project, experiment, attempt, and generated run ID. A multi-project run uses `multi-project` for the project segment. + +| Artifact | Confirmed contents | +| --- | --- | +| `run-summary.json` | The CLI run output, including identities, project status, case summaries and failures, matrix summaries, and aggregated run scores. | +| `events.jsonl` | Ordered run, task, case, score, metric, and custom event envelopes captured during execution. | +| `cases.jsonl` | One normalized final record per observed task/case, with identities, state, timing, a derived retry count, scores, metrics, and optional input/output. | +| `metrics-summary.json` | Overall score counts, sums, and averages derived from normalized case records. | +| `otlp/traces.json`, `otlp/logs.json`, `otlp/metrics.json` | Local OTLP-shaped projections derived from the case records. | + +`vieval report index` creates another artifact, `index/runs.jsonl` by default. It is not written by `vieval run`. + +Automatic attempts require one reading caveat. `run-summary.json` contains the task score aggregated from all completed attempt evidence, while `cases.jsonl` keeps a final projection for each task/case. Use `events.jsonl` when retry or attempt sequence matters. + +The `retryCount` field is derived from lifecycle starts. When events do not provide a `retryIndex`, repeated starts from additional attempts may increase it too; use `events.jsonl` to distinguish retries from attempts precisely. + +::: warning Treat report artifacts as potentially sensitive +Artifacts may contain case inputs and outputs, error messages, custom metrics, model or benchmark identifiers, and other event payloads. Store them under an appropriate retention policy, restrict access, and remove or redact fields that should not leave the evaluation environment. +::: + +## Index, inspect, and analyze reports + +Every report command requires a report path. It can usually be a single run directory or a higher report root that contains multiple runs. + +```bash [Terminal — build a run index] +vieval report index +``` + +`index` discovers `run-summary.json` files recursively and writes compact run rows to `/index/runs.jsonl` unless `--output` changes the path. `--format table|json|jsonl` controls stdout, not the index file format. + +```bash [Terminal — inspect normalized cases] +vieval report cases +``` + +`cases` reads `cases.jsonl`. Repeat `--where key=value` for equality filters, add `--group-by ` for grouped score summaries, and choose `--format table|json|jsonl`. JSON and JSONL are stdout formats; this command does not write a new case artifact. + +```bash [Terminal — analyze runs] +vieval report analyze +``` + +`analyze` reads run summaries and events, filters runs, and rolls them up by workspace and experiment. Its filters include workspace, project, experiment, attempt, run, event or error text, and run/eval matrix selectors. `--task-state` and `--case-state` currently accept `passed`, `failed`, or `skipped`; `timeout` is not currently accepted by `--case-state`. `--format table|json|jsonl|csv` controls stdout. + +## Compare two existing report sets + +The report comparison command requires a left baseline and a right candidate: + +```bash [Terminal] +vieval report compare +``` + +It reads normalized cases, aligns them by a case key, and reports matched deltas plus added and removed cases. Per-case and grouped deltas use matched records and are `right - left`; grouping uses the right-side record's selector value. A missing selected score on a matched case is treated as `0`. The overall delta instead subtracts the two full record-set means, ignoring records that lack the selected score, so additions and removals affect it and it is not the mean matched delta. `--score-kind ` defaults to `exact`, and output can be `table` or `json`. + +The default alignment key is the `benchmark.case.id` metric when present, then `vieval.case.id`, then the record's `caseId`. Use `--case-key ` when the benchmark has another stable identifier. An explicit key must exist on every record. Duplicate resolved keys on either side are an error rather than an arbitrary match. + +Selectors used by `--case-key`, `--group-by`, and `report cases` look for an exact metric name first, then `scores.` or a bare score name, then a direct case-record field. For example, `benchmark.category` can refer to a metric, while `state` refers to the normalized record field. + +Do not confuse this command with top-level `vieval compare`: + +- `vieval report compare ` compares case artifacts that already exist. +- `vieval compare --config ... --comparison ...` loads a comparison-mode config, executes every configured method against one benchmark, shares the configured benchmark cache namespace, and can write its aggregate comparison artifact with `--output`. + +## Run one complete comparison flow + +The following flow retains two executions, indexes the combined root, inspects candidate cases, analyzes all runs, and finally compares case scores. It does not depend on a fabricated run ID because report commands discover runs recursively. + +```bash [Terminal — retain the baseline] +vieval run \ + --config ./vieval.config.ts \ + --workspace local \ + --experiment baseline \ + --attempt attempt-a \ + --report-out .vieval/reports/baseline +``` + +```bash [Terminal — retain the candidate] +vieval run \ + --config ./vieval.config.ts \ + --workspace local \ + --experiment candidate \ + --attempt attempt-a \ + --report-out .vieval/reports/candidate +``` + +```bash [Terminal — index all retained runs] +vieval report index .vieval/reports +``` + +```bash [Terminal — inspect candidate cases] +vieval report cases .vieval/reports/candidate \ + --where state=failed \ + --format jsonl +``` + +```bash [Terminal — analyze run-level reliability] +vieval report analyze .vieval/reports --format json +``` + +```bash [Terminal — compare aligned case scores] +vieval report compare \ + .vieval/reports/baseline \ + .vieval/reports/candidate \ + --case-key benchmark.case.id \ + --score-kind exact \ + --format table +``` + +The final command requires every case to emit a unique `benchmark.case.id`. Omit `--case-key` if that guarantee does not exist and the built-in fallback identities are appropriate. + +A common failure is treating `--json` as persistence or pointing report commands at a directory created by a run that omitted `--report-out`. Another is aligning on generated or duplicate identifiers; comparison is meaningful only when the chosen key denotes the same benchmark case on both sides. + +See [API](/en/api/) for public entrypoints and [Config](/en/config/) for the current configuration surface. diff --git a/docs/content/en/guide/learn/tasks-cases-and-inputs.md b/docs/content/en/guide/learn/tasks-cases-and-inputs.md new file mode 100644 index 0000000..addedde --- /dev/null +++ b/docs/content/en/guide/learn/tasks-cases-and-inputs.md @@ -0,0 +1,88 @@ +--- +title: Tasks, Cases, and Inputs +prev: + text: Core Concepts + link: /en/guide/core-concepts +next: + text: Assertions, Scores, and Metrics + link: /en/guide/learn/assertions-scores-and-metrics +--- + +# Tasks, Cases, and Inputs + +This lesson turns a behavior you want to evaluate into a task with repeatable cases. By the end, you will know where case input appears at runtime and how to keep dataset rows comparable across reports. + +## Define a task boundary + +`describeTask` groups the cases for one evaluated behavior and registers that task when the evaluation module loads. Give the task a name that remains meaningful across the run and evaluation matrix variants that may expand it. + +Use `caseOf` when the task has one named scenario: + +```ts [evals/addition.eval.ts] +import { describeTask, expect } from 'vieval' + +describeTask('addition', ({ caseOf }) => { + caseOf('adds two positive numbers', ({ matrix }) => { + const { a, b, expected } = matrix.inputs + expect(a + b).toBe(expected) + }, { + input: { a: 20, b: 22, expected: 42 }, + }) +}) +``` + +The string passed to `caseOf` is the case's human-readable name. The `input` option becomes the callback's typed `matrix.inputs` value. + +## Generate cases from a dataset + +`casesFromInputs` registers one case for every item in a readonly array. The callback receives the complete array item, so the dataset can carry both the values under test and a stable identifier: + +```ts [evals/addition-dataset.eval.ts] +import { describeTask, expect } from 'vieval' + +const inputs = [ + { input: { a: 1, b: 2, expected: 3 }, name: 'addition-small' }, + { input: { a: 20, b: 22, expected: 42 }, name: 'addition-large' }, +] + +describeTask('addition', ({ casesFromInputs }) => { + casesFromInputs('addition', inputs, ({ matrix, metric }) => { + const { a, b, expected } = matrix.inputs.input + + metric('benchmark.case.id', matrix.inputs.name) + expect(a + b).toBe(expected) + }) +}) +``` + +This call registers `addition #1` and `addition #2`. The `name` property belongs to each input object; `casesFromInputs` does not use it as the generated case name. Here it is emitted as `benchmark.case.id`, which report comparison prefers as its stable matching key. + +## Follow input resolution + +The case callback context has a `matrix` that combines the resolved task matrix with the case input: + +```text +project/eval/task matrix layers + -> context.task.matrix.run / eval / meta + +caseOf(..., { input }) or casesFromInputs(...) + -> context.matrix.inputs +``` + +For the dataset example, `matrix.inputs` is one object from `inputs`, so the arithmetic values are under `matrix.inputs.input`. The dataset's `name` is therefore `matrix.inputs.name`. + +`context.task.matrix` still contains only the scheduled run, evaluation, and metadata rows. Case input is added to the case-scoped `context.matrix`; reading `context.task.matrix.inputs` is a boundary error. + +## Choose stable names and identifiers + +An explicit `caseOf` name is under your control. Keep it stable when the scenario is the same, because the generated report `caseId` is derived from the case's position and name. + +For `casesFromInputs`, generated names use the prefix plus a one-based position: ` #1`, ` #2`, and so on. Inserting or reordering rows changes those generated names and IDs. If reports must compare the same logical sample across changing dataset order, carry a unique identifier in the input and emit it as `benchmark.case.id` as shown above. That value must be unique across every case in each compared report, including cases from other tasks or projects. Duplicate comparison keys are rejected rather than matched ambiguously. + +Report selectors can address direct case fields such as `caseName` and `caseId`, scores, or emitted metric names. Default case comparison looks for `benchmark.case.id`, then `vieval.case.id`, and finally the generated `caseId`. + +::: warning Keep registration inside the task +`caseOf` and `casesFromInputs` require an active `describeTask` scope. Calling either function outside that scope throws during evaluation-module loading. +::: + +The task now defines what runs and which input each case receives. Next, decide what evidence each case should produce in [Assertions, Scores, and Metrics](/en/guide/learn/assertions-scores-and-metrics). For package entrypoints, see the [API overview](/en/api/). diff --git a/docs/content/en/guide/why.md b/docs/content/en/guide/why.md new file mode 100644 index 0000000..709cac2 --- /dev/null +++ b/docs/content/en/guide/why.md @@ -0,0 +1,34 @@ +--- +title: Why Vieval +prev: false +next: + text: Getting Started + link: /en/guide/getting-started +--- + +# Why Vieval + +Evaluation code is most useful when it can evolve with the behavior it measures. Vieval keeps evaluation files next to product code, where they can be reviewed, versioned, and changed through the same development workflow. + +## Repeatability needs more than a callback + +A callback can check one behavior once. A repeatable evaluation also needs to find the intended files, expand variations consistently, schedule the resulting work, and preserve evidence after execution. + +Vieval owns that evaluation lifecycle: discovery, matrix expansion, scheduling, execution evidence, reports, and comparison. Tasks remain ordinary TypeScript definitions, while the runner supplies the surrounding structure needed to execute the same evaluation again or across several model and scenario combinations. + +## One run serves people and tools + +During a run, the CLI provides human-readable progress and summaries. Pass `--json` to print machine-readable output, or `--report-out ` to write report artifacts that analysis, comparison, and automation can inspect. These output modes use the same discovery, scheduling, and execution path as the interactive CLI. + +## A good fit + +Vieval is a good fit when: + +- evaluations belong in a TypeScript repository and should be reviewed with product changes; +- the same task needs to run across model, scenario, rubric, or dataset variations; +- local runs and automation need consistent scheduling and evidence; +- case-level assertions, scores, and metrics need to remain available through explicitly written report artifacts. + +## Not a good fit + +Vieval does not provide hosted dataset management, an annotation UI or annotation product, or SaaS observability. Use dedicated services for those needs. Vieval can evaluate code that reads external data or calls remote models, but it remains the evaluation runner and artifact producer rather than the system that hosts those surrounding products. diff --git a/docs/content/zh-hans/api/index.md b/docs/content/zh-hans/api/index.md index e9934d0..d6605ad 100644 --- a/docs/content/zh-hans/api/index.md +++ b/docs/content/zh-hans/api/index.md @@ -1,14 +1,44 @@ # API -这一节会记录 Vieval 的包导出和运行时入口。 - -先从这些公开入口开始: - -- `vieval` -- `vieval/config` -- `vieval/core/runner` -- `vieval/core/assertions` -- `vieval/core/inference-executors` -- `vieval/plugins/chat-models` -- `vieval/expect` -- `vieval/testing/expect-extensions` +Vieval 提供以下公开导入路径。编写普通评测时,通常只需从 `vieval` 导入;需要自定义断言、执行流程或集成方式时,再选择相应的专用入口。 + +## 编写评测 + +| 导入路径 | 提供的功能 | +| --- | --- | +| `vieval` | 用于编写评测文件和 CLI 配置的常用 API,包括 `describeTask`、`describeEval`、`caseOf`、`casesFromInputs`、`defineConfig`、`loadEnv`、`requiredEnvFrom`,以及已安装 Vieval 匹配器(matcher)的 `expect`。 | +| `vieval/expect` | 仅导出已安装 Vieval 匹配器的 `expect`。不需要任务 DSL 时,可以使用此入口。 | +| `vieval/core/assertions` | 创建和执行结构化断言,并将断言结果转换为供运行器使用的分数。常用导出包括 `evaluateAssertions`、`expectRubric` 和 `toRunScores`。 | + +从 `vieval/core/assertions` 获得的断言结果不会自动写入任务的用例结果。调用方需要自行执行断言,并将结果传给 `toRunScores` 等后续处理函数。相关用法见[任务、用例与输入](/zh-hans/guide/learn/tasks-cases-and-inputs)和[断言、分数与指标](/zh-hans/guide/learn/assertions-scores-and-metrics)。 + +## 配置模型和执行器 + +| 导入路径 | 提供的功能 | +| --- | --- | +| `vieval/config` | 提供配置相关类型、插件契约,以及用于定义评测和任务对象的辅助函数,包括 `defineEval`、`defineTask` 和 `resolveModelByName`。CLI 配置文件使用的 `defineConfig` 和 `loadEnv` 从 `vieval` 导入。 | +| `vieval/plugins/chat-models` | 定义聊天模型和提供方,并根据评测矩阵或运行矩阵选择模型。常用导出包括 `chatModelFrom`、`ChatModels`、`ChatProviders`、`modelFromEval`、`modelFromMatrix` 和 `modelFromRun`。 | +| `vieval/core/inference-executors` | 创建提供方适配器、重试策略和 OpenAI 兼容客户端。常用导出包括 `createProviderAdapter`、`createRetryPolicy` 和 `createOpenAIFromEnv`。 | + +[配置](/zh-hans/config/)介绍配置文件的查找方式和各类配置项;[模型与推理执行器](/zh-hans/guide/learn/models-and-inference-executors)说明如何注册、选择和调用模型。 + +## 自定义执行流程 + +如果需要将 CLI 嵌入其他工具,或自行组织执行流程,可以使用下列入口。普通评测文件通常不需要使用这些入口。 + +| 导入路径 | 提供的功能 | +| --- | --- | +| `vieval/cli` | 解析顶层 CLI 参数并分派命令,导出 `parseTopLevelCliArguments` 和 `runTopLevelCli`。 | +| `vieval/core/runner` | 收集评测、生成调度任务、执行任务并汇总结果。常用导出包括 `collectEvalEntries`、`createRunnerSchedule`、`runScheduledTasks` 和 `aggregateRunResults`。 | +| `vieval/core/scheduler` | 分别控制工作区、项目、任务、尝试和用例的并发数,导出 `createSchedulerRuntime` 和 `getActiveScopes`。 | +| `vieval/core/processors/results` | 处理运行结果并执行内置的结果判定策略,导出 `processRunResults` 和 `ResultGateDecision`。 | + +如果需要自行组织运行器、调度器、重试和缓存逻辑,请先阅读[可靠执行](/zh-hans/guide/learn/reliable-execution)。结果汇总、判定和报告文件的关系见[报告与比较](/zh-hans/guide/learn/reports-and-comparisons)。 + +## 安装自定义匹配器 + +| 导入路径 | 提供的功能 | +| --- | --- | +| `vieval/testing/expect-extensions` | 为现有的 Vitest `expect` 显式安装 Vieval 匹配器,导出 `installVievalExpectMatchers`,以及 `KeywordMatcherOptions` 和 `ToolCallContainer` 类型。 | + +从 `vieval` 或 `vieval/expect` 导入的 `expect` 已安装 Vieval 匹配器。只有需要把匹配器安装到现有 Vitest `expect` 的集成代码,才需要调用 `installVievalExpectMatchers()`。 diff --git a/docs/content/zh-hans/config/index.md b/docs/content/zh-hans/config/index.md index 622382c..d5ab463 100644 --- a/docs/content/zh-hans/config/index.md +++ b/docs/content/zh-hans/config/index.md @@ -1,5 +1,45 @@ # 配置 -Vieval 配置应靠近拥有评测任务的项目。 +Vieval 会从执行命令时的工作目录开始逐级向上查找最近的 `vieval.config.*` 文件。支持的扩展名包括 `.ts`、`.mts`、`.cts`、`.js`、`.mjs`、`.cjs` 和 `.json`。使用 `--config ` 可以显式指定配置文件;相对路径以执行命令时的工作目录为基准。 -这一页后续会记录配置加载、模型提供方设置、runner 行为、reporters 和输出选项。 +配置文件所在的目录是默认根目录。项目没有设置 `root` 时,Vieval 会相对于该目录匹配 `include` 和 `exclude`;设置 `root` 后,则改为相对于项目根目录匹配。 + +## 配置内容 + +| 配置项 | 说明 | 延伸阅读 | +| --- | --- | --- | +| 环境变量 | 顶层 `env` 会在 `vieval run` 期间临时写入 `process.env`,运行结束后恢复原值。Vieval 不会自动读取 `.env*` 文件;如需读取,请在配置文件中从 `vieval` 导入并调用 `loadEnv`。 | [模型与推理执行器](/zh-hans/guide/learn/models-and-inference-executors) | +| 项目 | `projects` 用于设置项目名称、根目录、评测文件匹配规则、执行器、模型和项目级运行选项。在项目模式下,未提供 `projects` 时,Vieval 使用名为 `default` 的项目。 | [核心概念](/zh-hans/guide/core-concepts) | +| 工作区 | `workspaces` 中的每个 `{ id, root }` 会独立作为一个项目运行:`id` 是项目名称,`root` 是项目根目录。Vieval 不会另外创建工作区级任务。 | [核心概念](/zh-hans/guide/core-concepts) | +| 比较 | `comparisons` 用于配置 `vieval compare` 的基准、待比较方法和工作区查找规则。顶层的 `projects`、`workspaces` 和 `comparisons` 不能同时使用。 | [报告与比较](/zh-hans/guide/learn/reports-and-comparisons) | +| 插件与模型 | 顶层和项目级均可配置插件与模型。配置模型只会将模型加入该项目的可用模型列表;任务代码仍需主动选择并调用模型。 | [模型与推理执行器](/zh-hans/guide/learn/models-and-inference-executors) | +| 运行矩阵与评测矩阵 | 项目可通过 `runMatrix` 和 `evalMatrix` 定义变量组合。评测和任务还可以继续扩展、替换或禁用矩阵维度。Vieval 根据组合生成待执行任务,任务代码负责使用各维度的取值。 | [矩阵与数据集](/zh-hans/guide/learn/matrices-and-datasets) | +| 并发与执行策略 | 顶层 `concurrency` 可分别控制工作区、项目、任务、尝试和用例的并发数,项目级 `concurrency` 可控制项目、任务、尝试和用例的并发数。任务和用例还可以设置尝试次数、自动重试、超时和并发。CLI 参数为工作区、项目和任务设置并发上限;为尝试和用例设置的参数会覆盖配置值。`--attempt` 仅设置报告中的尝试标识,不会改变自动尝试次数。 | [可靠执行](/zh-hans/guide/learn/reliable-execution) | +| 缓存 | 任务通过 `TaskRunContext` 使用缓存。CLI 默认将项目缓存写入项目根目录下的 `.vieval/cache`。将多个方法与同一基准比较时,这些方法可以共享用例缓存。 | [可靠执行](/zh-hans/guide/learn/reliable-execution) | +| 报告器与报告文件 | 顶层 `reporters` 是项目的默认报告器列表;项目提供自己的 `reporters` 后,会替换该列表。`--report-out` 将报告写入指定目录,`--json` 只改变标准输出格式。 | [报告与比较](/zh-hans/guide/learn/reports-and-comparisons) | +| OpenTelemetry | `reporting.openTelemetry` 控制运行期间的 OpenTelemetry span 和运行结束回调。`--report-out` 生成的本地报告事件与 OTLP 结构文件不受该选项控制。 | [报告与比较](/zh-hans/guide/learn/reports-and-comparisons) | + +## 继承和覆盖 + +不同配置项采用不同的继承规则: + +- 顶层 `concurrency.workspace` 只控制工作区并发。顶层 `concurrency` 中的 `project`、`task`、`attempt` 和 `case` 会成为各项目的默认值。项目可以逐项覆盖,未设置的值继续使用顶层配置。 +- 顶层 `models` 和 `reporters` 是项目的默认列表。项目一旦提供同名数组,就会替换顶层列表,而不是与之合并。 +- `runMatrix` 和 `evalMatrix` 按「项目 → 评测 → 任务」的顺序处理。每一层可以通过 `extend`、`override` 或 `disable` 调整上一层的矩阵。 +- 顶层插件按声明顺序修改整个配置。项目插件只修改对应项目的配置。 + +只有类型定义明确列出的配置项才能在相应层级使用。上述继承关系不代表同一个字段可以写在任意层级。 + +## CLI 参数 + +运行时常用的配置相关参数包括: + +- `--config` 指定配置文件。 +- `--project` 选择要运行的项目。 +- 工作区、项目和任务的并发参数设置运行时上限;尝试和用例的并发参数覆盖配置值。 +- `--workspace`、`--experiment` 和 `--attempt` 设置报告中的标识。 +- `--json` 改变标准输出格式,`--report-out` 将报告写入目录。 + +这些参数只影响对应命令,不能用于随意覆盖配置文件中的其他字段。 + +配置相关的公开类型和辅助函数见 [API](/zh-hans/api/)。 diff --git a/docs/content/zh-hans/guide/core-concepts.md b/docs/content/zh-hans/guide/core-concepts.md new file mode 100644 index 0000000..75ae0da --- /dev/null +++ b/docs/content/zh-hans/guide/core-concepts.md @@ -0,0 +1,54 @@ +--- +title: 核心概念 +prev: + text: 快速开始 + link: /zh-hans/guide/getting-started +next: + text: 任务、用例与输入 + link: /zh-hans/guide/learn/tasks-cases-and-inputs +--- + +# 核心概念 + +先了解 Vieval 如何根据项目配置生成用例结果,才能看清矩阵、重试和报告选项分别作用于哪个步骤: + +```text +项目 + -> 发现评测项及其任务 + -> 读取推理执行器、运行矩阵和评测矩阵 + +每个评测项 × 每个推理执行器 × 每行运行矩阵 × 每行评测矩阵 + -> 待调度任务 + -> 用例 -> 断言结果 / 分数 / 指标 + +任务执行 + -> 尝试 + -> 每个用例先执行一次 + -> 失败或超时时,按 autoRetry 重试该用例 + -> 仍有用例失败或超时时,按 autoAttempt 开始下一次尝试 + -> 重新执行全部用例 + +执行 --json -> 机器可读的标准输出 +执行 --report-out -> 报告目录 -> 分析 / 比较 +``` + +**项目**指定从哪个目录发现评测文件,并保存推理执行器、矩阵等项目级配置。通过 `defineEval` 声明的评测项可以包含一个**任务**,`describeTask` 等 DSL 则会直接注册任务。运行器会组合每个评测项、每个推理执行器、每行运行矩阵和每行评测矩阵,为每组组合创建一个待调度任务。任务包含一个或多个**用例**;用例执行后会产生通过或失败的断言结果,也可以记录归一化分数和自定义指标。 + +Vieval 提供两种重复执行机制:尝试和重试。每次**尝试**都会执行任务中的全部用例;配置 `autoAttempt` 后,如果仍有符合条件的用例失败或超时,运行器会开始下一次尝试,并把各次结果都计入任务汇总。配置 `autoRetry` 后,**重试**只会在当前尝试中再次执行失败或超时的用例。 + +CLI 默认显示便于阅读的进度和摘要。`--json` 会把标准输出改为机器可读的结果,`--report-out ` 则会另外写入分析和比较命令所需的报告文件。 + +| 概念 | 职责 | 常见范围 | +| --- | --- | --- | +| 项目 | 设置评测文件的发现规则,并保存推理执行器和矩阵默认值 | 同一目录中的一组评测文件 | +| 推理执行器 | 项目可以注册多个推理执行器,运行器会分别针对每个执行器调度任务;它与运行矩阵中的模型选择是两个独立维度 | 项目 | +| 任务 | 定义要评测的行为;调度后会关联一个评测项、一个推理执行器,以及当前组合所采用的运行矩阵和评测矩阵取值 | 一项被测行为的一组具体配置 | +| 用例 | 执行一个输入或场景,并记录断言结果、分数和指标 | 任务中的一次检查 | +| 运行矩阵 | 展开模型、场景等执行侧变量 | 项目、评测定义或任务定义 | +| 评测矩阵 | 展开评分标准等评测侧变量 | 项目、评测定义或任务定义 | +| 尝试 | 执行任务中的全部用例;仍有用例失败或超时时,可按 `autoAttempt` 重新执行全部用例 | 一个已调度任务 | +| 重试 | 按 `autoRetry` 在同一次尝试中重新执行失败或超时的用例 | 一次用例执行 | +| 报告目录 | 使用 `--report-out` 时保存运行摘要、事件与用例记录 | 一次运行及其配置的输出目录 | +| 分析与比较 | 读取报告文件以检查结果,或比较多次运行和不同评测方案 | 一组或多组报告目录 | + +下一步学习如何编写[任务、用例与输入](/zh-hans/guide/learn/tasks-cases-and-inputs)。 diff --git a/docs/content/zh-hans/guide/getting-started.md b/docs/content/zh-hans/guide/getting-started.md index 852f171..a6ebb30 100644 --- a/docs/content/zh-hans/guide/getting-started.md +++ b/docs/content/zh-hans/guide/getting-started.md @@ -1,15 +1,162 @@ +--- +title: 快速开始 +prev: + text: 为什么选择 Vieval + link: /zh-hans/guide/why +next: + text: 核心概念 + link: /zh-hans/guide/core-concepts +--- + # 快速开始 -安装包: +本页会创建并运行一个确定性评测。所有代码都在本地 Node.js 进程中执行,不会调用模型提供方。 + +## 安装 Vieval -```sh +::: code-group + +```sh [pnpm] pnpm add -D vieval ``` -创建 Vieval DSL 评测任务,配置模型提供方,然后运行 CLI。 +```sh [npm] +npm install --save-dev vieval +``` + +::: + +## 配置项目 + +在项目根目录创建配置文件: + +::: code-group + +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' + +export default defineConfig({ + projects: [ + { + include: ['evals/*.eval.ts'], + inferenceExecutors: [ + { + id: 'local', + }, + ], + models: [ + { + aliases: [], + id: 'local:deterministic', + inferenceExecutor: 'local', + inferenceExecutorId: 'local', + model: 'deterministic', + }, + ], + name: 'getting-started', + root: '.', + }, + ], +}) +``` + +::: + +Vieval 会以配置文件所在目录为基准解析 `root`。本例中的 `root: '.'` 因此指向项目根目录,`include` 再从该目录中查找匹配 `evals/*.eval.ts` 的文件。`models` 中的条目是本地占位配置。当前版本要求项目至少注册一个模型,才会自动执行已发现的 DSL 任务。本例任务代码不会读取这项配置,也不会调用模型提供方。 + +## 编写评测 + +创建与配置匹配的评测文件: + +::: code-group + +```ts [evals/arithmetic.eval.ts] +import { caseOf, describeTask, expect } from 'vieval' + +describeTask('arithmetic', () => { + caseOf('adds two numbers', () => { + expect(20 + 22).toBe(42) + }) +}) +``` + +::: + +`describeTask` 注册任务,`caseOf` 为该任务注册一个用例,`expect` 检查断言。如果断言不成立,`expect` 会抛出错误,该用例也会失败。 + +## 运行评测 + +::: code-group + +```sh [pnpm] +pnpm vieval run --config ./vieval.config.ts +``` + +```sh [npm] +npx vieval run --config ./vieval.config.ts +``` + +::: + +运行器会发现一个项目,调度一个任务,并记录一个通过的用例。CLI 的实际文案和耗时可能随版本或运行环境变化,不必与本文逐字一致。 + +::: info 无需凭据 +这个任务只执行本地算术断言,不会发起模型请求。因此不需要配置 API 凭据,也不会产生模型调用费用。 +::: + +## 可选:如何阅读后续示例 + +普通行高亮会标出当前需要关注的完整代码行: + +::: code-group + +```ts [evals/arithmetic.eval.ts] +const expected = 42 // [!code highlight] +expect(20 + 22).toBe(expected) +``` + +::: + +词高亮只标出指定的标识符或片段: + +::: code-group -```sh -pnpm vieval run +```ts [evals/arithmetic.eval.ts] +expect(20 + 22).toBe(42) // [!code word:toBe] ``` -详细设置指南会在公开 API 和示例稳定后继续补充。 +::: + +指南会用错误高亮指出导致失败的代码行: + +::: code-group + +```ts [evals/arithmetic.eval.ts] +expect(20 + 22).toBe(41) // [!code error] +``` + +::: + +删除行和新增行标记会同时展示原代码与修正结果: + +::: code-group + +```ts [evals/arithmetic.eval.ts] +expect(20 + 22).toBe(41) // [!code --] +expect(20 + 22).toBe(42) // [!code ++] +``` + +::: + +当示例需要突出修改过程时,也会使用 diff 代码块: + +::: code-group + +```diff [evals/arithmetic.eval.ts] +- expect(20 + 22).toBe(41) ++ expect(20 + 22).toBe(42) +``` + +::: + +至此,`vieval.config.ts` 已经指定评测文件的位置,`evals/arithmetic.eval.ts` 则注册了要执行的任务和用例。接下来阅读[核心概念](/zh-hans/guide/core-concepts),了解项目、任务、用例、尝试与报告之间的关系。 diff --git a/docs/content/zh-hans/guide/index.md b/docs/content/zh-hans/guide/index.md index 1e01890..d01b43c 100644 --- a/docs/content/zh-hans/guide/index.md +++ b/docs/content/zh-hans/guide/index.md @@ -1,11 +1,20 @@ -# 介绍 +# Vieval 指南 -Vieval 是一个面向 agents、模型和模型驱动工作流的 Vitest 风格评测框架。 +你可以在 TypeScript 工程中用 Vieval 定义并重复运行评测。Vieval 会组合模型、场景等变量,并据此调度任务。运行结果既能以终端摘要呈现,也能输出为 JSON 或报告文件。 -当前中文文档先提供站点结构和占位内容,后续可以逐步补充完整指南。 +## 从这里开始 -## 这里会放什么 +- [为什么选择 Vieval](/zh-hans/guide/why):了解 Vieval 负责哪些工作,以及哪些工作需要交给其他服务。 +- [快速开始](/zh-hans/guide/getting-started):运行一次确定性的评测,不调用模型提供方,不需要凭据,也不会产生模型调用费用。 +- [核心概念](/zh-hans/guide/core-concepts):了解评测中的核心对象和完整执行过程。 -- 评测流程里的概念和术语。 -- 任务、断言、模型执行器和 reporters 的使用示例。 -- 从包 API 维护或生成的参考文档。 +## 沿着一次评测学习 + +1. [任务、用例与输入](/zh-hans/guide/learn/tasks-cases-and-inputs) +2. [断言、分数与指标](/zh-hans/guide/learn/assertions-scores-and-metrics) +3. [模型与推理执行器](/zh-hans/guide/learn/models-and-inference-executors) +4. [矩阵与数据集](/zh-hans/guide/learn/matrices-and-datasets) +5. [可靠执行](/zh-hans/guide/learn/reliable-execution) +6. [报告与比较](/zh-hans/guide/learn/reports-and-comparisons) + +如需查询配置选项或公开 API,请继续阅读[配置参考](/zh-hans/config/)与 [API 参考](/zh-hans/api/)。 diff --git a/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md b/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md new file mode 100644 index 0000000..87fa677 --- /dev/null +++ b/docs/content/zh-hans/guide/learn/assertions-scores-and-metrics.md @@ -0,0 +1,115 @@ +--- +title: 断言、分数与指标 +prev: + text: 任务、用例与输入 + link: /zh-hans/guide/learn/tasks-cases-and-inputs +next: + text: 模型与推理执行器 + link: /zh-hans/guide/learn/models-and-inference-executors +--- + +# 断言、分数与指标 + +评测用例需要明确记录通过或失败,并在必要时补充分数和指标。本节先介绍可由代码直接判断的断言,再说明如何记录取值在 `0..1` 之间的分数和报告指标。 + +## 从确定性断言开始 + +Vieval 导出了与 Vitest 兼容的 `expect`。如果匹配失败,`expect` 会抛出错误,当前用例也会随之失败。没有记录自定义分数的用例通过时,会得到 `1` 分的 `exact` 分数;失败时则得到 `0` 分。 + +::: code-group +```ts [evals/normalized-answer.eval.ts] +import { describeTask, expect } from 'vieval' + +describeTask('normalized answer', ({ caseOf }) => { + caseOf('removes surrounding whitespace', () => { + const answer = ' forty-two '.trim() + expect(answer).toBe('forty-two') + }) +}) +``` +::: + +精确值、数据结构、必需字段,以及其他可以由代码可靠判断的规则,都适合使用 `expect`。 + +## 选择断言、分数或指标 + +| 术语 | 在 Vieval 中的含义 | 对任务用例的影响 | +| --- | --- | --- | +| 断言(Assertion) | 检查结果是否满足要求。`expect` 失败时会抛出错误;`vieval/core/assertions` 还提供返回结构化结果的断言函数。 | `expect` 抛出的错误会使当前用例失败。核心断言函数的返回值不会自动写入任务结果。 | +| 分数(Score) | 通过 `context.score(value, kind)` 记录的 `0..1` 数值。分数类型可以是 `exact` 或 `judge`,默认为 `exact`。 | 用例通过时,分数会参与任务结果的汇总;相应事件也会写入报告中的用例记录。 | +| 指标(Metric) | 通过 `context.metric(name, value)` 记录的具名数据。值可以是字符串、数值、布尔值、`null`,或这些值组成的数组。 | 指标会写入报告事件和用例记录,但不会改变用例状态,也不参与分数汇总。 | +| 评分标准(rubric) | 一类核心断言。`judge` 回调返回评分理由和分数,Vieval 再将分数与 `minScore` 比较;默认阈值为 `0.7`。 | 返回一个分数类型为 `judge` 的 `AssertionOutcome`。你需要自行决定如何将它写入当前用例。 | + +`caseOf` 和 `casesFromInputs` 收到的用例回调上下文提供 `score` 与 `metric` 方法。 + +## 在用例中记录分数与指标 + +需要计入任务平均分的数据应使用 `score`。只用于筛选、分组或排查问题的数据应使用 `metric`。 + +::: code-group +```ts [evals/retrieval.eval.ts] +import { describeTask, expect } from 'vieval' + +describeTask('retrieval', ({ caseOf }) => { + caseOf('finds the expected documents', ({ metric, score }) => { + const expected = new Set(['doc-a', 'doc-b']) + const retrieved = ['doc-a', 'doc-c'] + const matches = retrieved.filter(id => expected.has(id)).length + const recall = matches / expected.size + + score(recall, 'exact') + metric('benchmark.case.id', 'retrieval-basic') + metric('retrieved.count', retrieved.length) + + expect(recall).toBeGreaterThan(0) + }) +}) +``` +::: + +分数必须是 `0..1` 之间的有限数值。在同一个用例中,如果再次记录相同类型的分数,新值会替换旧值。因此,每种分数类型最好只记录一次最终结果。 + +任务汇总分数和报告中的用例记录并非同时写入。每次调用 `score` 都会立即发出事件,所以用例失败前记录的分数仍可能出现在报告中。 + +如果用例最终失败或超时,任务汇总会忽略这些自定义分数,并为该用例计入一个 `exact: 0`。用例结束时,只有报告中的用例记录还没有 `exact` 分数,Vieval 才会补写 `exact: 0`。因此,排查结果时应同时检查用例的 `state` 和分数记录。 + +指标用于筛选、分组和诊断报告。单独记录指标不会让用例通过或失败,也不会提高任务汇总分数。 + +## 使用评分标准断言 + +`vieval/core/assertions` 提供 `expectRubric` 和 `evaluateAssertions`。`expectRubric` 用来创建断言;当 `evaluateAssertions` 执行该断言时,它会调用 `judge`,将返回的分数限制在 `0..1` 之间,再用 `minScore` 判断是否通过,最后返回结构化的 `AssertionOutcome`: + +::: code-group +```ts [rubric.ts] +import { evaluateAssertions, expectRubric } from 'vieval/core/assertions' + +const [outcome] = await evaluateAssertions([ + expectRubric({ + id: 'concise-answer', + judge: async ({ text }) => ({ + reason: text.length <= 80 ? 'Answer is concise.' : 'Answer is too long.', + score: text.length <= 80 ? 1 : 0, + }), + minScore: 0.8, + }), +], { + text: 'A short answer.', +}) +``` +::: + +这个示例只执行本地代码,不会调用模型。`expectRubric` 本身既不选择模型,也不请求模型服务;是否调用模型完全由 `judge` 回调决定。这个回调也可以使用本地规则或人工评分结果。 + +核心断言 API 与任务 DSL 是两套独立的公开接口。`evaluateAssertions` 只返回断言结果,不会自动调用用例上下文中的 `score`、`metric` 或 `expect`。`toRunScores(outcomes)` 可以把这些结果转换成 `RunScore[]`,但同样不会将分数写入当前用例。 + +得到 `outcome` 后,你可以调用 `score(outcome.score, outcome.scoreKind)` 记录分数,在 `outcome.pass` 为 `false` 时抛出错误,也可以将评分理由记为指标。选择哪种方式取决于你希望它如何影响用例状态和任务汇总。 + +如果你通过抛出错误使当前用例失败,任务汇总会为该用例计入 `exact: 0`。抛错前已经发出的分数事件仍可能保留在报告的用例记录中。 + +::: warning 模型评分会带来运行成本 +如果 `judge` 回调调用模型,就需要配置对应的模型服务和凭据,并承担请求延迟和费用。并发运行用例时,可能同时发出多次评分请求,因此应根据服务方的速率限制和配额设置并发上限。还应检查服务方如何处理数据,并决定哪些用例输入、模型输出、评分提示词、评分理由、指标和事件可以写入报告文件。 +::: + +常见错误包括把指标当成分数,或误以为评分标准断言的结果会自动写入当前用例。编写评测时,应分别处理用例状态、参与汇总的分数,以及用于排查问题的指标。 + +下一步在[模型与推理执行器](/zh-hans/guide/learn/models-and-inference-executors)中学习如何注册并选择模型。所有核心断言 API 都可以从 [`vieval/core/assertions`](/zh-hans/api/) 导入。 diff --git a/docs/content/zh-hans/guide/learn/matrices-and-datasets.md b/docs/content/zh-hans/guide/learn/matrices-and-datasets.md new file mode 100644 index 0000000..2f341b1 --- /dev/null +++ b/docs/content/zh-hans/guide/learn/matrices-and-datasets.md @@ -0,0 +1,216 @@ +--- +title: 矩阵与数据集 +prev: + text: 模型与推理执行器 + link: /zh-hans/guide/learn/models-and-inference-executors +next: + text: 可靠执行 + link: /zh-hans/guide/learn/reliable-execution +--- + +# 矩阵与数据集 + +`runMatrix` 和 `evalMatrix` 中各维度的取值会组成矩阵组合,每增加一个矩阵组合,就会增加相应的调度任务。`casesFromInputs` 则为数组中的每个输入注册一个用例,因此输入元素会增加每个调度任务中的用例数。 + +## 从一个模型维度开始 + +先在项目的 `runMatrix` 中添加 `model` 维度。这里只有一个模型别名,因此运行矩阵只有一种组合: + +::: code-group + +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' + +export default defineConfig({ + projects: [ + { + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + runMatrix: { + extend: { + model: ['assistant-default'], + }, + }, + }, + ], +}) +``` + +::: + +再添加 `scenario`。Vieval 会组合所有维度的取值,因此一个模型和两个场景会产生两种运行矩阵组合: + +::: code-group + +```ts [vieval.config.ts] +export default defineConfig({ + projects: [ + { + name: 'chat-evals', + runMatrix: { + extend: { + model: ['assistant-default'], + scenario: ['baseline', 'stress'], // [!code ++] + }, + }, + }, + ], +}) +``` + +::: + +任务可以从 `context.task.matrix.run` 读取本次调度选中的值。维度名称本身不会触发任何操作:`scenario` 只会让调度器创建更多矩阵组合。任务需要自行读取该值,并决定不同场景分别执行什么逻辑。 + +## 区分运行矩阵与评测矩阵 + +`runMatrix` 用来区分被测方案,例如模型、提示词语言或运行场景。`evalMatrix` 用来区分评测方法,例如评分标准或评分模型。 + +::: code-group + +```ts [vieval.config.ts] +export default defineConfig({ + projects: [ + { + evalMatrix: { + extend: { + rubric: ['strict', 'lenient'], + }, + }, + name: 'chat-evals', + runMatrix: { + extend: { + model: ['assistant-default'], + scenario: ['baseline', 'stress'], + }, + }, + }, + ], +}) +``` + +::: + +上面的配置有两种运行矩阵组合和两种评测矩阵组合。Vieval 会为每个「评测项 + 推理执行器」组合创建四个调度任务。任务可以分别从 `context.task.matrix.run` 和 `context.task.matrix.eval` 读取两类配置。 + +矩阵只负责组合配置,不会自动调用模型或执行评分。即使维度名为 `model` 或 `rubric`,任务和断言仍需读取对应值,并实现模型调用或评分逻辑。 + +## 按项目、评测和任务三层合并矩阵配置 + +Vieval 按以下顺序合并矩阵配置: + +1. `vieval.config.*` 中项目级的 `runMatrix` 和 `evalMatrix`。 +2. `defineEval` 中评测级的 `matrix`。 +3. `defineTask` 中任务级的 `matrix`。 + +每一层内部再按以下顺序处理: + +1. `disable` 删除从外层继承的指定维度;它的值是维度名称数组。 +2. `extend` 添加新维度,或向已有维度追加值;重复值会被删除。 +3. `override` 用当前层给出的值替换整个维度。 + +::: code-group + +```ts [evals/layered.eval.ts] +import { defineEval, defineTask } from 'vieval/config' + +export default defineEval({ + description: 'Shows matrix layering.', + matrix: { + runMatrix: { + disable: ['scenario'], + extend: { + promptLanguage: ['en', 'zh'], + }, + override: { + model: ['assistant-default'], + }, + }, + }, + name: 'layered', + task: defineTask({ + id: 'layered', + matrix: { + evalMatrix: { + override: { + rubric: ['strict'], + }, + }, + }, + run(context) { + return { + scores: [{ + kind: 'exact', + score: context.task.matrix.eval.rubric === 'strict' ? 1 : 0, + }], + } + }, + }), +}) +``` + +::: + +在前一节项目配置的基础上,评测级配置会删除 `scenario`,添加 `en` 和 `zh` 两种 `promptLanguage`,并把 `model` 替换为 `assistant-default`。任务级配置再把 `rubric` 替换为 `strict`。 + +项目配置中的 `runMatrix` 和 `evalMatrix` 可以直接写成扁平的矩阵对象,Vieval 会把它们当作 `extend` 处理。`defineEval` 的评测级矩阵和 `defineTask` 的任务级矩阵只接受分层形式,需要通过 `extend`、`override` 或 `disable` 调整继承的配置。不要在同一个对象中混写这些控制项和普通维度名称,否则配置会报错。 + +## 添加输入用例而不增加矩阵组合 + +`casesFromInputs` 接受一个由你的代码加载或构造的数组。它为每个元素注册一个用例,并在回调中通过 `matrix.inputs` 提供当前元素: + +::: code-group + +```ts [evals/dataset.eval.ts] +import { describeTask, expect } from 'vieval' + +const inputs = [ + { expected: 4, left: 2, right: 2 }, + { expected: 7, left: 3, right: 4 }, + { expected: 12, left: 5, right: 7 }, +] + +describeTask('arithmetic dataset', ({ casesFromInputs }) => { + casesFromInputs('addition', inputs, ({ matrix }) => { + const result = matrix.inputs.left + matrix.inputs.right + expect(result).toBe(matrix.inputs.expected) + }) +}) +``` + +::: + +这个 API 不会自行读取外部数据集。数据来自 JSON、数据库或其他来源时,请先在评测代码中加载并校验,再把得到的数组传给 `casesFromInputs`。 + +运行结构如下: + +```text +项目 + -> 已发现的评测项及其任务 + -> 推理执行器 × 运行矩阵组合 × 评测矩阵组合 + -> 一次调度任务执行 + -> 通过 caseOf 注册的用例 + casesFromInputs 为每个输入注册的用例 +``` + +输入元素不会增加矩阵组合或调度任务。每个已经创建的调度任务都会为这些元素分别运行一个用例。 + +## 在运行前计算展开规模 + +矩阵展开后,调度任务数为: + +```text +已发现评测项 × 项目推理执行器 × 运行矩阵组合数 × 评测矩阵组合数 +``` + +如果任务只调用了一次 `casesFromInputs`,并传入 `N` 个元素,那么首次执行的用例回调次数就是调度任务数乘以 `N`。任务中每增加一次 `caseOf` 调用或另一处 `casesFromInputs` 调用,还会相应增加用例数。 + +`autoAttempt` 和 `autoRetry` 不会创建新的调度任务。它们只会在已有的调度任务中再次执行用例回调,因此需要另行计算增加的执行次数。 + +例如,一个评测项使用一个推理执行器、两种运行矩阵组合、两种评测矩阵组合和三个输入时,会创建四个调度任务,并首次执行十二次用例回调。 + +::: warning 注意组合增长与速率限制 +为矩阵维度增加取值会增加矩阵组合;增加输入则会增加每个调度任务中的用例。两者都会延长运行时间,并增加报告中的记录数。只有任务或用例代码实际调用模型服务时,调用次数和费用才会随之增加;一个用例也可能发起多次调用。请按实际调用次数估算费用,并根据服务方的限流规则设置并发数。 +::: + +下一步可在[可靠执行](/zh-hans/guide/learn/reliable-execution)中设置重试、自动尝试、超时和并发。矩阵与任务的完整类型见[配置参考](/zh-hans/config/)和[API 参考](/zh-hans/api/)。 diff --git a/docs/content/zh-hans/guide/learn/models-and-inference-executors.md b/docs/content/zh-hans/guide/learn/models-and-inference-executors.md new file mode 100644 index 0000000..d33424f --- /dev/null +++ b/docs/content/zh-hans/guide/learn/models-and-inference-executors.md @@ -0,0 +1,171 @@ +--- +title: 模型与推理执行器 +prev: + text: 断言、分数与指标 + link: /zh-hans/guide/learn/assertions-scores-and-metrics +next: + text: 矩阵与数据集 + link: /zh-hans/guide/learn/matrices-and-datasets +--- + +# 模型与推理执行器 + +注册模型后,评测代码可以通过固定的名称查找模型配置。注册和查找都不会发出推理请求;只有评测代码或智能体显式发起模型请求,才会访问远程或本地模型服务。本节将依次说明如何注册模型、选择模型,以及获取调用模型所需的配置。 + +## 注册带提供方配置的模型 + +内置聊天模型插件会把每次调用 `chatModelFrom` 返回的模型定义添加到配置的 `models` 列表。下面的配置通过 `loadEnv` 读取 `.env` 文件;插件处理模型配置时,`requiredEnvFrom` 会检查 `OPENAI_API_KEY` 是否存在: + +::: code-group +```ts [vieval.config.ts] +import { cwd } from 'node:process' + +import { defineConfig, loadEnv, requiredEnvFrom } from 'vieval' // [!code ++] +import { chatModelFrom, ChatModels } from 'vieval/plugins/chat-models' // [!code ++] + +export default defineConfig({ + env: loadEnv('test', cwd(), ''), // [!code ++] + plugins: [ + ChatModels({ // [!code ++] + models: [ + chatModelFrom({ + aliases: ['assistant-default'], + apiKey: config => requiredEnvFrom(config.env, { // [!code ++] + name: 'OPENAI_API_KEY', // [!code ++] + type: 'string', // [!code ++] + }), // [!code ++] + inferenceExecutor: 'openai', + model: 'gpt-4.1-mini', + }), + ], + }), + ], + projects: [ + { + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + runMatrix: { + extend: { + model: ['assistant-default'], + }, + }, + }, + ], +}) +``` +::: + +如果没有传入 `id`,`chatModelFrom` 会根据执行器 ID 和具体模型名生成 `openai:gpt-4.1-mini`。这个模型也可以通过别名 `assistant-default` 查找。 + +::: warning 凭据 +不要把 `OPENAI_API_KEY` 提交到版本控制中。`requiredEnvFrom` 只负责检查环境变量,不能替你生成、轮换或保护密钥。也不要将处理后的模型参数写入指标、日志或报告事件,因为其中可能包含凭据。 +::: + +::: tip 在矩阵轴中使用稳定别名 +`modelFromRun` 可以通过已注册模型的 ID、具体模型名或别名查找模型。矩阵中使用 `assistant-default` 这类表示用途的别名后,更换具体模型时无需同时修改矩阵轴中的模型名称。 +::: + +## 区分模型配置与调度配置 + +`ChatModels` 将模型定义加入 Vieval 配置。需要让多个模型共用服务地址、凭据或其他参数时,可以通过 `ChatProviders` 注册一份提供方配置,再让模型通过 `provider` 引用它。 + +Vieval 会按照 `plugins` 数组中的顺序处理插件。`ChatProviders` 必须写在引用该提供方的 `ChatModels` 前面,否则处理模型配置时会抛出 `Unknown chat provider` 错误: + +::: code-group +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' +import { + chatModelFrom, + ChatModels, + chatProviderFrom, + ChatProviders, +} from 'vieval/plugins/chat-models' + +export default defineConfig({ + plugins: [ + ChatProviders({ + providers: [ + chatProviderFrom({ + id: 'openai-default', + inferenceExecutor: 'openai', + requiredEnv: { + apiKey: 'OPENAI_API_KEY', + }, + }), + ], + }), + ChatModels({ + models: [ + chatModelFrom({ + aliases: ['assistant-default'], + model: 'gpt-4.1-mini', + provider: 'openai-default', + }), + ], + }), + ], + projects: [ + { + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + }, + ], +}) +``` +::: + +`requiredEnv` 的键是要写入模型参数的名称,值是环境变量名。以上配置会读取 `OPENAI_API_KEY`,并将它作为 `apiKey` 参数加入提供方配置。以下字段名称相近,但用途不同: + +| 值 | 职责 | +| --- | --- | +| `model.provider` | 指向 `ChatProviders` 中注册的提供方 ID。插件处理配置时,会将提供方的执行器和参数合并到模型定义中;模型自身的参数优先。 | +| `model.inferenceExecutor` | 模型定义中的执行器配置。内置聊天模型可以使用 `'openai'`、`'openrouter'`、`'ollama'`,也可以传入兼容的执行器对象。Vieval 核心不会解释这个字段。 | +| `model.inferenceExecutorId` | 标识该模型使用哪类运行配置。执行器是字符串时,`chatModelFrom` 会用它生成此字段;`openaiFromRunContext` 等辅助函数则根据它校验并返回对应的配置结构。模型定义中的 `inferenceExecutorId` 与报告里的同名字段来源不同:报告字段来自当前调度任务的 `context.task.inferenceExecutor.id`。 | +| `projects[].inferenceExecutors` | 项目要调度的执行目标列表。每个发现的评测项都会针对列表中的每一项运行一次;默认值是 `[{ id: 'default' }]`。Vieval 不会根据 `models` 自动生成这个列表。 | +| `context.task.inferenceExecutor` | 当前任务从 `projects[].inferenceExecutors` 中选中的执行目标。它不一定与矩阵所选模型中的执行器配置相同。 | + +评测代码按以下步骤取得模型配置: + +```text +矩阵别名 + -> 已注册的 ModelDefinition + -> 根据 model.inferenceExecutorId 校验运行配置 + -> 评测代码或智能体使用该配置调用远程/本地模型 +``` + +`projects[].inferenceExecutors` 单独控制调度次数。Vieval 不会自动把其中的 ID 与模型的 `inferenceExecutorId` 视为同一个值。只有你的执行代码依赖这种对应关系时,才需要让两处 ID 保持一致。 + +## 在评测代码中查找所选模型 + +`modelFromRun` 读取 `context.task.matrix.run` 中指定轴的值,再根据这个值查找 `context.models`。`modelFromEval` 则读取 `context.task.matrix.eval`,适合查找评分模型或其他只用于评测的模型。 + +::: code-group +```ts [evals/model-selection.eval.ts] +import { describeTask, expect } from 'vieval' +import { modelFromRun, openaiFromRunContext } from 'vieval/plugins/chat-models' + +describeTask('model selection', ({ caseOf }) => { + caseOf('resolves the run model', (context) => { + const selectedModel = modelFromRun(context, { axis: 'model' }) + const runtimeConfig = openaiFromRunContext(selectedModel) + + expect(selectedModel.aliases).toContain('assistant-default') + expect(runtimeConfig.model).toBe('gpt-4.1-mini') + }) +}) +``` +::: + +调用时必须指定轴名。`{ axis: 'model' }` 表示读取 `context.task.matrix.run.model`;如果模型名称位于 `judgeModel` 这样的评测矩阵轴中,则应使用 `modelFromEval(context, { axis: 'judgeModel' })`。 + +`openaiFromRunContext` 会校验模型定义,并返回 `apiKey`、`model` 等 OpenAI 调用参数。它不会创建客户端,也不会调用 OpenAI。任务代码或它调用的智能体还需要把这些参数交给所选的模型调用库或运行时,并显式发起请求。`ChatModels` 同样只负责登记和处理模型配置,不是模型服务客户端。 + +::: warning 费用与数据 +注册和查找模型只会处理本地配置。任务、智能体或评分代码发起远程请求后,会产生调用费用和网络延迟,还可能受到服务方的速率限制,并向外部服务传输数据。启用远程调用前,应检查提示词、输入、输出、遥测数据和报告文件中是否含有敏感信息。 + +如果需要让推理输入完全留在本地,应使用本地模型,并确认执行过程中没有经过外部代理或遥测服务。本地推理仍会占用计算和存储资源,也仍需遵守项目自身的网络和数据管理要求。 +::: + +下一步将在[矩阵与数据集](/zh-hans/guide/learn/matrices-and-datasets)中使用模型别名生成多组评测组合。完整配置见[配置参考](/zh-hans/config/),可导入的辅助函数见 [API 参考](/zh-hans/api/)。 diff --git a/docs/content/zh-hans/guide/learn/reliable-execution.md b/docs/content/zh-hans/guide/learn/reliable-execution.md new file mode 100644 index 0000000..41d0770 --- /dev/null +++ b/docs/content/zh-hans/guide/learn/reliable-execution.md @@ -0,0 +1,183 @@ +--- +title: 可靠执行 +prev: + text: 矩阵与数据集 + link: /zh-hans/guide/learn/matrices-and-datasets +next: + text: 报告与比较 + link: /zh-hans/guide/learn/reports-and-comparisons +--- + +# 可靠执行 + +模型输出可能不稳定,外部服务也可能暂时失败。Vieval 提供自动尝试、重试和超时来处理这些情况,并通过并发限制控制同时执行的任务和用例。对于可以重复使用的数据准备结果,还可以使用任务缓存减少重复计算。 + +## 理解用例的执行顺序 + +一个调度任务可以包含多个用例。启用 `autoAttempt` 后,Vieval 会先运行全部用例,等它们结束后再判断是否需要下一次尝试。只要有失败或超时的用例尚未达到自己的尝试次数上限,Vieval 就会再次运行全部用例。每次运行单个用例时,`autoRetry` 还可以在失败后立即重试该用例。 + +```text +已调度任务 + -> 尝试 0 + -> 用例 A:首次运行 -> 重试 1 -> 重试 2 + -> 用例 B:首次运行 + -> 尝试 1,仅当失败或超时用例仍可再次尝试时运行 + -> 用例 A:首次运行…… + -> 用例 B:首次运行…… +``` + +同一次尝试中的用例可以并发执行,但不同尝试目前会依次执行。后续尝试会重新运行全部用例,包括上一次已经通过的用例。 + +CLI 参数 `--attempt` 只为本次运行设置报告中的 `attemptId`。它不会启用 `autoAttempt`,也不会改变用例的执行次数。 + +## 为不同工作设置并发上限 + +`vieval run` 支持五种并发设置。它们分别控制不同工作,并非每个设置都对应一个独立的并行队列。 + +| 名称 | 控制对象 | 可配置位置 | +| --- | --- | --- | +| 工作区 | 一次 CLI 运行最多同时执行多少个项目;默认上限为 `1`。 | 顶层 `concurrency.workspace`;`--workspace-concurrency` | +| 项目 | 项目执行调度任务时采用的上限;它会与任务上限取较小值。 | 顶层或项目级 `concurrency.project`;`--project-concurrency` | +| 任务 | 一个项目最多同时执行多少个调度任务;默认上限为 `1`。 | 顶层或项目级 `concurrency.task`;`--task-concurrency` | +| 尝试 | 自动尝试目前依次执行;该值会保存在配置中,但暂不改变执行方式。 | 顶层、项目级或任务级 `concurrency.attempt`;`--attempt-concurrency` | +| 用例 | 一个任务最多同时执行多少个用例。单次 `casesFromInputs` 调用可以使用自己的队列和上限。 | 顶层、项目级或任务级 `concurrency.case`;`casesFromInputs(..., { concurrency })`;`--case-concurrency` | + +对于工作区、项目和任务,CLI 参数只能收紧配置中的上限,不能把较小的配置值调高。对于尝试和用例,CLI 参数会覆盖任务级或项目级的值。项目配置不能设置 `workspace`;任务配置只能设置 `attempt` 和 `case`。 + +::: code-group + +```ts [vieval.config.ts] +import { defineConfig } from 'vieval' + +export default defineConfig({ + concurrency: { + workspace: 1, + }, + projects: [ + { + concurrency: { + case: 4, + project: 2, + task: 2, + }, + include: ['evals/*.eval.ts'], + name: 'chat-evals', + root: '.', + }, + ], +}) +``` + +::: + +任务可以单独设置用例并发,某一次 `casesFromInputs` 调用还可以使用更小的上限: + +::: code-group + +```ts [evals/retrieval.eval.ts] +import { describeTask } from 'vieval' + +describeTask('retrieval', ({ casesFromInputs }) => { + casesFromInputs('query', ['alpha', 'beta', 'gamma'], async ({ matrix }) => { + await evaluateQuery(matrix.inputs) + }, { + concurrency: 2, + }) +}, { + concurrency: { + case: 4, + }, +}) +``` + +::: + +这里的任务级上限是 `4`,但这三个输入共享一个上限为 `2` 的队列。传入 `--case-concurrency` 时,运行时值会覆盖项目级、任务级和这一组输入的用例并发。 + +::: warning 提高并发会增加同时发出的请求数 +矩阵、推理执行器注册和并发配置本身不会调用模型服务。如果用例会调用模型或其他计费服务,提高任务或用例并发可能触发限流,并在短时间内产生更多费用。请根据服务方允许的请求速率,以及每个用例实际发出的请求数设置上限。 +::: + +## 选择自动尝试、重试与超时 + +可以在 `describeTask` 上设置整个任务的执行策略,也可以在 `caseOf` 或 `casesFromInputs` 上覆盖某些用例的策略。 + +::: code-group + +```ts [evals/provider-health.eval.ts] +import { caseOf, describeTask, expect } from 'vieval' + +describeTask('provider health', () => { + caseOf('returns a usable answer', async ({ signal }) => { + const answer = await requestAnswer({ signal }) + expect(answer.length).toBeGreaterThan(0) + }, { + autoAttempt: 1, + autoRetry: 2, + input: 'health-check', + timeout: 10_000, + }) +}) +``` + +::: + +`autoRetry: 2` 表示首次运行失败后最多再执行两次。只要其中一次通过,就不会继续重试;这些运行都计入同一次尝试。默认等待时间按重试序号增加:第一次重试前等待 500 毫秒,第二次重试前等待 1,000 毫秒。也可以把 `autoRetryDelay` 设为固定的非负毫秒数,或传入一个根据重试序号返回等待时间的函数。 + +`autoAttempt: 1` 表示首次尝试结束后,最多再运行一次完整用例集。只有某个用例在用完重试次数后仍然失败或超时,并且尚未达到自己的尝试上限,Vieval 才会开始下一次尝试。每次完成的尝试都会参与任务分数计算。例如,同一个用例第一次失败、第二次通过时,最终的 `exact` 平均分为 `0.5`,后一次通过不会覆盖前一次失败。 + +::: tip 根据要回答的问题选择重复方式 +如果要观察多次执行的通过率,请使用 `autoAttempt`,因为每次尝试的结果都会参与任务分数计算。`autoRetry` 更适合应对网络抖动等临时故障:只要某次重试通过,这次尝试就按通过计算,不会把此前的失败计入任务分数。各次运行仍会写入生命周期事件,便于排查问题。 +::: + +`timeout` 会为每次用例运行单独计时。超时后,Vieval 把该次运行标记为 `timeout`,并中止传给用例的 `signal`,随后根据配置重试当前用例或开始下一次尝试。请把这个信号继续传给网络请求、模型调用等下游操作。下游代码如果不响应取消信号,仍可能在 Vieval 停止记录该用例的分数和指标后继续执行,并产生外部副作用。 + +## 缓存可重复使用的数据准备结果 + +任务回调可以通过 `context.cache` 读写文件缓存。Vieval 不会自动缓存函数返回值,因此代码需要明确判断文件是否存在,并负责写入和读取: + +::: code-group + +```ts [evals/dataset.eval.ts] +import { describeTask } from 'vieval' + +describeTask('dataset-backed eval', ({ caseOf }) => { + caseOf('loads prepared cases', async ({ cache, signal }) => { + const file = cache.namespace('dataset-v1').file({ + ext: 'json', + key: ['prepared', 'source-sha256'], + }) + + if (!await file.exists()) { + await file.writeJson(await prepareDataset({ signal })) + } + + const cases = await file.readJson>() + await evaluatePreparedCases(cases) + }) +}) +``` + +::: + +CLI 默认把缓存写到项目根目录下的 `.vieval/cache`。完整路径依次包含工作区 ID、项目名、命名空间、键和扩展名。Vieval 会把这些值转换为安全的路径片段,并以原子方式写入文本、JSON 和二进制文件。 + +缓存不会自动计算输入哈希、设置过期时间或检查内容是否过时。凡是会改变文件内容的版本号、数据源标识或参数,都应写入命名空间或键。例如,数据源变化后,示例中的 `source-sha256` 也应随之变化。 + +只有实际使用的缓存根目录、工作区 ID、项目名、命名空间和键全部相同,多次尝试或后续运行才会读到同一个文件。顶层 `vieval compare` 会把比较配置中的 `benchmark.sharedCaseNamespace` 用作各方案共同的缓存项目名。不过,如果方案位于不同的项目根目录,它们仍会写入各自的 `.vieval/cache`;仅统一缓存项目名并不能让两个物理目录复用同一个文件。 + +## 区分任务汇总与报告文件 + +任务分数和报告文件保留的信息不同: + +- 失败或超时的用例会为任务增加一个值为 `0` 的 `exact` 分数;没有自定义分数的通过用例会增加 `1`。 +- 通过用例发出的 `exact` 和 `judge` 分数会参与任务计算;失败用例在运行中发出的自定义分数不会参与任务计算。 +- 每次完成的自动尝试都会再向任务分数加入一组用例结果。 +- 执行过程会产生生命周期、分数和指标事件。使用 `--report-out` 时,这些事件按顺序写入 `events.jsonl`。 +- `cases.jsonl` 最终为每个「任务 ID + 用例 ID」组合写一条记录,多次重试或自动尝试不会分别占一行。要查看执行顺序,请读取 `events.jsonl`;要查看包含所有尝试结果的任务分数,请读取 `run-summary.json`。 + +正如[断言、分数与指标](/zh-hans/guide/learn/assertions-scores-and-metrics)所述,用例在失败前发出的分数事件可能仍会出现在报告记录中,但任务分数会把该用例计为失败。排查问题时,请同时检查用例状态和分数。 + +不要使用 `autoRetry` 统计多次执行的通过率:重试后恢复的用例在任务分数中只算一次通过。如果希望早期失败也参与最终分数,请使用 `autoAttempt`,并注意它会重新运行已经通过的用例。`--attempt` 和 `--attempt-concurrency` 都不会增加自动尝试次数。 + +下一步可在[报告与比较](/zh-hans/guide/learn/reports-and-comparisons)中保存运行记录,并检查或比较结果。 diff --git a/docs/content/zh-hans/guide/learn/reports-and-comparisons.md b/docs/content/zh-hans/guide/learn/reports-and-comparisons.md new file mode 100644 index 0000000..ae588f1 --- /dev/null +++ b/docs/content/zh-hans/guide/learn/reports-and-comparisons.md @@ -0,0 +1,194 @@ +--- +title: 报告与比较 +prev: + text: 可靠执行 + link: /zh-hans/guide/learn/reliable-execution +--- + +# 报告与比较 + +CLI 默认在终端中显示本次运行的摘要。如果需要在运行结束后检查用例、汇总多次运行,或比较基线与候选方案,请把报告保存到文件。 + +## 保存一次运行的报告 + +`vieval run` 默认向标准输出写入便于阅读的摘要。`--json` 会把标准输出改为 JSON,但不会创建报告文件。要保存报告目录,请使用 `--report-out`。 + +::: code-group + +```bash [终端] +vieval run \ + --config ./vieval.config.ts \ + --workspace local \ + --experiment baseline \ + --attempt attempt-a \ + --report-out .vieval/reports +``` + +::: + +上面的命令仍会显示可读摘要,并把报告写入 `.vieval/reports` 下的本次运行目录。如果脚本需要从标准输出读取运行结果和最终的 `reportDirectory`,可以再添加 `--json`。可读摘要不会显示报告目录路径。 + +未使用 `--report-out` 时,JSON 结果中的 `reportDirectory` 为 `null`,后续的 `vieval report ...` 命令也无法读取这次运行的数据。 + +## 了解一次运行写入的内容 + +Vieval 在报告根目录下按工作区、项目、实验、尝试和自动生成的运行 ID 保存每次运行。如果一次运行包含多个项目,项目目录名会使用 `multi-project`。 + +| 文件 | 内容 | +| --- | --- | +| `run-summary.json` | 完整的 CLI 运行结果,包括各类 ID、项目状态、用例计数与失败信息、矩阵摘要和汇总分数。 | +| `events.jsonl` | 按发生顺序记录运行、任务、用例、分数、指标和自定义事件。 | +| `cases.jsonl` | 每个「任务 ID + 用例 ID」组合的最终记录,包括状态、耗时、根据事件推算的重试次数、分数、指标以及可选的输入和输出。 | +| `metrics-summary.json` | 从 `cases.jsonl` 计算出的分数数量、总和与平均值。 | +| `otlp/traces.json`、`otlp/logs.json`、`otlp/metrics.json` | 从用例记录转换得到的本地 OTLP 数据。 | + +`vieval report index` 默认会创建 `index/runs.jsonl`,也可以通过 `--output` 指定其他位置;`vieval run` 本身不会写入这个索引文件。 + +使用自动尝试时,应根据问题选择文件:`run-summary.json` 中的任务分数包含所有已完成尝试的结果;`cases.jsonl` 为每个「任务 ID + 用例 ID」组合只保留最终记录;`events.jsonl` 则保留重试和尝试的执行顺序。 + +`cases.jsonl` 中的 `retryCount` 根据用例开始事件计算。如果事件没有 `retryIndex`,后续自动尝试产生的重复开始事件也可能让这个数字增加。要准确区分重试和自动尝试,请直接查看 `events.jsonl`。 + +::: warning 报告文件可能包含敏感信息 +报告可能包含用例输入和输出、错误信息、自定义指标、模型或评测基准标识符,以及其他事件数据。请设置合适的保留期限和访问权限,并删除或脱敏不应离开评测环境的字段。 +::: + +## 索引、检查与分析报告 + +每个报告命令都需要一个报告路径。这个路径通常既可以是单次运行目录,也可以是包含多次运行的上层目录。 + +::: code-group + +```bash [终端 — 建立运行索引] +vieval report index <报告目录> +``` + +::: + +`index` 会递归查找 `run-summary.json`,并把每次运行的摘要写入 `<报告目录>/index/runs.jsonl`。可以用 `--output` 修改索引文件路径。`--format table|json|jsonl` 只改变标准输出的格式,索引文件始终使用 JSONL。 + +::: code-group + +```bash [终端 — 检查用例记录] +vieval report cases <报告目录> +``` + +::: + +`cases` 读取 `cases.jsonl`。可以多次传入 `--where key=value` 筛选记录,用 `--group-by ` 按字段汇总分数,并通过 `--format table|json|jsonl` 选择输出格式。命令只向标准输出写入结果,不会生成新的用例文件。 + +::: code-group + +```bash [终端 — 分析运行] +vieval report analyze <报告目录> +``` + +::: + +`analyze` 读取运行摘要和事件。它可以按工作区、项目、实验、尝试、运行、事件、错误文本,以及运行或评测矩阵的选中值筛选,再按工作区和实验汇总结果。`--task-state` 和 `--case-state` 目前接受 `passed`、`failed` 或 `skipped`;`--case-state` 暂不接受 `timeout`。`--format table|json|jsonl|csv` 用于选择标准输出格式。 + +## 比较两组已有报告 + +比较命令的第一个目录是基线,第二个目录是候选方案: + +::: code-group + +```bash [终端] +vieval report compare <基线报告目录> <候选方案报告目录> +``` + +::: + +命令按以下规则计算结果: + +1. **先对齐用例。** 命令从基线和候选方案中读取 `cases.jsonl`,根据用例键找到同一个用例。候选方案独有的记录记为新增,基线独有的记录记为移除。 +2. **再计算匹配项。** 单个用例和分组的差值都按「候选方案减基线」计算,并且只使用已经对齐的记录。使用 `--group-by` 时,分组值取自候选方案。 +3. **处理缺失分数。** 已对齐的用例如果在某份报告中缺少 `--score-kind` 指定的分数,对应分数按 `0` 计算。`--score-kind` 默认为 `exact`。 +4. **单独计算总体差值。** 命令分别对基线和候选方案中所有包含该分数的记录求平均值,再用候选平均值减去基线平均值。新增和移除的用例会影响总体差值;缺少所选分数的记录不会进入这一步。因此,总体差值不一定等于匹配用例差值的平均值。 + +输出格式支持 `table` 和 `json`。 + +为了对齐用例,命令默认依次尝试指标 `benchmark.case.id`、指标 `vieval.case.id`,最后使用记录中的 `caseId`。如果评测基准有其他稳定标识符,可以传入 `--case-key `。显式指定的键必须存在于每条记录中;基线或候选方案中出现重复键时,命令会报错,不会自行选择其中一条记录。 + +`--case-key`、`--group-by` 和 `report cases` 的字段选择规则相同:先查找同名指标,再查找 `scores.` 或分数名称,最后查找用例记录的直接字段。例如,`benchmark.category` 可以引用指标,`state` 可以引用用例状态字段。 + +不要把该命令与顶层 `vieval compare` 混淆: + +- `vieval report compare <基线报告目录> <候选方案报告目录>` 读取并比较已经保存的用例报告。 +- `vieval compare --config ... --comparison ...` 读取比较模式配置,针对同一个评测基准运行所有已配置方案。Vieval 会把 `benchmark.sharedCaseNamespace` 用作各方案共同的缓存项目名(内部的 `cacheProjectName`),并可通过 `--output` 保存汇总后的比较结果。 + +## 完成一次完整比较流程 + +下面的流程使用两份配置分别运行基线和候选方案,再对它们共同的报告根目录建立索引。请确保两份配置会发现同一组评测用例,并分别指向需要比较的模型、提示词或被测项目版本。`--experiment` 只为报告设置实验标识,不会切换模型、提示词或被测代码。 + +报告命令会递归查找运行目录,因此不需要手动填写运行 ID。 + +::: code-group + +```bash [终端 — 运行并保存基线报告] +vieval run \ + --config ./vieval.baseline.config.ts \ + --workspace local \ + --experiment baseline \ + --attempt attempt-a \ + --report-out .vieval/reports/baseline +``` + +::: + +::: code-group + +```bash [终端 — 运行并保存候选方案报告] +vieval run \ + --config ./vieval.candidate.config.ts \ + --workspace local \ + --experiment candidate \ + --attempt attempt-a \ + --report-out .vieval/reports/candidate +``` + +::: + +::: code-group + +```bash [终端 — 为所有报告建立索引] +vieval report index .vieval/reports +``` + +::: + +::: code-group + +```bash [终端 — 检查候选用例] +vieval report cases .vieval/reports/candidate \ + --where state=failed \ + --format jsonl +``` + +::: + +::: code-group + +```bash [终端 — 汇总运行结果] +vieval report analyze .vieval/reports --format json +``` + +::: + +::: code-group + +```bash [终端 — 比较已对齐的用例分数] +vieval report compare \ + .vieval/reports/baseline \ + .vieval/reports/candidate \ + --case-key benchmark.case.id \ + --score-kind exact \ + --format table +``` + +::: + +最后一条命令要求每个用例都有唯一的 `benchmark.case.id`。如果无法保证这一点,并且默认标识符可以稳定对应同一个用例,请省略 `--case-key`。 + +常见错误包括:把 `--json` 当作保存报告的选项;试图用报告命令读取没有通过 `--report-out` 保存的运行;或使用自动生成、可能重复的标识符对齐用例。只有当所选键在基线和候选方案中都表示同一个评测基准用例时,比较结果才有意义。 + +可以在[API](/zh-hans/api/)中查看公开入口,在[配置](/zh-hans/config/)中查看当前配置接口。 diff --git a/docs/content/zh-hans/guide/learn/tasks-cases-and-inputs.md b/docs/content/zh-hans/guide/learn/tasks-cases-and-inputs.md new file mode 100644 index 0000000..99fde72 --- /dev/null +++ b/docs/content/zh-hans/guide/learn/tasks-cases-and-inputs.md @@ -0,0 +1,94 @@ +--- +title: 任务、用例与输入 +prev: + text: 核心概念 + link: /zh-hans/guide/core-concepts +next: + text: 断言、分数与指标 + link: /zh-hans/guide/learn/assertions-scores-and-metrics +--- + +# 任务、用例与输入 + +本节介绍如何用任务组织一组可重复运行的用例。读完后,你将知道如何为单个用例传入数据、如何从数据集批量生成用例,以及如何让不同报告准确匹配同一条数据。 + +## 创建任务 + +`describeTask` 用来组织评测同一项功能的用例。Vieval 加载评测文件时,会注册其中声明的任务。任务名称应直接说明要评测什么,并且不应随运行矩阵或评测矩阵的取值变化。 + +使用 `caseOf` 可以声明一个有名称的用例: + +::: code-group +```ts [evals/addition.eval.ts] +import { describeTask, expect } from 'vieval' + +describeTask('addition', ({ caseOf }) => { + caseOf('adds two positive numbers', ({ matrix }) => { + const { a, b, expected } = matrix.inputs + expect(a + b).toBe(expected) + }, { + input: { a: 20, b: 22, expected: 42 }, + }) +}) +``` +::: + +`caseOf` 的第一个参数是显示在报告中的用例名称。传入 `input` 选项后,TypeScript 会根据它推导回调中 `matrix.inputs` 的类型。 + +## 从数据集生成用例 + +`casesFromInputs` 会为只读数组中的每个元素创建一个用例。回调每次收到一个完整的数组元素,因此可以把待测数据和用于匹配报告的标识放在同一个对象中: + +::: code-group +```ts [evals/addition-dataset.eval.ts] +import { describeTask, expect } from 'vieval' + +const inputs = [ + { input: { a: 1, b: 2, expected: 3 }, name: 'addition-small' }, + { input: { a: 20, b: 22, expected: 42 }, name: 'addition-large' }, +] + +describeTask('addition', ({ casesFromInputs }) => { + casesFromInputs('addition', inputs, ({ matrix, metric }) => { + const { a, b, expected } = matrix.inputs.input + + metric('benchmark.case.id', matrix.inputs.name) + expect(a + b).toBe(expected) + }) +}) +``` +::: + +这段代码会创建 `addition #1` 和 `addition #2` 两个用例。输入对象中的 `name` 只是普通字段,`casesFromInputs` 不会把它当作用例名称。示例将该字段记录为 `benchmark.case.id`;比较两份报告时,Vieval 会优先用这个指标匹配同一条数据。 + +## 读取用例输入 + +用例回调中的 `matrix` 同时包含当前任务的矩阵取值和当前用例的输入: + +```text +当前任务的矩阵取值 + -> context.task.matrix.run / eval / meta + +caseOf(..., { input }) 或 casesFromInputs(...) + -> context.matrix.inputs +``` + +在上面的数据集示例中,`matrix.inputs` 对应 `inputs` 数组中的当前元素。因此,算术数据位于 `matrix.inputs.input`,数据标识位于 `matrix.inputs.name`。 + +`context.task.matrix` 只保存当前任务在调度时选中的运行矩阵值、评测矩阵值和元数据。Vieval 只把用例输入添加到 `context.matrix.inputs`,因此不要从 `context.task.matrix.inputs` 读取它。 + +## 选择稳定的名称与标识 + +你可以直接指定 `caseOf` 的名称。同一个用例应始终使用同一名称,因为报告中的 `caseId` 由用例位置和名称生成。 + +`casesFromInputs` 按前缀和从 1 开始的序号生成名称:` #1`、` #2`,依此类推。插入数据或调整顺序都会改变自动生成的名称和 ID。如果数据顺序可能变化,而你仍要比较不同报告中的同一条数据,请为每条输入添加唯一标识,并像上例一样将它记录为 `benchmark.case.id`。 + +在参与比较的每一份报告中,`benchmark.case.id` 都不能重复,即使这些用例来自不同任务或项目。如果出现重复值,比较命令会报错,而不会自行猜测应该如何配对。 + +报告选择器可以读取 `caseName`、`caseId` 等用例字段,也可以读取分数和已记录的指标。比较用例时,Vieval 默认依次尝试 `benchmark.case.id`、`vieval.case.id` 和自动生成的 `caseId`。 + +::: warning 在任务或评测中注册用例 +`caseOf` 和 `casesFromInputs` 必须在 `describeTask` 或 `describeEval` 的回调中调用。在其他位置调用时,Vieval 会在加载评测文件时抛出错误。 +::: + +定义好任务和输入后,下一步是在[断言、分数与指标](/zh-hans/guide/learn/assertions-scores-and-metrics)中记录每个用例的判断结果。完整的导入路径见 [API 概览](/zh-hans/api/)。 diff --git a/docs/content/zh-hans/guide/why.md b/docs/content/zh-hans/guide/why.md new file mode 100644 index 0000000..6852ec2 --- /dev/null +++ b/docs/content/zh-hans/guide/why.md @@ -0,0 +1,34 @@ +--- +title: 为什么选择 Vieval +prev: false +next: + text: 快速开始 + link: /zh-hans/guide/getting-started +--- + +# 为什么选择 Vieval + +把业务代码与评测代码纳入同一套工程流程,开发者修改业务行为时,就能同步评审和更新相关评测,并用版本控制记录两者的变化。Vieval 为 TypeScript 工程提供这套评测工作流。 + +## 重复运行一项评测需要哪些步骤 + +单独调用一次评测函数,只能得到当次结果。要在不同时间或不同配置下重复评测,还需要确定要加载哪些文件、怎样组合变量、按什么顺序调度任务,以及执行后保存哪些用例结果和事件记录。 + +Vieval 的运行器(runner)负责发现评测文件、组合矩阵变量和调度任务。执行期间,它汇总断言结果、分数与指标;传入 `--report-out` 时,它还会写入后续分析和比较所需的文件。评测任务仍由普通 TypeScript 代码定义,同一个任务可以重复执行,也可以在多组模型和场景配置下执行。 + +## 按用途选择输出格式 + +评测运行时,CLI 默认显示便于阅读的进度和摘要。传入 `--json` 后,标准输出会改为机器可读的结果;传入 `--report-out ` 后,Vieval 会写入可供分析、比较或自动化脚本读取的报告目录。无论选择哪种输出,Vieval 都会使用同一套发现、调度和执行流程。 + +## 适合的场景 + +以下需求适合使用 Vieval: + +- 需要在 TypeScript 仓库中编写评测,并与业务改动一起评审; +- 同一个任务需要按模型、场景、评分标准或数据集变量执行; +- 本地运行与自动化脚本需要采用相同的任务发现和调度规则; +- 用例的断言结果、分数、指标与事件记录需要写入报告目录,供后续检查。 + +## 不适合的场景 + +Vieval 不负责托管数据集,也不提供数据标注界面或 SaaS 可观测性服务。评测代码可以读取外部数据或调用远程模型;数据集托管、标注和可观测性需要交给相应的专用服务。 diff --git a/docs/content/zh-hans/index.md b/docs/content/zh-hans/index.md index deb2efb..e6f9264 100644 --- a/docs/content/zh-hans/index.md +++ b/docs/content/zh-hans/index.md @@ -1,13 +1,13 @@ --- title: Vieval -titleTemplate: 面向 Agent 的评测框架 +titleTemplate: 面向智能体的评测框架 layout: home theme: dark home: logoAlt: Vieval 图标 eyebrow: 评测框架 - heroTitle: 对智能体和模型进行可靠评测 - heroDescription: 智能体是概率性的,输出都不稳定,Vieval 让你可以自信地进行测试和基准评测,Vieval = Vitest + 评测框架。 + heroTitle: 可靠地评测智能体与模型 + heroDescription: Vieval 用熟悉的测试写法组织可重复执行的评测,并汇总用例结果与分数,生成报告。可以把它理解为 Vitest 式开发体验与评测运行器的结合。 primaryAction: text: 快速开始 link: /zh-hans/guide/ @@ -25,29 +25,28 @@ home: 4 files, 6 entries, 24 runs cases 198 passed | 3 failed | 0 timeout matrix run 4 [model|scenario] / eval 2 [rubric] - report .vieval/reports/local/baseline/attempt-a/... why: - eyebrow: Vieval 的意义是什么? - title: 用更可靠的方式衡量概率结果。 + eyebrow: 为什么使用 Vieval? + title: 通过反复运行评测,衡量不稳定的行为。 body: - - Vieval 让评测用例保持可读,同时为多模型并行矩阵调度、分数评判、跨指标横评、自定义断言和报告提供稳定的基础设施。 - - 原生支持 OpenAI 的 对话补全(Chat Completions),也可以利用在 YOLO,VLM,Robotics 相关领域的评测用例。 + - 用 TypeScript 编写评测用例,再由 Vieval 发现文件、组合矩阵变量、调度任务,并汇总断言结果、分数和指标。 + - 模型注册与模型调用彼此分离。内置聊天模型插件负责管理 OpenAI、OpenRouter 和 Ollama 等运行配置;任务代码决定是否以及何时调用模型。 action: text: 阅读指南 link: /zh-hans/guide/ features: - title: 测试风格的评测文件 - details: 使用 describeTask、caseOf、casesFromInputs 和 expect,让评测用例贴近它们验证的产品代码。 - - title: 原生数据集用例 - details: casesFromInputs 可以接入 HuggingFace datasets、S3 objects、本地 fixtures 或动态生成的输入。 - - title: 从上到下的矩阵能力 - details: 通过 workspace、project、eval 和 task 的矩阵参数,对比一个或多个实现。 - - title: 内置 Rubric 评估 - details: 当 agent 行为无法直接断言时,可以请另一个模型或 agent 根据 rubric 打分。 - - title: Chat models、Voice models 等都可扩展 - details: ChatModels 只是一个插件:第一天先定义聊天提供方,之后可用同一插件接口扩展新的运行时。 - - title: 同时适合 LLM 和人类阅读的报告 - details: 内置 CLI 工具帮助 agents 分析和检查运行结果,让 coding agents 能优化 agent 或模型运行,同时保持可读的人类 DX。 + details: 使用 describeTask、caseOf、casesFromInputs 和 expect,把任务、输入与断言写在同一份 TypeScript 文件中。 + - title: 从输入数组批量生成用例 + details: 先用代码加载或构造输入数组,再交给 casesFromInputs 为每个元素注册一个用例。 + - title: 分层配置矩阵 + details: 在项目、评测项和任务三个层级组合模型、场景与评分方式等变量,并对每组组合执行同一个任务。 + - title: 内置评分标准断言 + details: 当行为无法用确定性断言判断时,可以让评分标准(rubric)的评判回调返回评分理由和分数;是否调用另一个模型由评测代码决定。 + - title: 可扩展的模型注册 + details: 内置 ChatModels 插件管理聊天模型元数据;也可以编写项目插件,向配置中注册其他模型定义。 + - title: 面向开发者和工具的输出 + details: CLI 默认显示便于阅读的进度与摘要,也可输出 JSON,或将运行摘要、事件和用例记录写入报告目录。 ---