diff --git a/.env.example b/.env.example index 5f5439ae..2a4b2c1c 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,8 @@ # POSTGRES_BACKEND_PORT=5436 # Second Postgres (object storage): compose default host port is 5434 (see docker-compose.yml). Override if that port is busy; object-storage still connects to postgres-object-storage:5432 inside Compose: # POSTGRES_OBJECT_STORAGE_PORT=5436 +# MLTools Postgres: compose default host port is 5436. +# POSTGRES_MLTOOLS_PORT=5437 # Redis: compose default host port is 6380 (see docker-compose.yml). Override if busy; scalars still uses redis://redis:6379/0 inside Compose: # REDIS_PORT=6381 # CLICKHOUSE_HTTP_PORT=8123 @@ -17,6 +19,7 @@ # SCALARS_PORT=8001 # BACKEND_PORT=8000 # WEB_PORT=3000 +# MLTOOLS_PORT=8003 # --- Web container (server-side BFF → backend). Browser still uses NEXT_PUBLIC_BASE_URL (build arg). --- # When `web` runs in this Compose file, keep the default so Route Handlers reach `backend` over the Docker network: @@ -32,6 +35,7 @@ # --- Backend / security (override compose defaults in production) --- # JWT_SECRET=change-me-in-docker-env # ADMIN_PANEL_KEY=admin +# MLTOOLS_BACKEND_API_TOKEN=pat_token_with_experiments.view_and_metrics.view # --- CORS (comma-separated origins: scheme + host + port; must match the UI as the browser sends `Origin`) --- # Local: diff --git a/AGENTS.md b/AGENTS.md index 294bc86b..d813579a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,11 +12,15 @@ flowchart LR API["python/backend\n(FastAPI + Postgres)"] Scalars["python/scalars_service\n(FastAPI + ClickHouse)"] Blobs["python/object_storage\n(FastAPI + MinIO/S3)"] + MLTools["python/mltools\n(FastAPI + Celery + Postgres)"] SDK["python/sdk\n(client library)"] Web -->|"HTTP / BFF routes"| API API --> Scalars API --> Blobs + API --> MLTools + MLTools --> API + MLTools --> Blobs SDK --> API ``` @@ -24,6 +28,7 @@ flowchart LR - **Backend (`python/backend`)**: Primary API (`api.main:app`), users/teams/RBAC, projects, experiments, hypotheses, metrics orchestration. Calls **scalars_service** and **object_storage** via HTTP clients in `src/clients/`. - **Scalars service (`python/scalars_service`)**: Stores scalar runs, tags, **artifacts_info** tables (per-project), and related query APIs. Backed by **ClickHouse** (and supporting infra as configured in that package). - **Object storage (`python/object_storage`)**: Upload/download/delete for experiment and project blobs; uses **MinIO** or **S3** and metadata in Postgres. +- **MLTools (`python/mltools`)**: Asynchronous ML analysis jobs. The first bounded context is hyperparameter importance analysis; it reads scoped project data through the backend and stores model artifacts in S3-compatible storage. - **SDK (`python/sdk`)**: `experiment_tracker_sdk` — typed HTTP client used by training jobs and tools to talk to the backend API. - **Shared (`python/shared`)**: Shared Python types/utilities consumed by other Python packages where applicable. @@ -36,6 +41,7 @@ flowchart LR | `python/backend/src/` | FastAPI app: `api/` routes, `domain/*` bounded contexts, `clients/*` HTTP clients, `db/`, `lib/`. | | `python/scalars_service/src/` | FastAPI scalars/artifacts_info service. `GET /scalars/get/...` paginates **experiments** first, then loads each metric column with ClickHouse `IS NOT NULL` + per-(experiment, column) uniform `max_points` sampling (`columns_per_query` controls parallel column queries; default 1). Cross-table ClickHouse work (delete experiment rows across scalars + artifacts_info + last_logged, usage, admin table listing) is under **`/projects`** (`projects` domain); compaction stays **`POST /scalars/projects/{id}/compact-columns`**. | | `python/object_storage/src/` | FastAPI storage service (buckets, experiment/project artifacts). | +| `python/mltools/src/mltools/` | FastAPI/Celery ML analysis service. Domain logic is grouped under `domain/hparam_importance/`; outbound adapters live under `clients/`; worker composition lives under `workers/`. | | `python/sdk/src/experiment_tracker_sdk/` | Public Python SDK for the tracker API. | | `python/sdk/src/experiment_tracker_sdk/api_access.py` | Singleton :class:`ExpTrackerApiAccess` — shared ``APIRequestsRegistry`` / :class:`ExperimentTrackerClient` construction (used by :class:`ExpTracker` and CLI). | | `python/sdk/src/experiment_tracker_sdk/constants.py` | Default API base URL and ``/api`` prefix literals shared with settings. | @@ -122,6 +128,7 @@ Work from the package root, for example: - `cd python/backend && uv run uvicorn api.main:app --reload --port 8000` - `cd python/scalars_service && uv run pytest` - `cd python/object_storage && uv run pytest` +- `cd python/mltools && uv run pytest` - `cd python/sdk && uv run pytest` Do **not** assume a single global `python/backend`-only layout; **scalars_service**, **object_storage**, and **sdk** are first-class packages with their own `uv` workflows. @@ -189,4 +196,4 @@ Prefer updating this file or code comments when changing global runbooks; avoid For **changing how in-app docs render** (remark/rehype directives, sanitize allowlist, `DocsMarkdown` components), follow and keep in sync **`apps/web/content/docs/contributing/extending-doc-pipeline.md`** (published at `/docs/contributing/extending-doc-pipeline`). -Don't fight bugs! Every time you encounter the same error by accident, research the web and find 3-5 possible ways to fix it. Then choose the most effective solution and implement it. \ No newline at end of file +Don't fight bugs! Every time you encounter the same error by accident, research the web and find 3-5 possible ways to fix it. Then choose the most effective solution and implement it. diff --git a/apps/web/content/docs/domains/experiments.md b/apps/web/content/docs/domains/experiments.md index 1ec541d6..70d9bce6 100644 --- a/apps/web/content/docs/domains/experiments.md +++ b/apps/web/content/docs/domains/experiments.md @@ -46,9 +46,9 @@ tracker.progress(100) Experiment colors make runs visually distinct. They are especially important on scalar plots, where multiple experiments may share the same chart. Colors must be hex strings such as `#3366cc` or `#3366ccff`. -## Features and parent diffs +## Features, hyperparameters, and parent diffs -Features are a tree of named nodes. Today they are the closest equivalent to hyperparameters and run configuration in the product. +Features are a tree of named nodes that describe semantic experiment changes. ```python tracker.features([ @@ -66,7 +66,13 @@ tracker.features([ ]) ``` -When an experiment has a parent, the sidebar can show differences from the parent experiment and let users edit the feature tree. Use this for "what changed in this run?" information until a dedicated hyperparameter domain exists. +When an experiment has a parent, the sidebar can show differences from the parent +experiment and let users edit the feature tree. Use this for "what changed in this +run?" information. + +Hyperparameters are stored separately as nested JSON and can be logged through +`tracker.log_hparams(...)` or edited from the experiment sidebar. The Compare page +shows baseline-relative added, removed, and changed hyperparameter values. ## Parent experiments and DAG diff --git a/apps/web/content/docs/sdk/experiment-logging.md b/apps/web/content/docs/sdk/experiment-logging.md index b442e233..376bb610 100644 --- a/apps/web/content/docs/sdk/experiment-logging.md +++ b/apps/web/content/docs/sdk/experiment-logging.md @@ -184,10 +184,18 @@ tracker.features([ {"name": "data", "children": [{"name": "mnist"}]}, {"name": "model", "children": [{"name": "small-cnn"}]}, ]) +tracker.log_hparams({ + "optimizer": {"name": "adamw", "lr": 3e-4}, + "training": {"batch_size": 32, "epochs": 50}, +}) ``` `progress` accepts `0..100` integers or `0..1` floats. `parent_experiment` resolves by name or id inside the current project. +`features` describe semantic changes and research ideas. `log_hparams` stores the +configurable training values used by the run. Each `log_hparams` call fully replaces +the experiment's previous hyperparameter document. + ## Run the repository example ```bash @@ -572,12 +580,36 @@ tracker.features([ ]) ``` -Updates the experiment feature tree. Features are currently the best place to store hyperparameter-like run structure and "what changed" information. +Updates the experiment feature tree. Use features for semantic changes, ablations, +new mechanisms, and "what changed" information rather than configurable training +values. When an experiment has a parent, the sidebar can show feature differences from that parent. --- +### `log_hparams(...)` + +```python +tracker.log_hparams({ + "optimizer": { + "name": "adamw", + "lr": 0.001, + }, + "training": { + "batch_size": 64, + "seed": 42, + }, +}) +``` + +Validates and stores a nested hyperparameter JSON object. Repeated calls fully replace +the previous document; they do not deep-merge it. Common values such as `Path`, `Enum`, +`date`, `datetime`, and NumPy scalar values are converted when safe. Unsupported values +raise `HparamsSerializationError` with the failing parameter path. + +--- + ### `name(...)` ```python diff --git a/apps/web/package.json b/apps/web/package.json index 81a916e6..aae5e90e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "web", - "version": "0.11.8", + "version": "0.12.1", "private": true, "scripts": { "dev": "next dev", diff --git a/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx b/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx index b6918c39..1dc56a18 100644 --- a/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx +++ b/apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx @@ -1,7 +1,7 @@ "use client"; import { useParams } from "next/navigation"; -import { CompareShell } from "@/domain/compare/components/compare-shell"; +import { CompareShell } from "@/domain/compare/components"; export default function ComparePage() { const { projectId } = useParams<{ projectId: string }>(); diff --git a/apps/web/src/components/shared/experiment-diff-ui.tsx b/apps/web/src/components/shared/experiment-diff-ui.tsx new file mode 100644 index 00000000..216c1c27 --- /dev/null +++ b/apps/web/src/components/shared/experiment-diff-ui.tsx @@ -0,0 +1,61 @@ +import type { ReactNode } from "react"; +import { CircleMinus, CirclePlus, PencilLine } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/utils"; + +export type ExperimentDiffStatus = "unchanged" | "added" | "removed" | "changed"; + +export function ExperimentDiffCountBadge({ + status, + label, + value, + compact = false, +}: { + status: Exclude; + label: string; + value: number; + compact?: boolean; +}) { + const marker = status === "added" ? "+" : status === "removed" ? "-" : "~"; + return ( + + {compact ? `${marker}${value}` : `${label} ${value}`} + + ); +} + +export function ExperimentDiffIcon({ + status, + title, +}: { + status: ExperimentDiffStatus; + title?: string; +}) { + const iconClassName = "h-3.5 w-3.5"; + let icon: ReactNode = null; + if (status === "added") { + icon = ; + } else if (status === "removed") { + icon = ; + } else if (status === "changed") { + icon = ; + } + return icon ? ( + + {icon} + + ) : null; +} + +export function experimentDiffSurfaceClass(status: ExperimentDiffStatus): string { + if (status === "added") return "bg-green-500/10 text-green-800 dark:text-green-300"; + if (status === "removed") return "bg-red-500/10 text-red-800 dark:text-red-300"; + if (status === "changed") return "bg-amber-500/10 text-amber-800 dark:text-amber-300"; + return "text-foreground/80"; +} + +function experimentDiffBadgeClass(status: Exclude): string { + if (status === "added") return "border-green-500/20 bg-green-500/10 text-green-700"; + if (status === "removed") return "border-red-500/20 bg-red-500/10 text-red-700"; + return "border-amber-500/20 bg-amber-500/10 text-amber-700"; +} diff --git a/apps/web/src/components/shared/experiment-features-panel.tsx b/apps/web/src/components/shared/experiment-features-panel.tsx index 9f4c2fc4..cd7927d1 100644 --- a/apps/web/src/components/shared/experiment-features-panel.tsx +++ b/apps/web/src/components/shared/experiment-features-panel.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { ArrowLeftRight, ArrowRight, CircleMinus, CirclePlus, GitCompare, Maximize2, PencilLine, Save } from "lucide-react"; +import { ArrowLeftRight, ArrowRight, GitCompare, Maximize2, PencilLine, Save } from "lucide-react"; import type { Experiment } from "@/domain/experiments/types"; import { experimentsService } from "@/domain/experiments/services"; import { QUERY_KEYS } from "@/lib/constants/query-keys"; @@ -15,7 +15,6 @@ import { type FeatureNode, } from "@/lib/features/feature-tree"; import { Button } from "@/components/ui/button"; -import { Badge } from "@/components/ui/badge"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Dialog, @@ -34,6 +33,11 @@ import { } from "@/components/ui/select"; import { FeatureBulletEditor } from "@/components/shared/feature-bullet-editor"; import { FeatureEditorLabelWithHelp } from "@/components/shared/feature-editor-help"; +import { + ExperimentDiffCountBadge, + ExperimentDiffIcon, + experimentDiffSurfaceClass, +} from "@/components/shared/experiment-diff-ui"; type ExperimentFeaturesPanelProps = { experiment: Experiment; @@ -127,9 +131,9 @@ export function ExperimentFeaturesPanel({ {parentExperiment && showDiffs ? (
- - - + + +
) : null} - {label} {value} - - ); -} - function FeatureView({ childFeatures, parentExperimentExists, @@ -240,9 +228,9 @@ function FeatureExpandedModal({ {showDiffs ? (
- - - + + +
) : null} @@ -330,7 +318,7 @@ function FeatureChangeRow({ row }: { row: FlatDiffLine }) { if ((row.status === "renamed" || row.status === "changed") && row.parentName && row.childName) { return ( } + icon={} previousName={row.parentName} name={row.childName} depth={row.depth} @@ -340,15 +328,10 @@ function FeatureChangeRow({ row }: { row: FlatDiffLine }) { return ( } + icon={} name={displayName} depth={row.depth} - className={cn( - row.status === "added" && "bg-green-500/10 text-green-800 dark:text-green-300", - row.status === "removed" && "bg-red-500/10 text-red-800 dark:text-red-300", - (row.status === "renamed" || row.status === "changed") && - "bg-amber-500/10 text-amber-800 dark:text-amber-300" - )} + className={experimentDiffSurfaceClass(row.status === "renamed" ? "changed" : row.status)} /> ); } @@ -410,30 +393,6 @@ function FeatureNodeDot() { return