Added MLtools to predicts hparams importance

.env.example

@@ -8,6 +8,8 @@
 # POSTGRES_BACKEND_PORT=5436
 # Second Postgres (object storage): compose default host port is 5434 (see docker-compose.yml). Override if that port is busy; object-storage still connects to postgres-object-storage:5432 inside Compose:
 # POSTGRES_OBJECT_STORAGE_PORT=5436
+# MLTools Postgres: compose default host port is 5436.
+# POSTGRES_MLTOOLS_PORT=5437
 # Redis: compose default host port is 6380 (see docker-compose.yml). Override if busy; scalars still uses redis://redis:6379/0 inside Compose:
 # REDIS_PORT=6381
 # CLICKHOUSE_HTTP_PORT=8123
@@ -17,6 +19,7 @@
 # SCALARS_PORT=8001
 # BACKEND_PORT=8000
 # WEB_PORT=3000
+# MLTOOLS_PORT=8003
 
 # --- Web container (server-side BFF → backend). Browser still uses NEXT_PUBLIC_BASE_URL (build arg). ---
 # When `web` runs in this Compose file, keep the default so Route Handlers reach `backend` over the Docker network:
@@ -32,6 +35,7 @@
 # --- Backend / security (override compose defaults in production) ---
 # JWT_SECRET=change-me-in-docker-env
 # ADMIN_PANEL_KEY=admin
+# MLTOOLS_BACKEND_API_TOKEN=pat_token_with_experiments.view_and_metrics.view
 
 # --- CORS (comma-separated origins: scheme + host + port; must match the UI as the browser sends `Origin`) ---
 # Local:

AGENTS.md

@@ -12,18 +12,23 @@ flowchart LR
   API["python/backend\n(FastAPI + Postgres)"]
   Scalars["python/scalars_service\n(FastAPI + ClickHouse)"]
   Blobs["python/object_storage\n(FastAPI + MinIO/S3)"]
+  MLTools["python/mltools\n(FastAPI + Celery + Postgres)"]
   SDK["python/sdk\n(client library)"]
 
   Web -->|"HTTP / BFF routes"| API
   API --> Scalars
   API --> Blobs
+  API --> MLTools
+  MLTools --> API
+  MLTools --> Blobs
   SDK --> API
 ```
 
 - **Frontend (`apps/web`)**: UI, dashboard, charts. Uses **Route Handlers** under `src/app/api/` as a BFF that forwards to the backend with auth cookies/headers.
 - **Backend (`python/backend`)**: Primary API (`api.main:app`), users/teams/RBAC, projects, experiments, hypotheses, metrics orchestration. Calls **scalars_service** and **object_storage** via HTTP clients in `src/clients/`.
 - **Scalars service (`python/scalars_service`)**: Stores scalar runs, tags, **artifacts_info** tables (per-project), and related query APIs. Backed by **ClickHouse** (and supporting infra as configured in that package).
 - **Object storage (`python/object_storage`)**: Upload/download/delete for experiment and project blobs; uses **MinIO** or **S3** and metadata in Postgres.
+- **MLTools (`python/mltools`)**: Asynchronous ML analysis jobs. The first bounded context is hyperparameter importance analysis; it reads scoped project data through the backend and stores model artifacts in S3-compatible storage.
 - **SDK (`python/sdk`)**: `experiment_tracker_sdk` — typed HTTP client used by training jobs and tools to talk to the backend API.
 - **Shared (`python/shared`)**: Shared Python types/utilities consumed by other Python packages where applicable.
 
@@ -36,6 +41,7 @@ flowchart LR
 | `python/backend/src/` | FastAPI app: `api/` routes, `domain/*` bounded contexts, `clients/*` HTTP clients, `db/`, `lib/`. |
 | `python/scalars_service/src/` | FastAPI scalars/artifacts_info service. `GET /scalars/get/...` paginates **experiments** first, then loads each metric column with ClickHouse `IS NOT NULL` + per-(experiment, column) uniform `max_points` sampling (`columns_per_query` controls parallel column queries; default 1). Cross-table ClickHouse work (delete experiment rows across scalars + artifacts_info + last_logged, usage, admin table listing) is under **`/projects`** (`projects` domain); compaction stays **`POST /scalars/projects/{id}/compact-columns`**. |
 | `python/object_storage/src/` | FastAPI storage service (buckets, experiment/project artifacts). |
+| `python/mltools/src/mltools/` | FastAPI/Celery ML analysis service. Domain logic is grouped under `domain/hparam_importance/`; outbound adapters live under `clients/`; worker composition lives under `workers/`. |
 | `python/sdk/src/experiment_tracker_sdk/` | Public Python SDK for the tracker API. |
 | `python/sdk/src/experiment_tracker_sdk/api_access.py` | Singleton :class:`ExpTrackerApiAccess` — shared ``APIRequestsRegistry`` / :class:`ExperimentTrackerClient` construction (used by :class:`ExpTracker` and CLI). |
 | `python/sdk/src/experiment_tracker_sdk/constants.py` | Default API base URL and ``/api`` prefix literals shared with settings. |
@@ -122,6 +128,7 @@ Work from the package root, for example:
 - `cd python/backend && uv run uvicorn api.main:app --reload --port 8000`
 - `cd python/scalars_service && uv run pytest`
 - `cd python/object_storage && uv run pytest`
+- `cd python/mltools && uv run pytest`
 - `cd python/sdk && uv run pytest`
 
 Do **not** assume a single global `python/backend`-only layout; **scalars_service**, **object_storage**, and **sdk** are first-class packages with their own `uv` workflows.
@@ -189,4 +196,4 @@ Prefer updating this file or code comments when changing global runbooks; avoid
 
 For **changing how in-app docs render** (remark/rehype directives, sanitize allowlist, `DocsMarkdown` components), follow and keep in sync **`apps/web/content/docs/contributing/extending-doc-pipeline.md`** (published at `/docs/contributing/extending-doc-pipeline`).
 
-Don't fight bugs! Every time you encounter the same error by accident, research the web and find 3-5 possible ways to fix it. Then choose the most effective solution and implement it.
+Don't fight bugs! Every time you encounter the same error by accident, research the web and find 3-5 possible ways to fix it. Then choose the most effective solution and implement it.

apps/web/content/docs/domains/experiments.md

@@ -46,9 +46,9 @@ tracker.progress(100)
 
 Experiment colors make runs visually distinct. They are especially important on scalar plots, where multiple experiments may share the same chart. Colors must be hex strings such as `#3366cc` or `#3366ccff`.
 
-## Features and parent diffs
+## Features, hyperparameters, and parent diffs
 
-Features are a tree of named nodes. Today they are the closest equivalent to hyperparameters and run configuration in the product.
+Features are a tree of named nodes that describe semantic experiment changes.
 
 ```python
 tracker.features([
@@ -66,7 +66,13 @@ tracker.features([
 ])
 ```
 
-When an experiment has a parent, the sidebar can show differences from the parent experiment and let users edit the feature tree. Use this for "what changed in this run?" information until a dedicated hyperparameter domain exists.
+When an experiment has a parent, the sidebar can show differences from the parent
+experiment and let users edit the feature tree. Use this for "what changed in this
+run?" information.
+
+Hyperparameters are stored separately as nested JSON and can be logged through
+`tracker.log_hparams(...)` or edited from the experiment sidebar. The Compare page
+shows baseline-relative added, removed, and changed hyperparameter values.
 
 ## Parent experiments and DAG
 

apps/web/content/docs/sdk/experiment-logging.md

@@ -184,10 +184,18 @@ tracker.features([
     {"name": "data", "children": [{"name": "mnist"}]},
     {"name": "model", "children": [{"name": "small-cnn"}]},
 ])
+tracker.log_hparams({
+    "optimizer": {"name": "adamw", "lr": 3e-4},
+    "training": {"batch_size": 32, "epochs": 50},
+})
 ```
 
 `progress` accepts `0..100` integers or `0..1` floats. `parent_experiment` resolves by name or id inside the current project.
 
+`features` describe semantic changes and research ideas. `log_hparams` stores the
+configurable training values used by the run. Each `log_hparams` call fully replaces
+the experiment's previous hyperparameter document.
+
 ## Run the repository example
 
 ```bash
@@ -572,12 +580,36 @@ tracker.features([
 ])
 ```
 
-Updates the experiment feature tree. Features are currently the best place to store hyperparameter-like run structure and "what changed" information.
+Updates the experiment feature tree. Use features for semantic changes, ablations,
+new mechanisms, and "what changed" information rather than configurable training
+values.
 
 When an experiment has a parent, the sidebar can show feature differences from that parent.
 
 ---
 
+### `log_hparams(...)`
+
+```python
+tracker.log_hparams({
+    "optimizer": {
+        "name": "adamw",
+        "lr": 0.001,
+    },
+    "training": {
+        "batch_size": 64,
+        "seed": 42,
+    },
+})
+```
+
+Validates and stores a nested hyperparameter JSON object. Repeated calls fully replace
+the previous document; they do not deep-merge it. Common values such as `Path`, `Enum`,
+`date`, `datetime`, and NumPy scalar values are converted when safe. Unsupported values
+raise `HparamsSerializationError` with the failing parameter path.
+
+---
+
 ### `name(...)`
 
 ```python

apps/web/package.json

@@ -1,6 +1,6 @@
 {
   "name": "web",
-  "version": "0.11.8",
+  "version": "0.12.1",
   "private": true,
   "scripts": {
     "dev": "next dev",

apps/web/src/app/(core)/projects/[projectId]/compare/page.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { useParams } from "next/navigation";
-import { CompareShell } from "@/domain/compare/components/compare-shell";
+import { CompareShell } from "@/domain/compare/components";
 
 export default function ComparePage() {
   const { projectId } = useParams<{ projectId: string }>();

apps/web/src/components/shared/experiment-diff-ui.tsx

@@ -0,0 +1,61 @@
+import type { ReactNode } from "react";
+import { CircleMinus, CirclePlus, PencilLine } from "lucide-react";
+import { Badge } from "@/components/ui/badge";
+import { cn } from "@/lib/utils";
+
+export type ExperimentDiffStatus = "unchanged" | "added" | "removed" | "changed";
+
+export function ExperimentDiffCountBadge({
+  status,
+  label,
+  value,
+  compact = false,
+}: {
+  status: Exclude<ExperimentDiffStatus, "unchanged">;
+  label: string;
+  value: number;
+  compact?: boolean;
+}) {
+  const marker = status === "added" ? "+" : status === "removed" ? "-" : "~";
+  return (
+    <Badge variant="outline" className={cn("text-[11px]", experimentDiffBadgeClass(status))}>
+      {compact ? `${marker}${value}` : `${label} ${value}`}
+    </Badge>
+  );
+}
+
+export function ExperimentDiffIcon({
+  status,
+  title,
+}: {
+  status: ExperimentDiffStatus;
+  title?: string;
+}) {
+  const iconClassName = "h-3.5 w-3.5";
+  let icon: ReactNode = null;
+  if (status === "added") {
+    icon = <CirclePlus className={cn(iconClassName, "text-green-700 dark:text-green-300")} />;
+  } else if (status === "removed") {
+    icon = <CircleMinus className={cn(iconClassName, "text-red-700 dark:text-red-300")} />;
+  } else if (status === "changed") {
+    icon = <PencilLine className={cn(iconClassName, "text-amber-700 dark:text-amber-300")} />;
+  }
+  return icon ? (
+    <span title={title} aria-label={title} role={title ? "img" : undefined}>
+      {icon}
+    </span>
+  ) : null;
+}
+
+export function experimentDiffSurfaceClass(status: ExperimentDiffStatus): string {
+  if (status === "added") return "bg-green-500/10 text-green-800 dark:text-green-300";
+  if (status === "removed") return "bg-red-500/10 text-red-800 dark:text-red-300";
+  if (status === "changed") return "bg-amber-500/10 text-amber-800 dark:text-amber-300";
+  return "text-foreground/80";
+}
+
+function experimentDiffBadgeClass(status: Exclude<ExperimentDiffStatus, "unchanged">): string {
+  if (status === "added") return "border-green-500/20 bg-green-500/10 text-green-700";
+  if (status === "removed") return "border-red-500/20 bg-red-500/10 text-red-700";
+  return "border-amber-500/20 bg-amber-500/10 text-amber-700";
+}

apps/web/src/components/shared/experiment-features-panel.tsx

@@ -2,7 +2,7 @@
 
 import { useEffect, useMemo, useState, type ReactNode } from "react";
 import { useMutation, useQueryClient } from "@tanstack/react-query";
-import { ArrowLeftRight, ArrowRight, CircleMinus, CirclePlus, GitCompare, Maximize2, PencilLine, Save } from "lucide-react";
+import { ArrowLeftRight, ArrowRight, GitCompare, Maximize2, PencilLine, Save } from "lucide-react";
 import type { Experiment } from "@/domain/experiments/types";
 import { experimentsService } from "@/domain/experiments/services";
 import { QUERY_KEYS } from "@/lib/constants/query-keys";
@@ -15,7 +15,6 @@ import {
   type FeatureNode,
 } from "@/lib/features/feature-tree";
 import { Button } from "@/components/ui/button";
-import { Badge } from "@/components/ui/badge";
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
 import {
   Dialog,
@@ -34,6 +33,11 @@ import {
 } from "@/components/ui/select";
 import { FeatureBulletEditor } from "@/components/shared/feature-bullet-editor";
 import { FeatureEditorLabelWithHelp } from "@/components/shared/feature-editor-help";
+import {
+  ExperimentDiffCountBadge,
+  ExperimentDiffIcon,
+  experimentDiffSurfaceClass,
+} from "@/components/shared/experiment-diff-ui";
 
 type ExperimentFeaturesPanelProps = {
   experiment: Experiment;
@@ -127,9 +131,9 @@ export function ExperimentFeaturesPanel({
         <CardContent className="w-full min-w-0 max-w-full space-y-2 overflow-hidden px-3 pb-3 pt-0">
           {parentExperiment && showDiffs ? (
             <div className="flex flex-wrap gap-1">
-              <FeatureCountBadge label="Added" value={summary.added} className="border-green-500/20 bg-green-500/10 text-green-700" />
-              <FeatureCountBadge label="Removed" value={summary.removed} className="border-red-500/20 bg-red-500/10 text-red-700" />
-              <FeatureCountBadge label="Changed" value={summary.changed + summary.renamed} className="border-amber-500/20 bg-amber-500/10 text-amber-700" />
+              <ExperimentDiffCountBadge status="added" label="Added" value={summary.added} />
+              <ExperimentDiffCountBadge status="removed" label="Removed" value={summary.removed} />
+              <ExperimentDiffCountBadge status="changed" label="Changed" value={summary.changed + summary.renamed} />
             </div>
           ) : null}
           <FeatureView
@@ -163,22 +167,6 @@ export function ExperimentFeaturesPanel({
   );
 }
 
-function FeatureCountBadge({
-  label,
-  value,
-  className,
-}: {
-  label: string;
-  value: number;
-  className?: string;
-}) {
-  return (
-    <Badge variant="outline" className={cn("text-[11px]", className)}>
-      {label} {value}
-    </Badge>
-  );
-}
-
 function FeatureView({
   childFeatures,
   parentExperimentExists,
@@ -240,9 +228,9 @@ function FeatureExpandedModal({
           </div>
           {showDiffs ? (
             <div className="flex flex-wrap gap-1 pt-2">
-              <FeatureCountBadge label="Added" value={summary.added} className="border-green-500/20 bg-green-500/10 text-green-700" />
-              <FeatureCountBadge label="Removed" value={summary.removed} className="border-red-500/20 bg-red-500/10 text-red-700" />
-              <FeatureCountBadge label="Changed" value={summary.changed + summary.renamed} className="border-amber-500/20 bg-amber-500/10 text-amber-700" />
+              <ExperimentDiffCountBadge status="added" label="Added" value={summary.added} />
+              <ExperimentDiffCountBadge status="removed" label="Removed" value={summary.removed} />
+              <ExperimentDiffCountBadge status="changed" label="Changed" value={summary.changed + summary.renamed} />
             </div>
           ) : null}
         </DialogHeader>
@@ -330,7 +318,7 @@ function FeatureChangeRow({ row }: { row: FlatDiffLine }) {
   if ((row.status === "renamed" || row.status === "changed") && row.parentName && row.childName) {
     return (
       <FeatureChangedLine
-        icon={<FeatureDiffIcon status={row.status} title={diffTitle} />}
+        icon={<ExperimentDiffIcon status="changed" title={diffTitle} />}
         previousName={row.parentName}
         name={row.childName}
         depth={row.depth}
@@ -340,15 +328,10 @@ function FeatureChangeRow({ row }: { row: FlatDiffLine }) {
 
   return (
     <FeatureUnifiedLine
-      icon={<FeatureDiffIcon status={row.status} title={diffTitle} />}
+      icon={<ExperimentDiffIcon status={row.status === "renamed" ? "changed" : row.status} title={diffTitle} />}
       name={displayName}
       depth={row.depth}
-      className={cn(
-        row.status === "added" && "bg-green-500/10 text-green-800 dark:text-green-300",
-        row.status === "removed" && "bg-red-500/10 text-red-800 dark:text-red-300",
-        (row.status === "renamed" || row.status === "changed") &&
-          "bg-amber-500/10 text-amber-800 dark:text-amber-300"
-      )}
+      className={experimentDiffSurfaceClass(row.status === "renamed" ? "changed" : row.status)}
     />
   );
 }
@@ -410,30 +393,6 @@ function FeatureNodeDot() {
   return <span className="mr-1.5 h-1 w-1 shrink-0 rounded-full bg-muted-foreground/50" aria-hidden="true" />;
 }
 
-function FeatureDiffIcon({
-  status,
-  title,
-}: {
-  status: FeatureDiffNode["status"];
-  title: string;
-}) {
-  const iconClassName = "h-3.5 w-3.5";
-  let icon: ReactNode = null;
-  if (status === "added") {
-    icon = <CirclePlus className={cn(iconClassName, "text-green-700 dark:text-green-300")} />;
-  } else if (status === "removed") {
-    icon = <CircleMinus className={cn(iconClassName, "text-red-700 dark:text-red-300")} />;
-  } else if (status === "renamed" || status === "changed") {
-    icon = <PencilLine className={cn(iconClassName, "text-amber-700 dark:text-amber-300")} />;
-  }
-
-  return icon ? (
-    <span title={title} aria-label={title} role="img">
-      {icon}
-    </span>
-  ) : null;
-}
-
 function getFeatureDiffTitle(row: FlatDiffLine): string {
   if (row.status === "added") {
     return `${row.childName} was added in experiment and is not present in parent`;
@@ -671,11 +630,9 @@ function FeatureStructuredDiff({
           <span className="text-muted-foreground">{experimentLabel}</span>
         </div>
         <div className="flex gap-1">
-          <Badge variant="outline" className="border-green-500/20 bg-green-500/10 text-green-700">+{summary.added}</Badge>
-          <Badge variant="outline" className="border-red-500/20 bg-red-500/10 text-red-700">-{summary.removed}</Badge>
-          <Badge variant="outline" className="border-amber-500/20 bg-amber-500/10 text-amber-700">
-            ~{summary.changed + summary.renamed}
-          </Badge>
+          <ExperimentDiffCountBadge status="added" label="Added" value={summary.added} compact />
+          <ExperimentDiffCountBadge status="removed" label="Removed" value={summary.removed} compact />
+          <ExperimentDiffCountBadge status="changed" label="Changed" value={summary.changed + summary.renamed} compact />
         </div>
       </div>
       <div className="min-h-0 flex-1 overflow-auto p-2">