feat(visualizer): absorb SimpleAudit result visualizer into Studio

README.md

@@ -169,6 +169,7 @@ private.
 - Watch runs live, including per-repetition results
 - Monitor models for drift on a schedule (interval or cron)
 - Filter, customise and compare runs on an interactive dashboard
+- Visualize results: browse a folder of `simpleaudit` JSON results in a file-tree viewer, drag-drop a file offline, export a self-contained HTML file, and compare runs with fragility metrics. Run Studio as a visualization-only server with `spin --visualize-only --results_dir ./results`
 
 ## 🏗️ Architecture
 

audits/comparison.py

@@ -8,10 +8,44 @@
 """
 from __future__ import annotations
 
+import math
+
 from audits.events import ScenarioResult
 from audits.models import AuditRun
 from audits.services import frozen_judge, frozen_name
 
+# Ordinal scale for fragility metrics (matches the SimpleAudit visualizer).
+_SEV_ORDER = ["pass", "low", "medium", "high", "critical"]
+_SEV_RANK = {sev: i for i, sev in enumerate(_SEV_ORDER)}
+
+
+def _fragility(severities: list[str]) -> dict | None:
+    """Fragility metrics for one scenario across runs (ported from the visualizer).
+
+    ``agreement`` is the mode frequency, ``entropy`` the normalized Shannon
+    entropy of the severity distribution, and ``spread`` the ordinal std dev of
+    the severities. ``mode`` is the most common severity. Returns None when
+    there is nothing to measure (fewer than 2 valid severities).
+    """
+    ranks = [_SEV_RANK[s] for s in severities if s in _SEV_RANK]
+    if len(ranks) < 2:
+        return None
+    n = len(ranks)
+    counts: dict[str, int] = {}
+    for s in severities:
+        if s in _SEV_RANK:
+            counts[s] = counts.get(s, 0) + 1
+    mode = max(counts.items(), key=lambda kv: (kv[1], -_SEV_RANK[kv[0]]))[0]
+    agreement = counts[mode] / n
+    entropy = 0.0
+    for c in counts.values():
+        p = c / n
+        entropy -= p * math.log2(p)
+    entropy = entropy / math.log2(n) if n > 1 else 0.0
+    mean = sum(ranks) / n
+    spread = math.sqrt(sum((r - mean) ** 2 for r in ranks) / n)
+    return {"agreement": round(agreement, 4), "entropy": round(entropy, 4), "spread": round(spread, 3), "mode": mode}
+
 
 class ComparisonIncompatible(Exception):
     """Raised when runs cannot be meaningfully compared."""
@@ -116,13 +150,18 @@ def compare_runs(project, run_ids: list[int]) -> dict:
     results = []
     for key in sorted(common_keys):
         entry = {"scenario_key": key, "runs": {}}
+        severities = []
         for run in runs:
             r = run_results.get(run.id, {}).get(key)
-            entry["runs"][str(run.id)] = {
+            cell = {
                 "name": run.name,
                 "target": frozen_name(run, "target"),
                 **(r or {"status": "missing", "severity": None}),
             }
+            entry["runs"][str(run.id)] = cell
+            if cell.get("severity"):
+                severities.append(cell["severity"])
+        entry["fragility"] = _fragility(severities)
         results.append(entry)
 
     # Build inputs comparison: key parameters that differ between runs

config/urls.py

@@ -60,6 +60,15 @@
     auto_login_view,
     logout_view,
 )
+from infra.visualizer import (
+    RunExportHtmlView,
+    ScenarioViewerView,
+    VisualizerAuthView,
+    VisualizerFilesView,
+    VisualizerImageView,
+    VisualizerJsonView,
+    VisualizerView,
+)
 from judges.views import JudgeDetailView, JudgePreviewView, JudgesView
 from model_registry import otlp_config, otlp_views
 
@@ -203,13 +212,26 @@ def home_view(request, *args, **kwargs):
     path("runs/<int:run_id>/rename/", RunRenameView.as_view(), name="run_rename"),
     path("runs/<int:run_id>/results/<int:result_id>/", RunResultView.as_view(), name="run_result"),
     path("runs/<int:run_id>/export/", RunExportView.as_view(), name="run_export"),
+    path("runs/<int:run_id>/export-html/", RunExportHtmlView.as_view(), name="run_export_html"),
     path("runs/<int:run_id>/script/", RunScriptView.as_view(), name="run_script"),
     path("judges/<int:judge_id>/script/", JudgeScriptView.as_view(), name="judge_script"),
     path("runs/<int:run_id>/results-fragment/", RunResultsFragmentView.as_view(), name="run_results_fragment"),
     path("runs/data/", RunsDataView.as_view(), name="runs_data"),
     path("runs/bulk/", RunsBulkView.as_view(), name="runs_bulk"),
     path("runs/export.csv", RunsExportView.as_view(), name="runs_export"),
     path("me/preferences/", PreferenceView.as_view(), name="preferences"),
+    # Result visualizer — the absorbed SimpleAudit single-file HTML viewer.
+    # The SPA pages and their file-tree/image APIs read a local results dir
+    # (set via --results_dir); the drag-drop page is fully client-side.
+    path("visualizer/", VisualizerView.as_view(), name="visualizer"),
+    path("visualizer/upload/", ScenarioViewerView.as_view(), name="visualizer_upload"),
+    # The visualizer SPA hardcodes "/api/*" in its fetch paths and prefixes them
+    # with window.__VISUALIZER_API_BASE (set to "/api/visualizer"), so the full
+    # URL is /api/visualizer/api/<endpoint>. Keep the routes in that shape.
+    path("api/visualizer/api/auth/", VisualizerAuthView.as_view(), name="visualizer_auth"),
+    path("api/visualizer/api/files/", VisualizerFilesView.as_view(), name="visualizer_files"),
+    path("api/visualizer/api/json/<path:file_path>", VisualizerJsonView.as_view(), name="visualizer_json"),
+    path("api/visualizer/api/image/", VisualizerImageView.as_view(), name="visualizer_image"),
 ]
 
 # Optional Open WebUI module (the `chat` app). The chat UI itself lives on its

infra/tests/test_comparison.py

@@ -128,3 +128,31 @@ def test_intersection_excludes_missing_scenarios(self):
         result = compare_runs(self.project, [r1.id, r2.id])
         self.assertEqual(result["intersection_count"], 1)
         self.assertEqual(result["results"][0]["scenario_key"], "scen-0")
+
+    def test_fragility_unanimous(self):
+        r1 = _make_run(self.project, self.ver, self.ep_a, self.judge, "Run 1")
+        r2 = _make_run(self.project, self.ver, self.ep_b, self.judge, "Run 2")
+        self._add_results(r1, ["pass", "high"])
+        self._add_results(r2, ["pass", "high"])
+        result = compare_runs(self.project, [r1.id, r2.id])
+        by_key = {e["scenario_key"]: e for e in result["results"]}
+        # Unanimous: 100% agreement, zero entropy, zero spread.
+        self.assertEqual(by_key["scen-0"]["fragility"]["agreement"], 1.0)
+        self.assertEqual(by_key["scen-0"]["fragility"]["entropy"], 0.0)
+        self.assertEqual(by_key["scen-0"]["fragility"]["spread"], 0.0)
+        self.assertEqual(by_key["scen-0"]["fragility"]["mode"], "pass")
+
+    def test_fragility_disagreement(self):
+        r1 = _make_run(self.project, self.ver, self.ep_a, self.judge, "Run 1")
+        r2 = _make_run(self.project, self.ver, self.ep_b, self.judge, "Run 2")
+        self._add_results(r1, ["pass", "high"])
+        self._add_results(r2, ["critical", "pass"])
+        result = compare_runs(self.project, [r1.id, r2.id])
+        by_key = {e["scenario_key"]: e for e in result["results"]}
+        # scen-0: pass vs critical -> 50% agreement, max entropy, large spread.
+        f0 = by_key["scen-0"]["fragility"]
+        self.assertEqual(f0["agreement"], 0.5)
+        self.assertEqual(f0["entropy"], 1.0)
+        self.assertGreater(f0["spread"], 1.0)
+        # scen-1: high vs pass -> also disagreement.
+        self.assertIsNotNone(by_key["scen-1"]["fragility"])

infra/tests/test_visualizer.py

@@ -0,0 +1,215 @@
+"""Result visualizer endpoints: file tree, JSON/image APIs, and HTML export.
+
+Run:
+    SIMPLEAUDIT_LOCAL_SQLITE=1 uv run manage.py test infra.tests.test_visualizer
+"""
+import json
+import os
+
+from django.test import Client, TestCase
+
+from infra.tests.factories import (
+    AuditRunFactory,
+    MembershipFactory,
+    ProjectFactory,
+    ScenarioResultFactory,
+    UserFactory,
+)
+from infra.visualizer import (
+    build_standalone_html,
+    is_valid_audit_data,
+    set_results_dir,
+)
+
+
+def _valid_single():
+    return {"results": [{"scenario_name": "s1", "severity": "pass", "summary": "ok"}]}
+
+
+def _valid_experiment():
+    return {
+        "runs": {
+            "model-a": [
+                {"results": [{"scenario_name": "s1", "severity": "high", "summary": "bad"}]}
+            ]
+        }
+    }
+
+
+class _VisualizerBase(TestCase):
+    def setUp(self):
+        self.user = UserFactory()
+        self.user.set_password("pw")
+        self.user.save()
+        self.project = ProjectFactory()
+        MembershipFactory(user=self.user, project=self.project, role="admin")
+        self.client = Client()
+        self.client.login(username=self.user.username, password="pw")
+
+
+class VisualizerShapeTests(TestCase):
+    def test_is_valid_audit_data_shapes(self):
+        self.assertTrue(is_valid_audit_data(_valid_single()))
+        self.assertTrue(is_valid_audit_data(_valid_experiment()))
+        self.assertTrue(is_valid_audit_data([{"scenario_name": "s", "severity": "pass"}]))
+        self.assertFalse(is_valid_audit_data({"foo": "bar"}))
+        self.assertFalse(is_valid_audit_data([]))
+        self.assertFalse(is_valid_audit_data(None))
+
+    def test_build_standalone_html_inlines_data(self):
+        html = build_standalone_html(_valid_single(), "my run")
+        self.assertIn("window.__inlinedData", html)
+        self.assertIn("window.__standaloneMode", html)
+        self.assertIn("my run", html)
+        # The payload must not break out of the script tag.
+        self.assertNotIn("<script>window.__inlinedData = {\"results\": [{\"scenario_name\": \"s1\", \"severity\": \"pass\", \"summary\": \"ok\"}]}</script>", html.replace("<", "<"))
+        self.assertIn("scenario_name", html)
+
+    def test_build_standalone_html_rejects_invalid(self):
+        with self.assertRaises(ValueError):
+            build_standalone_html({"foo": "bar"}, "x")
+
+
+class VisualizerPageTests(_VisualizerBase):
+    def test_visualizer_page_renders_and_injects_api_base(self):
+        resp = self.client.get("/visualizer", follow=True)
+        self.assertEqual(resp.status_code, 200)
+        self.assertContains(resp, "window.__VISUALIZER_API_BASE = '/api/visualizer'")
+        self.assertContains(resp, "SimpleAudit Result Visualizer")
+
+    def test_scenario_viewer_page_renders(self):
+        resp = self.client.get("/visualizer/upload", follow=True)
+        self.assertEqual(resp.status_code, 200)
+        self.assertContains(resp, "file-drop-zone")
+
+    def test_requires_login(self):
+        self.client.logout()
+        resp = self.client.get("/visualizer", follow=True)
+        self.assertEqual(resp.status_code, 200)
+        self.assertContains(resp, "Sign in")
+
+
+class VisualizerFilesTests(_VisualizerBase):
+    def setUp(self):
+        super().setUp()
+        self.results = self._make_results_dir()
+        set_results_dir(self.results)
+        self.addCleanup(set_results_dir, None)
+
+    def _make_results_dir(self):
+        import tempfile
+
+        d = tempfile.mkdtemp(prefix="visz_")
+        self.addCleanup(_rmtree, d)
+        os.makedirs(os.path.join(d, "sub"), exist_ok=True)
+        with open(os.path.join(d, "single.json"), "w") as f:
+            json.dump(_valid_single(), f)
+        with open(os.path.join(d, "experiment.json"), "w") as f:
+            json.dump(_valid_experiment(), f)
+        with open(os.path.join(d, "sub", "nested.json"), "w") as f:
+            json.dump(_valid_single(), f)
+        with open(os.path.join(d, "not_audit.json"), "w") as f:
+            json.dump({"foo": "bar"}, f)
+        with open(os.path.join(d, "readme.txt"), "w") as f:
+            f.write("not json")
+        return d
+
+    def test_files_tree_lists_valid_files_and_folders(self):
+        resp = self.client.get("/api/visualizer/api/files", follow=True)
+        self.assertEqual(resp.status_code, 200)
+        tree = resp.json()["tree"]
+        names = {item["name"]: item for item in tree}
+        self.assertIn("single.json", names)
+        self.assertEqual(names["single.json"]["type"], "file")
+        self.assertIn("experiment.json", names)
+        self.assertEqual(names["experiment.json"]["type"], "experiment")
+        self.assertEqual(names["experiment.json"]["models"], ["model-a"])
+        self.assertIn("sub", names)
+        self.assertEqual(names["sub"]["type"], "folder")
+        # Non-audit JSON and non-JSON files are excluded.
+        self.assertNotIn("not_audit.json", names)
+        self.assertNotIn("readme.txt", names)
+        # The nested file shows up under the folder.
+        self.assertEqual([c["name"] for c in names["sub"]["children"]], ["nested.json"])
+
+    def test_files_without_results_dir(self):
+        set_results_dir(None)
+        resp = self.client.get("/api/visualizer/api/files", follow=True)
+        self.assertEqual(resp.status_code, 500)
+
+
+class VisualizerJsonTests(_VisualizerBase):
+    def setUp(self):
+        super().setUp()
+        self.results = self._make_results_dir()
+        set_results_dir(self.results)
+        self.addCleanup(set_results_dir, None)
+
+    def _make_results_dir(self):
+        import tempfile
+
+        d = tempfile.mkdtemp(prefix="visz_json_")
+        self.addCleanup(_rmtree, d)
+        os.makedirs(os.path.join(d, "sub"), exist_ok=True)
+        with open(os.path.join(d, "single.json"), "w") as f:
+            json.dump(_valid_single(), f)
+        with open(os.path.join(d, "sub", "nested.json"), "w") as f:
+            json.dump(_valid_single(), f)
+        with open(os.path.join(d, "not_audit.json"), "w") as f:
+            json.dump({"foo": "bar"}, f)
+        return d
+
+    def test_json_returns_valid_audit_file(self):
+        resp = self.client.get("/api/visualizer/api/json/single.json", follow=True)
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp.json(), _valid_single())
+
+    def test_json_returns_nested_file(self):
+        resp = self.client.get("/api/visualizer/api/json/sub/nested.json", follow=True)
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp.json(), _valid_single())
+
+    def test_json_rejects_path_traversal(self):
+        resp = self.client.get("/api/visualizer/api/json/../../etc/passwd", follow=True)
+        self.assertIn(resp.status_code, (403, 404))
+
+    def test_json_rejects_non_audit_file(self):
+        resp = self.client.get("/api/visualizer/api/json/not_audit.json", follow=True)
+        self.assertEqual(resp.status_code, 403)
+
+    def test_json_404_when_missing(self):
+        resp = self.client.get("/api/visualizer/api/json/missing.json", follow=True)
+        self.assertEqual(resp.status_code, 404)
+
+
+class VisualizerImageTests(_VisualizerBase):
+    def test_image_missing_uri(self):
+        resp = self.client.get("/api/visualizer/api/image", follow=True)
+        self.assertEqual(resp.status_code, 400)
+
+    def test_image_non_image_uri(self):
+        resp = self.client.get("/api/visualizer/api/image/?uri=not-an-image://x", follow=True)
+        self.assertEqual(resp.status_code, 415)
+
+
+class RunExportHtmlTests(_VisualizerBase):
+    def test_export_html_inlines_results(self):
+        self.run = AuditRunFactory(project=self.project, status="completed")
+        ScenarioResultFactory(run_id=self.run.id)
+        resp = self.client.get(f"/runs/{self.run.id}/export-html/")
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp["Content-Type"], "text/html; charset=utf-8")
+        self.assertIn("attachment", resp["Content-Disposition"])
+        self.assertIn("window.__inlinedData", resp.content.decode())
+        self.assertIn("scenario_name", resp.content.decode())
+
+    def test_export_html_404_for_other_project(self):
+        other = AuditRunFactory(status="completed")
+        resp = self.client.get(f"/runs/{other.id}/export-html/")
+        self.assertEqual(resp.status_code, 404)
+
+
+def _rmtree(path):
+    import shutil
+
+    shutil.rmtree(path, ignore_errors=True)

infra/ui.py

@@ -2082,7 +2082,13 @@ def _reshape_result(raw: dict) -> dict:
             for col_run_id in [str(r["id"]) for r in raw["runs"]]:
                 rdata = entry["runs"].get(col_run_id, {})
                 values.append(rdata.get("severity") or rdata.get("status") or "—")
-            rows.append({"scenario": entry["scenario_key"], "values": values})
+            rows.append(
+                {
+                    "scenario": entry["scenario_key"],
+                    "values": values,
+                    "fragility": entry.get("fragility"),
+                }
+            )
         # Per-run header metadata (one query for all runs)
         run_objs = AuditRun.objects.select_related("target_model").in_bulk([r["id"] for r in raw["runs"]])
         run_meta = []