From 00f621ea7e0b7b806abacb52eb1b9e32d1d41531 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Sun, 26 Apr 2026 09:12:01 -0400 Subject: [PATCH 01/38] docs: capture TOCHI prototype roadmap --- docs/agent-role-spec.md | 131 ++++++++++ docs/case-study-prototype-readiness.md | 90 +++++++ docs/closed-loop-smoke.md | 89 +++++++ docs/manual-bulk-test-checklist.md | 96 +++++++ docs/proofreading-redesign-notes.md | 131 ++++++++++ docs/prototype-completion-roadmap.md | 234 ++++++++++++++++++ docs/research/agentic-ui-design-pass.md | 93 +++++++ docs/research/claude-code-agent-patterns.md | 130 ++++++++++ .../research/tochi-hci-prototype-takeaways.md | 81 ++++++ 9 files changed, 1075 insertions(+) create mode 100644 docs/agent-role-spec.md create mode 100644 docs/case-study-prototype-readiness.md create mode 100644 docs/closed-loop-smoke.md create mode 100644 docs/manual-bulk-test-checklist.md create mode 100644 docs/proofreading-redesign-notes.md create mode 100644 docs/prototype-completion-roadmap.md create mode 100644 docs/research/agentic-ui-design-pass.md create mode 100644 docs/research/claude-code-agent-patterns.md create mode 100644 docs/research/tochi-hci-prototype-takeaways.md diff --git a/docs/agent-role-spec.md b/docs/agent-role-spec.md new file mode 100644 index 00000000..ab73dca9 --- /dev/null +++ b/docs/agent-role-spec.md @@ -0,0 +1,131 @@ +# Agent Role Specification + +This defines the intended role of the agent in the TOCHI-facing PyTC Client +prototype. + +## Product Role + +The agent is a workflow copilot for biomedical segmentation. It should help a +biologist understand the current state, choose the next safe action, and inspect +evidence. It is not a replacement for the human reviewer and should not silently +execute risky training, inference, overwrite, export, or retraining actions. + +## Default Behavior + +- Recommend the next concrete workflow action first. +- Keep answers skimmable: at most three bullets and usually under 90 words. +- Ground UI instructions in visible app controls and documented workflows. +- Show blockers clearly: missing image, label, checkpoint, prediction, export, + correction set, model version, or metric result. +- Prefer app actions over command-line instructions for normal users. +- Ask for approval before actions that run jobs, overwrite masks, stage + retraining, launch inference, launch training, or export evidence. + +## Agent Boundaries + +- The agent may navigate the user, summarize state, propose actions, explain + settings, populate form fields, and prepare commands. +- The agent may generate a training/inference command only when explicitly + asked for a command. +- The agent may not fabricate paths, scripts, shortcuts, results, model quality, + or hidden app features. +- The agent may not claim autonomous closed-loop improvement until the app has + executed baseline inference, proofreading export, retraining, candidate + inference, and before/after evaluation using app-generated artifacts. + +## UI Role + +The agent should appear as a compact decision-support layer across the app, not +as a verbose chatbot. It should connect File Management, Visualization, +Inference, Proofreading, Training, Evaluation, and Evidence Export through the +workflow evidence substrate. + +Implemented UI commitments: + +- A global workflow-agent strip should always answer "what should I do next?" + from current workflow evidence. +- The first action must match the recommendation. For example, "start + inference" should expose a start-inference action, not merely a navigation + button. +- The agent can open the assistant evidence/context drawer from any module so + the user can inspect artifacts, correction sets, metrics, and proposals + without losing the active workflow. +- Retraining handoff remains approval-gated: the agent may create a pending + proposal from corrected masks, but the human must approve staging. +- Chat responses for workflow questions should reuse the same recommendation + state and use compact `Do this`, `Why`, `Ready`, `Watch out` structure. + +## 2026-04-25 Orchestrator Pass + +- The agent should speak in user goals, not implementation nouns. Use + "Proofread this data", "Run model", "Use edits for training", and "Compare + results" instead of "open artifact", "candidate output", or "retraining + handoff" in biologist-facing controls. +- The primary action should mutate local app state when safe. `Proofread this + data` now carries a `start_proofreading` runtime action with image/mask/review + overrides so the proofreading workbench can load the current pair directly. +- Status panels should summarize readiness with domain labels: previous result, + new result, your saved edits, and reference mask. Full paths and dataset-key + options are still available, but should be secondary/debuggable details. +- The agent is allowed to orchestrate navigation, form prefill, proofreading + load, inference launch, training launch, metric comparison, and report export + through explicit client effects. Risky steps remain user-triggered and + event-logged. + +## Success Criteria + +- A biologist can skim the response in under ten seconds. +- The response identifies the next useful action or the single blocking input. +- Risky actions are auditable as proposals, approvals, rejections, or workflow + events. +- The agent improves workflow continuity without hiding provenance or control. + +## 2026-04-25 Prompt-Boundary Hardening + +- The active app assistant defaults to the deterministic workflow orchestrator + whenever a workflow exists. The generic documentation LLM is no longer allowed + to own the in-app agent persona for greetings or vague workflow requests. +- Internal system prompts, role text, routing rules, tool names, and response + style instructions are never valid user-facing output. If the generic LLM + leaks prompt scaffolding, the server replaces it with a compact safe fallback. +- The orchestrator now handles greetings, incomplete intents, status checks, + module navigation, model launch, proofreading launch, retraining handoff, + before/after metric comparison, and evidence export as explicit, auditable + app actions. +- The agent is not just a help sidebar: it is the controller that translates + user goals into workflow-state-aware next steps while preserving human + approval for long-running or artifact-changing operations. + +## 2026-04-25 Claude Code Pattern Pass + +- Treat chat as a control surface over typed workflow tools. The agent should + return app actions with risk metadata rather than long explanatory prose. +- Every action should be classifiable as `view only`, `sets form`, `opens + editor`, `runs job`, `writes record`, or `exports`. Long-running or + artifact-changing actions require an explicit user click and workflow-event + logging. +- Support deterministic slash-style workflow commands for power users: + `/status`, `/infer`, `/proofread`, `/train`, `/compare`, `/export`, and + `/help`. +- Keep command-like route details collapsed by default. Biologist-facing UI + should show the domain action first and reserve implementation details for + debugging/logging. +- Maintain a structured loop checklist in the response payload so the assistant + can answer "what is left?" without inventing state from chat history. +- Future subagents should be domain-specialized and tool-restricted: + data-curator, model-runner, proofreading-triage, and evidence-auditor. + +## 2026-04-25 Contextual Entry-Point Pass + +- Keep the main drawer/chat as the full conversation and evidence inspector. +- Do not keep a persistent next-step strip on the main canvas by default; it + competes with the biomedical task UI. +- Do not add per-module agent cards by default; they make the biomedical UI feel + busier than the task requires. +- If a module needs agent help, expose it as a normal button near the relevant + control, with any explanation in a hover tooltip. +- All agent-triggered buttons must still log client events and route through + `executeAssistantItem` or `queryAgent` so they remain auditable. +- Training/inference terminal logs should be hidden by default. The main UI + should show a plain run state such as "Running", "Completed", or "Needs + attention" and only reveal raw logs on demand. diff --git a/docs/case-study-prototype-readiness.md b/docs/case-study-prototype-readiness.md new file mode 100644 index 00000000..166264e5 --- /dev/null +++ b/docs/case-study-prototype-readiness.md @@ -0,0 +1,90 @@ +# Case-Study Prototype Readiness Protocol + +This protocol defines the researcher/facilitator evidence needed before using +the prototype in paper-facing case studies. These gates are not participant UI. +The participant-facing target remains an agent-mediated, closed-loop biomedical +segmentation workflow, not a standalone PyTC GUI and not a case-study protocol +manager. + +## Prototype Boundary + +- Do not expose case-study design, case-study readiness gates, or case-study + plan generation in the participant prototype. +- Keep researcher-only checks available through backend tests, backend routes, + exported bundles, and this protocol document. +- Participant-facing controls should stay biomedical-workflow focused: load + data, run inference, inspect hotspots, proofread, approve/reject risky agent + actions, retrain, compare outputs, and export evidence. +- If a facilitator UI is added later, label it separately as researcher/study + instrumentation rather than part of the biomedical user workflow. + +## User Workflow Loop + +1. Create or resume a workflow project. +2. Load image, label, mask, checkpoint, and configuration artifacts. +3. Run or load baseline inference. +4. Inspect results with workflow-aware agent guidance. +5. Review ranked failure hotspots. +6. Proofread selected masks and save corrections. +7. Export a `CorrectionSet`. +8. Preview correction impact and retraining readiness. +9. Review an agent-proposed next action. +10. Approve, reject, interrupt, or resume risky workflow actions. +11. Run retraining from the staged corrections. +12. Register the resulting `ModelVersion`. +13. Re-run inference with the new checkpoint. +14. Compare before/after outputs and metrics. +15. Export the evidence bundle for analysis and paper figures. + +## Case Studies + +| ID | Purpose | Participant Task | Required Prototype Gate | +| --- | --- | --- | --- | +| CS1 | Workflow continuity | Load data, create viewer, run or inspect inference, proofread, inspect timeline | Workflow context, data artifacts, baseline inference | +| CS2 | Agent-guided triage | Ask where to inspect, review hotspot ranking, correct a selected region | Persisted hotspots, workflow-grounded agent response | +| CS3 | Correction-to-retraining loop | Export corrections, preview impact, approve retraining, produce checkpoint | `CorrectionSet`, training `ModelRun`, `ModelVersion` | +| CS4 | Before/after evidence | Compare baseline and post-retraining inference | Two inference runs plus `EvaluationResult` | +| CS5 | Approval-gated control | Review, approve, reject, and inspect agent proposals | Proposal events, approval/rejection events | +| CS6 | Failure recovery | Recover from failed training, bad config, or interrupted run | Terminal failure events, interrupted/resumed agent plan, visible next action | +| CS7 | Study export | Export state, artifacts, metrics, events, screenshots, and task timing | Evidence bundle with typed records | + +## Instrumentation Checklist + +- Capture every task transition as a `WorkflowEvent`. +- Materialize image, mask, correction, checkpoint, prediction, and report paths as + `WorkflowArtifact` records. +- Record inference and training lifecycle as `WorkflowModelRun` records. +- Register candidate checkpoints as `WorkflowModelVersion` records. +- Store before/after metrics as `WorkflowEvaluationResult` records. +- Store bounded plan previews and approval/interruption state as + `WorkflowAgentPlan` and `WorkflowAgentStep` records for researcher-only + protocol verification, not participant-facing app controls. +- Export event logs, typed artifacts, run/version records, correction sets, + evaluation results, agent plans, and artifact-existence checks in the workflow + bundle. +- Use `/api/workflows/{workflow_id}/case-study-readiness` before each session. + +## UI Consistency Checklist + +- Every stage page starts with the same `StageHeader` anatomy. +- The workflow rail uses the same stage names as the paper: setup, + visualization, inference, proofreading, retraining staged, evaluation. +- Agent proposals always display title, proposal type, approval state, rationale, + artifact references, and approve/reject controls. +- Repeated functions keep consistent labels: `Approve`, `Reject`, `Refresh + Insights`, `Start Training`, `Start Inference`, `Open Proofreading`. +- Status colors are semantic, not decorative: amber for review/staging, green for + completed/model-ready, red for failure/high risk, blue/teal for inspection. +- Keyboard focus and accessible names must remain consistent across repeated + controls. +- Case-study readiness and protocol-plan previews are intentionally absent from + the participant prototype. + +## Pre-Session Exit Criteria + +- The readiness endpoint has no missing P0 gates for the selected study. +- The facilitator can complete the full scripted loop on the frozen sample data. +- The evidence bundle includes typed artifacts, events, model runs, model + versions, correction sets, evaluation records, and agent plan records. +- Screenshots for the paper can be regenerated from the scripted walkthrough. +- The paper wording matches the demonstrated prototype behavior. diff --git a/docs/closed-loop-smoke.md b/docs/closed-loop-smoke.md new file mode 100644 index 00000000..d228dd38 --- /dev/null +++ b/docs/closed-loop-smoke.md @@ -0,0 +1,89 @@ +# Closed-Loop Workflow Smoke + +Run this when you want a quick researcher/developer check that the workflow evidence substrate still works: + +```bash +uv run python scripts/run_closed_loop_smoke.py --output-dir /tmp/pytc-closed-loop-smoke +``` + +To run against the mito25 real image/segmentation pair without loading the full volume: + +```bash +uv run python scripts/run_closed_loop_smoke.py \ + --output-dir /tmp/pytc-mito25-smoke \ + --image-path /Users/adamg/seg.bio/pytc-client/testing_projects/mito25_paper_loop_smoke/data/image/mito25_smoke_13312-15360_im.h5 \ + --mask-path /Users/adamg/seg.bio/pytc-client/testing_projects/mito25_paper_loop_smoke/data/seg/mito25_smoke_13312-15360_seg.h5 \ + --image-dataset main \ + --mask-dataset data +``` + +If real prediction files exist, pass them directly instead of deriving +baseline/candidate masks from the segmentation: + +```bash +uv run python scripts/run_closed_loop_smoke.py \ + --output-dir /tmp/pytc-real-prediction-smoke \ + --image-path /path/to/image.h5 \ + --mask-path /path/to/initial-or-ground-truth-mask.h5 \ + --image-dataset main \ + --mask-dataset data \ + --baseline-prediction-path /path/to/baseline-prediction.h5 \ + --candidate-prediction-path /path/to/candidate-prediction.h5 \ + --baseline-dataset prediction \ + --candidate-dataset prediction \ + --crop 0:16,0:256,768:1024 +``` + +For PyTC outputs such as `result_xy.h5`, pass the HDF5 dataset and select the +prediction channel to compare: + +```bash +uv run python scripts/run_closed_loop_smoke.py \ + --output-dir /tmp/pytc-app-prediction-smoke \ + --image-path /Users/adamg/seg.bio/pytc-client/testing_projects/mito25_paper_loop_smoke/data/image/mito25_smoke_13312-15360_im.h5 \ + --mask-path /Users/adamg/seg.bio/pytc-client/testing_projects/mito25_paper_loop_smoke/data/seg/mito25_smoke_13312-15360_seg.h5 \ + --image-dataset main \ + --mask-dataset data \ + --baseline-prediction-path /Users/adamg/seg.bio/pytc-client/testing_projects/mito25_paper_loop_smoke/predictions/baseline_result_xy.h5 \ + --candidate-prediction-path /Users/adamg/seg.bio/pytc-client/testing_projects/mito25_paper_loop_smoke/predictions/candidate_result_xy.h5 \ + --baseline-dataset vol0 \ + --candidate-dataset vol0 \ + --baseline-channel 0 \ + --candidate-channel 0 +``` + +For manual UI checks, use File Management -> `Mount Test Project`. The suggested +project card should report detected image, label, prediction, config, and +checkpoint roles before mounting. + +The smoke creates or stages voxel artifacts, drives the FastAPI workflow API through dataset loading, baseline inference capture, proofreading correction capture, correction staging, retraining completion capture, candidate inference capture, before/after evaluation, researcher-only readiness gates, and bundle export. + +What this proves: + +- Workflow events materialize typed artifacts, model runs, model versions, correction sets, evaluation results, hotspots, and bundle records. +- Correction-set records include edit and region counts derived from the + proofreading event stream when corrections are exported. +- Evidence bundle exports are recorded as `workflow.bundle_exported` audit + events. +- The before/after evaluation path computes real segmentation metrics over TIFF, HDF5, NumPy, Zarr/N5, and common 2D image files, with optional NIfTI/MRC support if their parser packages are installed. +- The researcher-only readiness/export path can produce an evidence bundle for planning a case-study walkthrough. +- In `real_pair_real_predictions` mode, supplied prediction artifacts are used for the metric comparison. +- PyTC channel-first/channel-last prediction outputs can be reduced to 3D voxel volumes with `--baseline-channel` and `--candidate-channel`. + +What this does not prove yet: + +- Real PyTC training subprocess execution. +- Real PyTC inference subprocess execution. +- Browser-level proofreading interaction inside EHTool. +- Real biomedical or connectomics sample-data performance in synthetic mode. +- Real model predictions in `real_pair_derived_predictions` mode; baseline/candidate masks are derived from the provided segmentation unless prediction paths are supplied. +- Fresh PyTC subprocess launch quality; app-generated prediction files can now be captured into workflow evidence, but launching/recovering long-running jobs still needs hardening. +- Long-running job queue behavior, retries, cancellation, or recovery. + +Outputs: + +- `smoke-report.json`: concise pass/fail summary and explicit simulated gaps. +- `workflow-bundle.json`: exported workflow evidence bundle. +- `readiness.json`: researcher-only readiness gate state. +- `evaluation-report.json`: before/after metric report. +- `artifacts/`: synthetic TIFFs, placeholder checkpoint, and training-output directory. diff --git a/docs/manual-bulk-test-checklist.md b/docs/manual-bulk-test-checklist.md new file mode 100644 index 00000000..c343c8da --- /dev/null +++ b/docs/manual-bulk-test-checklist.md @@ -0,0 +1,96 @@ +# Manual Bulk Test Checklist + +Use this after large prototype checkpoints when you want to verify the current +TOCHI-facing workflow without running full PyTC training. + +## Startup + +- From `/Users/adamg/seg.bio/pytc-client`, run `scripts/start.sh`. +- Confirm Electron opens and all three backend services report ready. +- Ignore `.zshrc` `pyenv` warnings for this app unless they block Python. + +## File Management + +- Open File Management. +- Confirm the workflow-agent strip is visible under the module tabs. +- Confirm its decision text is short and its primary action matches the current + stage. +- Confirm the suggested smoke project card appears. +- Confirm it reports image, label, prediction, config, and checkpoint roles. +- Click `Mount Test Project` or `Open Test Project`. +- Confirm `mito25-paper-loop-smoke` opens without manual path browsing. +- Confirm `data/image`, `data/seg`, `configs`, `checkpoints`, and `predictions` + are present. +- Ask the assistant `what should I do next?` after mounting. +- Confirm the workflow no longer behaves like no data is mounted: it should + recommend inference/visualization instead of asking you to mount data. +- Right-click the mounted root and confirm `Unmount Project` is available. + +## Evidence Panel + +- Click `Status` in the workflow-agent strip. +- Confirm the assistant drawer opens with workflow context visible. +- Open Tensorboard/Monitoring. +- Confirm `Review status` is visible. +- Confirm the `Loop progress` map is visible. +- Confirm the panel shows the next incomplete gate if the loop is not complete. +- Confirm previous result, new result, your saved edits, and reference-mask rows + use short user-facing labels. +- Confirm `Compare results` is disabled with a useful missing-input message when + previous result, new result, or reference mask paths are absent. +- Click `Metric options` only when dataset keys/channels need manual override. + +## Metric Computation + +- For HDF5/PyTC outputs, enter dataset key `vol0` for baseline/candidate. +- Use channel `0` for PyTC `result_xy.h5` probability maps when applicable. +- Use ground-truth dataset `data` for the curated mito25 segmentation file. +- Click `Compare results` only after previous-result, new-result, and reference paths are + recorded in workflow evidence. +- Confirm metric deltas render for Dice, IoU, and voxel accuracy. +- Confirm the latest comparison report path appears. + +## Bundle Export + +- Click `Export report`. +- Confirm the UI reports file/run/comparison counts and missing path count. +- Refresh evidence or open the workflow timeline/context panel. +- Confirm a `workflow.bundle_exported` event exists. + +## Proofreading Provenance + +- Open Mask Proofreading on a small loaded volume. +- Confirm the editor exposes a visible `Save mask` control, not only a keyboard + shortcut. +- Paint or erase a small region and confirm an `unsaved edits` indicator appears. +- Save at least two mask edits in distinct instances/regions. +- Export masks. +- Confirm the workflow evidence panel shows a corrected-mask path. +- Confirm the correction set reports nonzero edits and regions once evidence is + refreshed. + +## Agent Controls + +- Ask the assistant: `what should I do next?` +- Confirm the reply uses a compact `Do this` / `Why` / `Ready` style rather than + a long explanation. +- From inference stage with no prediction output, confirm the global agent + primary action is `Run model`. +- From a mounted project with an image/mask pair, click `Proofread this data` + and confirm the app opens Proofread and starts loading that pair without + manual path entry. +- From proofreading after a mask export, confirm the global agent primary action + is `Use edits for training`. +- Click `Use edits for training` and confirm a pending agent proposal appears + in workflow context/timeline instead of silently staging retraining. +- Approve the proposal and confirm the workflow moves to `retraining_staged` and + the corrected-mask path is loaded into training labels. + +## Known Boundaries + +- This checklist does not prove full training quality. +- This checklist does not prove long-running cancellation/retry/recovery. +- This checklist does not prove browser-level proofreading at scale. +- A paper-ready case-study loop still needs a fresh app-launched baseline + inference, correction export, retraining/fine-tuning, candidate inference, and + before/after evaluation over those fresh outputs. diff --git a/docs/proofreading-redesign-notes.md b/docs/proofreading-redesign-notes.md new file mode 100644 index 00000000..40debd38 --- /dev/null +++ b/docs/proofreading-redesign-notes.md @@ -0,0 +1,131 @@ +# Proofreading Redesign Notes + +Updated: 2026-04-25 + +## Log Findings From Latest Run + +- The largest runtime defect was not Electron rendering. The app log showed + `proofreading_instance_filmstrip_served` for `kind=mask_all`, `axis=xy`, + `z_start=48`, `z_count=12`, `max_dim=384` taking `25774.6 ms`, with nearly + all time reported inside the resize/render path. +- The root cause was `labels_to_rgba` assigning colors by looping over every + unique label and then scanning the full slice for each label. On dense + connectomics labels, this turns all-label overlays into a pathological + `unique_labels x pixels` operation. +- Mask save itself completed and wrote an artifact: + `/Users/adamg/seg.bio/testing_data/snemi/seg/.pytc_instance_labels.tif`. + The save request took about `946 ms`, changed 86 pixels, and recorded 2622 + blocked pixels. +- `/app/log-event` request bookkeeping was flooding the JSONL app log. Keeping + client events is useful; duplicating every client-log POST start/finish is + not. The middleware should suppress normal bookkeeping for that endpoint while + still logging slow or failed log submissions. + +## SOTA UI/Workflow Takeaways + +- CAVE frames connectomics proofreading as collaborative, versioned, + time-travelable annotation infrastructure rather than a static post-hoc edit + tool. Prototype implication: save/export state must be explicit and visible, + not hidden behind a transient canvas. Source: + https://www.nature.com/articles/s41592-024-02426-z +- CAVEclient describes CAVE as microservices for versioning connectomics data, + annotations, metadata, and segmentations. Prototype implication: our local + app should model persistent artifacts, corrections, and review decisions as + first-class records even before becoming CAVE-scale. Source: + https://www.caveconnecto.me/CAVEclient/ +- WEBKNOSSOS presents the expected connectomics flow as upload, preprocessing, + collaborative annotation, data management, segmentation, proofreading, and + connectome viewing. Prototype implication: the proofreading screen should make + queue, canvas, review decision, and export/persistence adjacent. Source: + https://home.webknossos.org/use-cases/connectomics +- Neuroglancer is the reference WebGL volumetric viewer, with an ecosystem + around precomputed format, CloudVolume, TensorStore, meshes, and scalable + downsampling. Prototype implication: PyTC Client should avoid pretending a + React canvas can become a petascale viewer; instead, optimize the local editor + and leave a clear future path to Neuroglancer-compatible data services. + Source: https://github.com/google/neuroglancer +- Guided proofreading work identifies visual search as a bottleneck and treats + proofreading as correcting split/merge errors while generating training data. + Prototype implication: a review queue and fast slice navigation matter more + than generic file-browser controls inside proofreading. Source: + https://openaccess.thecvf.com/content_cvpr_2018/papers/Haehn_Guided_Proofreading_of_CVPR_2018_paper.pdf +- MONAI Label treats annotation as an AI-assisted service with active learning, + 3D Slicer/OHIF frontends, and incremental model improvement. Prototype + implication: the right product loop is annotate/proofread, persist, retrain, + evaluate, then pick the next useful sample or region. Source: + https://www.sciencedirect.com/science/article/pii/S1361841524001324 +- Human-AI interaction guidance emphasizes conveying local uncertainty and + setting expectations at the instance level. Prototype implication: agent and + UI language should be short, local, and action-oriented, with uncertainty + exposed near the item being reviewed. Source: + https://www.microsoft.com/en-us/research/articles/how-to-build-effective-human-ai-interaction-considerations-for-machine-learning-and-software-engineering/ +- micro-SAM shows current microscopy annotation tools moving toward interactive + 2D/3D segmentation and tracking through napari. Prototype implication: + prompt-based correction is a good later feature, but the current priority is a + reliable edit/save/export loop. Source: + https://github.com/computational-cell-analytics/micro-sam + +## Current Redesign Decisions + +- Replace drawer/collapse navigation with a stable workbench: + review queue left, editor center, current action/save/export panel right. +- Keep case-study/researcher controls out of the participant proofreading UI. +- Treat "save mask" as a persistence operation with visible artifact/export + state, not just a canvas action. +- Use fast image-only previews during aggressive slider scrubbing, then fetch + richer overlays only when the slice is committed or the UI is idle. +- Vectorize label-to-color rendering so all-label context overlays are not + catastrophically slow on real segmentation volumes. +- Keep app-event logging dense around real proofreading actions, but avoid + self-noise from logging the log endpoint itself. + +## 2026-04-25 Canvas-First UI Reset + +- The screenshot from the Mito25 smoke project showed the workbench becoming a + nested form UI instead of a proofreading viewer: large header card, separate + slice rail, duplicate embedded layer toolbar, left queue, right action panel, + and a small canvas boxed inside all of it. +- The redesign now treats the central image as the primary object. Review + decisions, axis selection, active instance metadata, mask readiness, and export + access sit in the viewer header. +- The slice scrubber remains crucial, but it is now attached to the viewer + surface as a bottom dimension control with tick marks and a visible current + slice label. The always-open black tooltip was removed because it visually + fought the image. +- The embedded editor's own layer toolbar is hidden in workbench mode so there + is one slice-navigation source of truth. +- The queue no longer shows a redundant "Load dataset" button during an active + session. Dataset switching is promoted to the workbench header instead. +- The right panel is now secondary "Details" material: active object, saved + output, opacity controls, and export/staging. It should not carry the main + review decision loop. +- A follow-up compression fix made the canvas fit the available stage and + clamps zoom/pan so the image cannot collapse into a tiny off-canvas thumbnail + after resizing or scrolling. The queue also collapses sooner on narrower + windows. +- Research rationale and source links are recorded in + `docs/research/agentic-ui-design-pass.md`. + +## Remaining Follow-Up + +- Add a tile/chunk-backed viewer path for large volumes instead of repeatedly + encoding full PNG slices. +- Add explicit split/merge proofreading tools; current paint/erase is only a + useful local correction primitive. +- Add a hotspot/review queue populated by model uncertainty and error evidence, + not just instance IDs. +- Add export adapters for OME-Zarr/NGFF and Neuroglancer precomputed when moving + beyond the local TIFF prototype. +- Add browser-level manual tests that measure scrub latency, save latency, and + artifact persistence on the Mito25 and SNEMI sample projects. + +## 2026-04-25 Agent-Orchestrated Proofreading Entry + +- The global agent can now emit a `start_proofreading` runtime action with + image, mask, and review-name overrides. The proofreading tab consumes that + action and calls the same loader endpoint used by the manual form. +- The loader language now starts from "What should I proofread?" and offers a + one-click "Start proofreading this pair" path for detected image/mask pairs. +- The active workbench uses "Change data", "Focus view", "Review details", + "Saved edits", and "Use edits for training" so the screen reads like an + editing task rather than backend provenance management. diff --git a/docs/prototype-completion-roadmap.md b/docs/prototype-completion-roadmap.md new file mode 100644 index 00000000..ca5f4a61 --- /dev/null +++ b/docs/prototype-completion-roadmap.md @@ -0,0 +1,234 @@ +# Prototype Completion Roadmap + +This is the long-term project memory for moving from the current PyTC Client +toward the paper-implied prototype: an agent-mediated, human-controlled, +closed-loop biomedical segmentation workflow. + +## Current Verdict + +The project is not yet a paper-ready prototype of the system described by the +paper. The evidence substrate is now real enough to support structured smoke +walkthroughs: real mito25 image/seg data can enter the path, supplied PyTC +prediction files can be evaluated, completed app/PyTC inference runtimes can be +synced into workflow records, correction exports become correction-set records, +and the participant UI exposes the closed-loop pipeline, evidence artifacts, +metric controls, and bundle export. The remaining gap is still the full fresh +closed loop: app-launched baseline inference, real proofreading edit/export, +actual retraining/fine-tuning, post-training inference, and before/after +evaluation over fresh app-generated outputs. + +## Prototype Boundary + +- Participant UI should expose biomedical workflow state and controls, not + case-study protocol controls. +- Case-study readiness, protocol planning, and study gates stay researcher-only + in docs, backend routes, tests, and exported evidence bundles. +- Do not reintroduce case-study readiness bars or protocol-plan cards into the + participant-facing app. + +## Implemented Checkpoints + +- Typed workflow evidence records exist for events, artifacts, model runs, model + versions, correction sets, evaluation results, hotspots, agent plans, and + agent steps. +- Workflow bundle export exists for researcher/paper evidence capture. +- Researcher-only readiness gates exist and are covered by backend tests. +- Synthetic closed-loop smoke exists in `scripts/run_closed_loop_smoke.py`. +- Real-pair smoke exists for mito25 HDF5 image/seg pairs, using HDF5 dataset + keys and crop support. +- Real prediction input mode exists for supplied baseline/candidate prediction + artifacts. +- PyTC channel-first/channel-last prediction outputs can be selected with + explicit baseline/candidate channels in the smoke harness. +- Completed PyTC worker inference runtimes can be synchronized into workflow + `inference.completed` events, `ModelRun` records, and prediction artifacts. +- Closed-loop evidence is visible in the participant UI: baseline prediction, + candidate prediction, corrected mask, reference label/ground truth, latest + evaluation report, and metric deltas. +- The participant UI can compute before/after evaluation metrics from recorded + baseline, candidate, and reference volume paths when all required evidence is + present. +- The participant UI exposes dataset key, crop, and channel controls for HDF5 + and PyTC-style predictions that cannot be safely auto-detected. +- The participant UI shows an explicit closed-loop pipeline map with the next + incomplete gate. +- Evidence bundle export is available from the participant workflow evidence + panel and logs a `workflow.bundle_exported` event for auditability. +- A workflow-agent recommendation endpoint now returns a stage-aware decision, + readiness checklist, blockers, top hotspot, impact preview, app actions, and + command blocks from the typed workflow evidence substrate. +- The participant UI now has a compact workflow-agent control strip across + modules. Its primary button can navigate, start safe runtime handoffs, propose + retraining staging from corrected masks, refresh insights, and open the + assistant evidence/context drawer. +- Workflow chat now uses the same recommendation substrate for "what next" + questions and keeps answers compact: `Do this`, `Why`, `Ready`, and at most + one blocker. +- Workflow-agent chat now carries typed intent, permission-mode, risk metadata, + and a structured loop checklist. Slash-style workflow commands are supported + for status, inference, proofreading, training, comparison, and export. +- Assistant command details are collapsed behind a route disclosure so the + visible UI stays domain-oriented rather than command-oriented. +- Proofreading mask exports materialize correction sets with edit and region + counts derived from preceding proofreading events. +- File Management has a one-click suggested mito25 smoke-project mount. +- Suggested project mounting profiles local folders for likely image, label, + prediction, config, checkpoint, and pair roles before the user has to browse + manually. +- Suggested project mounting now registers detected workflow paths: dataset, + image, label/mask, prediction, and checkpoint paths are pushed into the + workflow session and logged as a `dataset.loaded` event. +- File Management now keeps the suggested project setup card on the root + project view only, reducing repeated clutter once the user is inside a + mounted project. +- The proofreading editor exposes a visible `Save mask` control, tracks unsaved + edits, and caps undo history to reduce memory churn. +- HDF5/Zarr/NumPy volume loading can select prediction channels while applying + crops, avoiding unnecessary full channel-stack reads for PyTC outputs. +- Shared voxel loading exists for TIFF/OME-TIFF, HDF5, NumPy, Zarr/N5, common + 2D image formats, and optional NIfTI/MRC parser packages. +- Evaluation can compute before/after metrics across supported voxel formats. +- Participant-facing case-study UI has been ablated. + +## Next Priority + +Run the first fresh app-generated before/after loop: + +- Launch baseline PyTC inference from the app on a mito25 crop and sync the + completed runtime. +- Perform a real proofreading edit/export that creates a correction set from + the browser/editor path. +- Run retraining or a bounded fine-tuning smoke from that correction set. +- Launch post-training inference, sync the completed runtime, and compute + before/after metrics over the two app-generated outputs. + +## Major Workstreams + +1. Evidence substrate + - Keep workflow records typed and queryable. + - Keep artifact lineage explicit. + - Keep evidence bundle export deterministic and paper-friendly. + +2. Real data ingestion + - Continue broad voxel-format support. + - Add OME-Zarr/NGFF and Neuroglancer-precomputed details when the workflow + needs lab-scale interoperability. + - Preserve crop/key support so large volumes are testable without loading + full datasets. + +3. Real closed-loop smoke + - Synthetic smoke is complete. + - Real image/seg smoke with derived predictions is complete. + - Real prediction input smoke is complete for externally supplied + predictions. + - Existing app-generated PyTC prediction files can be ingested as real + prediction artifacts. + - Fresh app-launched PyTC inference smoke is next. + +4. Actual PyTC inference/training loop + - Launch inference on real data. + - Capture inference output as a `ModelRun`. + - Proofread or edit a mask and persist a `CorrectionSet`. + - Stage corrections for retraining. + - Run retraining or fine-tuning. + - Register the candidate checkpoint as a `ModelVersion`. + - Run post-training inference. + - Compare before/after outputs with real metrics. + +5. Workflow UI, not case-study UI + - Expose workflow stage, artifacts, runs, corrections, evaluation metrics, + and agent proposals. + - Implemented UI hardening: direct evaluation-compute controls, closed-loop + pipeline map, evidence bundle export, and suggested smoke-project mounting + without adding researcher case-study protocol gates. + - Implemented agent continuity: the global workflow-agent strip exposes the + next recommended workflow action and a one-click evidence/context view. + - Next UI hardening: expose model versions/config lineage more directly, + make file-role assignment editable when auto-detection guesses wrong, and + make the file/project mounting flow feel like guided biomedical dataset + setup rather than a generic file browser. + - Keep approval/rejection controls for risky agent actions. + - Refine workflow rail placement and visibility. + - Do not expose researcher protocol gates to participants. + +6. Agentic bounded autonomy + - Agent proposes actions; it does not silently execute risky steps. + - Agent role is formalized in `docs/agent-role-spec.md`. + - Claude Code pattern notes live in + `docs/research/claude-code-agent-patterns.md`. + - Implemented agent recommendation substrate: stage-aware decision, blockers, + readiness, top hotspot, impact preview, and app-executable actions. + - Implemented cross-module agent controls: open files, launch inference from + current settings, open proofreading, propose retraining handoff, prime + training labels, launch staged training, and open evidence context. + - UI simplification pass ablated visible per-module agent nudges and the + persistent next-step strip. The agent remains accessible through the chat + drawer and normal module buttons where needed. + - All chatbot agents should default to short, biologist-skimmable answers: + next action first, three bullets or fewer, and no internal implementation + details unless explicitly requested. + - Approval, rejection, interruption, and resume decisions must be auditable. + - Use LangGraph where durable state-machine behavior, checkpoints, and + interrupt/resume semantics become real execution requirements. + - Use LangChain only for model/tool wrappers and simple retrieval glue. + - Next agentic-system hardening: durable job/task state, approval modes, + workflow hooks/gates, focused biomedical subagents, and context compaction + for long assistant sessions. + +7. Evaluation and provenance + - Compute before/after segmentation metrics over real outputs. + - Link metrics to exact prediction, ground-truth, checkpoint, correction, and + config artifacts. + - Dataset key, crop, and prediction-channel selectors are implemented in the + workflow evidence panel. + - Bundle export is auditable through a workflow event, but a persisted + downloadable bundle directory/zip is still needed for paper artifact + packaging. + - Export paper-ready bundles with events, artifacts, runs, versions, + corrections, metrics, and figure inputs. + +8. Interop and SOTA relevance + - Align with connectomics/bioimage formats and workflows: HDF5, OME-Zarr/NGFF, + N5/Zarr, Neuroglancer-compatible artifacts, and eventual CAVE/Neuroglancer + relevance. + - Evaluate relevant model ecosystems when needed: PyTorch Connectomics, + MONAI, nnU-Net, Cellpose/Cellpose-SAM, micro-SAM, MedSAM/SAM2 variants, + StarDist, PlantSeg, DeepCell/Mesmer. + +9. Engineering hardening + - Replace process-global runtime state with durable job records. + - Add a real job queue for long-running inference/training. + - Add cancellation, retries, failure recovery, and resumable execution. + - Link TensorBoard, MLflow, W&B, or equivalent run tracking to workflow + records where useful. + - Make file management and project mounting substantially more user-friendly: + guided project setup, obvious mounted roots, automatic image/seg/config/ + checkpoint detection, duplicate/stale mount cleanup, better picker labels, + one-click test-project mounting for smoke workflows, and editable role + confirmation when auto-detected project roles are wrong. Current progress: + suggested projects are profiled and registered into workflow state; missing + work is an editable role-confirmation surface before registration. + +10. Paper alignment + - Maintain the paper-claim-vs-implementation matrix. + - Maintain a paper/prototype additions matrix. + - Keep paper claims honest until the real closed loop works. + - Refresh the claim wording after each prototype milestone. + +## Paper-Claim Risk Register + +| Claim Area | Current Status | Overclaim Risk | Required Evidence | +| --- | --- | --- | --- | +| Closed-loop improvement | Backend/evaluation path and external prediction input mode exist, real app-generated model loop missing | High | Real app-generated baseline inference, correction, retraining, candidate inference, before/after metrics | +| Human-controlled agent workflow | Proposal/approval substrate exists | Medium | Visible participant workflow controls and auditable approvals over real actions | +| Biomedical/connectomics relevance | Mito25 data ingestion works | Medium | Real inference/proofreading/retraining on mito/connectomics artifacts | +| Bounded autonomy | Researcher-only plan/proposal records exist | Medium | Durable interrupt/resume execution if claimed as operational | +| Reproducible evidence export | Bundle exists | Low/Medium | Bundle generated from real workflow artifacts and paper figures | +| SOTA alignment | Research notes/plans exist | Medium | Explicit comparison/baseline or clear positioning against relevant systems | + +## Researcher-Only Notes + +- Case-study protocol details live in `docs/case-study-prototype-readiness.md`. +- Smoke execution details live in `docs/closed-loop-smoke.md`. +- Manual bulk-test steps live in `docs/manual-bulk-test-checklist.md`. +- Evidence export details live in `docs/research/workflow-evidence-export.md`. diff --git a/docs/research/agentic-ui-design-pass.md b/docs/research/agentic-ui-design-pass.md new file mode 100644 index 00000000..e6a9ec96 --- /dev/null +++ b/docs/research/agentic-ui-design-pass.md @@ -0,0 +1,93 @@ +# Agentic UI Design Pass + +Updated: 2026-04-25 + +## Scope + +This note records the design pass behind the mask proofreading reset. The goal +is not to make PyTC Client look like a generic AI chat app. The goal is to make +the biologist-facing workflow behave like a scientific image viewer with bounded +agent assistance layered around the work, not competing with it. + +## Sources Checked + +- Microsoft Guidelines for Human-AI Interaction: + https://www.microsoft.com/en-us/research/project/guidelines-for-human-ai-interaction/2025-1-31/ +- Magentic-UI human-in-the-loop agentic systems: + https://arxiv.org/abs/2507.22358 +- WEBKNOSSOS proofreading workflows: + https://docs.webknossos.org/webknossos/proofreading/index.html +- WEBKNOSSOS connectomics workflow positioning: + https://home.webknossos.org/use-cases/connectomics +- Neuroglancer/PyTorch Connectomics viewer integration notes: + https://connectomics.readthedocs.io/en/latest/external/neuroglancer.html +- 3D Slicer Segment Editor: + https://slicer.readthedocs.io/en/5.8/user_guide/modules/segmenteditor.html +- napari viewer layout and dimension slider pattern: + https://napari.org/stable/tutorials/fundamentals/viewer.html +- Nature 2025 connectomic reconstruction manual proofreading workflow: + https://www.nature.com/articles/s41586-025-08985-1 + +## Design Principles For This App + +- Canvas first. The active image/mask view is the work object; controls should + orbit it rather than push it into a small nested card. +- One navigation source of truth. A volume viewer should not show duplicate + layer headers, duplicate slice controls, and duplicate next/previous controls + in competing places. +- Keep state local to the action. The current slice, active instance, edit + readiness, save state, and review decision should be visible next to the + canvas, not hidden in a generic side panel. +- Use progressive disclosure. Export paths, provenance details, and staging + controls matter, but they are secondary to inspect/edit/review/save. +- Preserve expert imaging patterns. napari and Slicer both keep slice navigation + tightly coupled to the image view; Slicer also treats keyboard-driven slice + movement and tool switching as core editing affordances. +- Bound agent behavior. Agent help should surface short, local suggestions and + never obscure the segmentation canvas or overload the user with long prose. +- Avoid case-study-specific participant UI. Study instrumentation belongs in + logs/research exports, not as participant-facing controls in the prototype. + +## Current Proofreading UI Reset Decisions + +- Replace the heavy top card with a compact workbench header. +- Keep the left review queue, but remove the redundant "Load dataset" action + from inside the active proofreading queue. +- Move review decisions into the viewer header so the user can inspect and + classify without moving attention to a separate panel. +- Hide the embedded editor's duplicate layer toolbar when used inside the + proofreading workbench. +- Attach the slice slider directly to the viewer surface and keep tick marks and + an explicit current-slice label. +- Keep export, overlay opacity, and artifact paths in a narrower details panel + rather than making them the primary workflow. + +## Remaining UI/Agent Work + +- Agent messages need a short "biologist skim" format: conclusion first, + 1-3 bullets, explicit action or no action. +- File/project mounting needs a separate flow redesign. The current path-based + mounting flow is still too manual and error-prone. +- The editor still needs a more complete split/merge-oriented tool model; paint + and erase are local correction primitives, not connectomics-grade proofreading. +- The canvas should eventually move toward tiled/chunked rendering for large + volumes rather than repeated full-slice PNG transfer. + +## 2026-04-25 Product-Language/Orchestrator Update + +- The UI problem is broader than density: the app was exposing research/backend + bookkeeping ("evidence", "artifacts", "candidate", "correction set") where a + biologist expects task language ("previous result", "new result", "your + edits", "reference mask"). +- The agent strip should be a workflow orchestrator, not a second dashboard. + It now emphasizes one next action plus a small status drawer; readiness counts + and path details should not compete with the user's image/mask task. +- "Proofread this data" is now an executable client effect, not just navigation: + the backend emits a `start_proofreading` runtime action and the frontend loads + the current image/mask pair into the proofreading workbench when available. +- Advanced metric dataset/channel fields are hidden behind "Metric options". + This follows progressive disclosure: common user action first, technical + disambiguation only when HDF5/channel details are required. +- Remaining design debt: turn the whole app into an intent ladder + (`choose data -> run model -> proofread -> train on edits -> compare`) rather + than separate modules that happen to share workflow state. diff --git a/docs/research/claude-code-agent-patterns.md b/docs/research/claude-code-agent-patterns.md new file mode 100644 index 00000000..aec19eab --- /dev/null +++ b/docs/research/claude-code-agent-patterns.md @@ -0,0 +1,130 @@ +# Claude Code Agent Pattern Pass + +Date: 2026-04-25 + +This note records what is useful for PyTC Client from the Claude Code source +inspection plus a quick literature/blog/documentation pass. The goal is not to +clone a coding CLI. The goal is to borrow agent-system patterns that make the +assistant a real biomedical segmentation workflow orchestrator. + +## Sources Reviewed + +- Local source inspection of `https://github.com/tanbiralam/claude-code/tree/main/src`. + Treat this repo as architectural inspiration only; do not copy large code or + user-facing strings. +- Anthropic Claude Code overview: agentic tool can read, edit, run commands, + integrate tools, use MCP, customize instructions, run subagents, schedule + tasks, and work across surfaces. +- Anthropic best practices: verify work, explore before plan before execution, + keep context small, use skills for reusable workflows, hooks for deterministic + checks, subagents for investigation, and permission allowlists/sandboxing for + safety. +- Claude Code permissions docs: tools are tiered by risk; read-only actions need + no approval while shell/file-changing actions do, and modes such as `plan`, + `auto`, and `dontAsk` define execution boundaries. +- Claude Code subagents docs and blog: use focused agents when a side task + would flood the main conversation; restrict their tools; return summaries + rather than raw logs; use hooks for lifecycle gates. +- Jason Liu context-engineering note: slash commands keep recurring workflows + focused; subagents keep verbose side work out of the main context. +- Liu et al. 2026 arXiv tech report on Claude Code design space: the simple + model-tool loop is only the center; the important system work lives in + permissions, compaction, extensibility, subagents, isolation, and append-only + session storage. + +## Source-Level Patterns Worth Copying + +1. Tool-first orchestration + - Claude Code models actions as typed tools with input/output schemas, + permission checks, concurrency rules, progress state, and result mapping. + - PyTC equivalent: every app action from chat should be typed as a workflow + action with risk metadata: navigation, form prefill, editor load, model + run, training run, metric computation, export, retraining proposal. + +2. Permission and risk are first-class data + - Claude Code separates read-only tools from risky tools and routes each + through a permission layer. + - PyTC equivalent: show whether an agent action is `view only`, `opens + editor`, `runs job`, `writes record`, or `exports`; never hide the approval + boundary. + +3. Slash commands are compact workflow entrypoints + - Claude Code uses slash commands and skills for repeatable work. + - PyTC equivalent: support `/status`, `/infer`, `/proofread`, `/train`, + `/compare`, `/export`, and `/help` as deterministic workflow commands. + +4. Task/checklist substrate beats free-form progress prose + - Claude Code has task/todo tools and structured background task state. + - PyTC equivalent: the workflow agent should expose a machine-readable loop + checklist from readiness gates and record actions as workflow events. + +5. Ask structured questions when intent is ambiguous + - Claude Code has an `AskUserQuestion` tool with short labels and bounded + options. + - PyTC equivalent: vague prompts like "I wanna..." should offer 2-4 concrete + app actions, not a lecture or hidden prompt scaffolding. + +6. Context hygiene is a runtime feature, not just prompt discipline + - Claude Code has compaction, tool-result budgeting, summaries, and + subagents/forks to prevent context floods. + - PyTC equivalent: agent responses stay short; heavy evidence stays in the + evidence panel/bundle; raw paths and logs should be expandable details. + +7. Verification gates matter + - Claude Code’s verification-agent pattern is adversarial: run checks, do not + merely read code. + - PyTC equivalent: after model/proofread/train/evaluate steps, the app needs + acceptance checks over artifacts, metrics, and workflow events. + +## Immediate Prototype Changes From This Pass + +- Add workflow-agent response metadata: `intent`, `permission_mode`, and + structured `tasks`. +- Add action/command risk metadata: `read_only`, `prefills_form`, + `loads_editor`, `runs_job`, `writes_workflow_record`, `exports_evidence`. +- Keep command pseudo-code collapsed behind a `Route` disclosure so biologists + see the app action first, not implementation text. +- Add deterministic slash aliases in the server and UI logging path: + `/status`, `/next`, `/help`, `/infer`, `/segment`, `/proofread`, `/train`, + `/compare`, `/metrics`, `/export`. + +## Next Agent-System Work + +- Add a real workflow task table or durable job table for long-running + inference/training instead of relying on transient runtime action state. +- Add explicit approval modes: `ask`, `auto_read_only`, `plan_only`, and + `blocked`, mapped to the biomedical workflow rather than copied literally + from coding. +- Add hooks/gates: + - Before training: require corrected mask path and correction-set record. + - Before evaluation: require baseline, candidate, and reference volumes. + - Before claiming improvement: require metrics and artifact lineage. + - Before export: verify bundle contains workflow events and key artifacts. +- Add focused biomedical subagents only when they become real: + - `data-curator`: inspect mounted project and identify image/mask/checkpoint + roles. + - `model-runner`: prepare and launch inference/training jobs. + - `proofreading-triage`: rank likely failure regions and produce review queue. + - `evidence-auditor`: verify artifact lineage before paper figures/export. +- Add context compaction for the assistant drawer: summarize old chat turns and + keep workflow state/event evidence as structured context. + +## Design Constraint For PyTC + +Claude Code’s user is a developer who can read commands. PyTC’s user is a +biologist or biomedical-image researcher trying to complete a segmentation +loop. Therefore the visible surface should say "Run model", "Proofread this +data", "Use edits for training", and "Compare results"; command-like details +belong behind a disclosure or in logs. + +## References + +- [Claude Code overview](https://code.claude.com/docs/en/overview) +- [Claude Code best practices](https://code.claude.com/docs/en/best-practices) +- [Claude Code permissions](https://code.claude.com/docs/en/permissions) +- [Claude Code subagents](https://code.claude.com/docs/en/sub-agents) +- [Claude Code hooks](https://code.claude.com/docs/en/hooks) +- [Claude Code skills/slash commands](https://code.claude.com/docs/en/slash-commands) +- [How and when to use subagents in Claude Code](https://claude.com/blog/subagents-in-claude-code) +- [Slash Commands vs Subagents](https://jxnl.co/writing/2025/08/29/context-engineering-slash-commands-subagents/) +- [Dive into Claude Code: The Design Space of Today's and Future AI Agent Systems](https://arxiv.org/abs/2604.14228) diff --git a/docs/research/tochi-hci-prototype-takeaways.md b/docs/research/tochi-hci-prototype-takeaways.md new file mode 100644 index 00000000..3c1dbf51 --- /dev/null +++ b/docs/research/tochi-hci-prototype-takeaways.md @@ -0,0 +1,81 @@ +# TOCHI/HCI Prototype Takeaways + +Purpose: keep implementation-facing notes from a quick TOCHI/HCI systems pass +while turning PyTC Client into a paper-defensible, human-controlled, +closed-loop biomedical segmentation prototype. + +## Sources Read + +- Dow et al., "Parallel Prototyping Leads to Better Design Results, More + Divergence, and Increased Self-Efficacy", ACM TOCHI 17(4), 2010. + +- Microsoft Research, "Guidelines for Human-AI Interaction" publication page: + CHI 2019 guidelines and a TOCHI 2022 follow-up on early assessment. + +- CHI 2025 Case Studies of HCI in Practice page, including recent examples of + practice-facing case studies and pilot case-study framing. + +- CHI 2026 Human-Agent Collaboration workshop call, especially mixed-initiative + workflows, transparency, failure recovery, and shared accountability. + +- Adobe Research CHI 2025 summary on compositional structures for human-AI + co-creation, emphasizing workflow-spanning prototypes, context, inspection, + and control. + + +## Implementation Takeaways + +- A TOCHI-grade prototype should not only "do the task"; it should make the + interaction logic inspectable. For this project, that means visible workflow + state, artifacts, metrics, agent proposals, approvals, and exported evidence. +- Parallel-prototyping logic argues against a single opaque model result. The + UI should support comparing baseline/candidate predictions and before/after + metrics side by side, rather than only showing the latest output. +- Human-AI interaction guidelines push for calibrated expectations, visible + uncertainty/failure, efficient correction, undo/dismissal, and clear handoff. + For PyTC Client, the agent must show what it will do, what evidence it used, + and whether it is blocked by missing files, long-running jobs, or failed + metrics. +- Human-agent collaboration work stresses mutual awareness and shared + accountability. The prototype should preserve an audit trail of agent plans, + user approvals/rejections, runtime events, and artifact lineage. +- Recent workflow-spanning human-AI creation systems treat AI as embedded in + multiple structures, not as a chat box bolted onto one screen. Here, the + assistant should connect setup, visualization, inference, proofreading, + retraining, evaluation, and evidence export through the workflow substrate. +- Case-study-facing HCI work needs evidence that is reproducible and + inspectable. Participant UI should stay focused on the biomedical workflow; + researcher case-study gates should remain in docs/tests/export bundles. + +## Prototype Implications + +- Keep the `Closed-loop Evidence` panel as the central participant-facing + evidence surface. It should show baseline/candidate predictions, correction + artifacts, reference labels, metric deltas, report paths, and bundle export + status. +- Add explicit controls whenever auto-detection can silently fail. Dataset keys, + crop windows, and channel selectors are necessary for HDF5/PyTC outputs. +- Treat long-running inference/training as accountable jobs, not transient UI + state. The next hardening phase should add durable job records, cancellation, + retry, and recovery. +- Make project setup boring. Suggested project mounting and auto-detected data + roles reduce demo friction and prevent participants from spending study time + debugging paths. +- Do not claim autonomous closed-loop improvement until the app can perform: + baseline inference, proofreading export, retraining/fine-tuning, candidate + inference, metric computation, and evidence bundle export using app-generated + artifacts. + +## Additions To Consider For The Paper + +- Frame the system as a workflow evidence substrate plus bounded agentic + control layer, not only as a GUI around PyTorch Connectomics. +- Include a figure that shows artifact lineage: source image/label, baseline + prediction, correction set, retraining run, model version, candidate + prediction, evaluation result, evidence bundle. +- Include a design rationale section grounded in human-AI interaction: + calibrated autonomy, visible provenance, interruptible/approvable actions, + and closed-loop comparison. +- If case studies are used, separate participant task flow from researcher + readiness gates. The latter supports rigor but should not appear as + participant-facing protocol UI. From d16987ada8d0f0a46a75b46b7b1c8eb4ded7dcf8 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Sun, 26 Apr 2026 09:12:12 -0400 Subject: [PATCH 02/38] feat: add closed-loop workflow evidence spine --- app_event_logger.py | 196 ++ pyproject.toml | 1 + scripts/dev.sh | 6 +- scripts/run_closed_loop_smoke.py | 831 +++++ scripts/start.sh | 62 +- scripts/summarize_app_log.py | 96 + server_api/main.py | 721 +++- server_api/utils/utils.py | 59 +- server_api/workflows/agent_plan.py | 316 ++ server_api/workflows/bundle_export.py | 65 +- server_api/workflows/db_models.py | 253 +- server_api/workflows/evaluation.py | 168 + server_api/workflows/router.py | 3096 +++++++++++++++++- server_api/workflows/service.py | 625 +++- server_api/workflows/volume_io.py | 475 +++ server_pytc/main.py | 118 +- server_pytc/services/model.py | 1244 ++++++- tests/test_app_event_logger.py | 48 + tests/test_closed_loop_smoke_script.py | 166 + tests/test_pytc_runtime_routes.py | 375 ++- tests/test_workflow_artifact_records.py | 347 ++ tests/test_workflow_case_study_acceptance.py | 214 ++ tests/test_workflow_routes.py | 333 +- tests/test_workflow_spine_smoke.py | 30 +- uv.lock | 4 +- 25 files changed, 9622 insertions(+), 227 deletions(-) create mode 100644 app_event_logger.py create mode 100644 scripts/run_closed_loop_smoke.py create mode 100644 scripts/summarize_app_log.py create mode 100644 server_api/workflows/agent_plan.py create mode 100644 server_api/workflows/evaluation.py create mode 100644 server_api/workflows/volume_io.py create mode 100644 tests/test_app_event_logger.py create mode 100644 tests/test_closed_loop_smoke_script.py create mode 100644 tests/test_workflow_artifact_records.py create mode 100644 tests/test_workflow_case_study_acceptance.py diff --git a/app_event_logger.py b/app_event_logger.py new file mode 100644 index 00000000..e074d7cc --- /dev/null +++ b/app_event_logger.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import json +import logging +import os +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +_ROOT_DIR = Path(__file__).resolve().parent +_DEFAULT_LOG_PATH = _ROOT_DIR / ".logs" / "app" / "app-events.jsonl" +_ORIGINAL_STDOUT = sys.stdout +_ORIGINAL_STDERR = sys.stderr +_WRITE_LOCK = threading.Lock() +_CONFIG_LOCK = threading.Lock() +_CONFIGURED_COMPONENTS: set[str] = set() +_STDIO_REDIRECTED = False + + +def get_app_event_log_path() -> Path: + raw_path = os.getenv("PYTC_APP_EVENT_LOG_PATH", "").strip() + if raw_path: + return Path(raw_path).expanduser().resolve(strict=False) + return _DEFAULT_LOG_PATH + + +def _detect_stream_level(default_level: str, message: str) -> str: + text = (message or "").strip() + upper = text.upper() + if upper.startswith("INFO:") or " INFO:" in upper: + return "INFO" + if upper.startswith("WARNING:") or upper.startswith("WARN:") or " WARNING:" in upper: + return "WARNING" + if upper.startswith("DEBUG:") or " DEBUG:" in upper: + return "DEBUG" + if upper.startswith("ERROR:") or " ERROR:" in upper: + return "ERROR" + if "TRACEBACK" in upper or "EXCEPTION" in upper or "FAILED" in upper: + return "ERROR" + return default_level.upper() + + +def _normalize_value(value: Any) -> Any: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, Path): + return str(value) + if isinstance(value, dict): + return {str(key): _normalize_value(val) for key, val in value.items()} + if isinstance(value, (list, tuple, set)): + return [_normalize_value(item) for item in value] + return str(value) + + +def append_app_event( + *, + component: str, + event: str, + level: str = "INFO", + message: str | None = None, + **fields: Any, +) -> dict[str, Any]: + log_path = get_app_event_log_path() + log_path.parent.mkdir(parents=True, exist_ok=True) + + record = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "pid": os.getpid(), + "component": component, + "event": event, + "level": level.upper(), + "message": message or "", + } + record.update( + { + key: _normalize_value(value) + for key, value in fields.items() + if value is not None + } + ) + + with _WRITE_LOCK: + with log_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(record, ensure_ascii=True) + "\n") + + return record + + +class _AppEventStream: + def __init__( + self, + *, + component: str, + stream_name: str, + level: str, + original_stream, + ) -> None: + self.component = component + self.stream_name = stream_name + self.level = level + self.original_stream = original_stream + self._buffer = "" + + def write(self, data) -> int: + text = str(data) + if not text: + return 0 + + self.original_stream.write(text) + self._buffer += text + + while "\n" in self._buffer: + line, self._buffer = self._buffer.split("\n", 1) + if line.strip(): + append_app_event( + component=self.component, + event=f"{self.stream_name}_line", + level=_detect_stream_level(self.level, line), + message=line, + stream=self.stream_name, + ) + + return len(text) + + def flush(self) -> None: + self.original_stream.flush() + if self._buffer.strip(): + append_app_event( + component=self.component, + event=f"{self.stream_name}_line", + level=_detect_stream_level(self.level, self._buffer), + message=self._buffer, + stream=self.stream_name, + ) + self._buffer = "" + + def isatty(self) -> bool: + return bool(getattr(self.original_stream, "isatty", lambda: False)()) + + def fileno(self) -> int: + return self.original_stream.fileno() + + +def configure_process_logging(component: str) -> Path: + global _STDIO_REDIRECTED + + with _CONFIG_LOCK: + if component in _CONFIGURED_COMPONENTS: + return get_app_event_log_path() + + if not _STDIO_REDIRECTED: + sys.stdout = _AppEventStream( + component=component, + stream_name="stdout", + level="INFO", + original_stream=_ORIGINAL_STDOUT, + ) + sys.stderr = _AppEventStream( + component=component, + stream_name="stderr", + level="ERROR", + original_stream=_ORIGINAL_STDERR, + ) + _STDIO_REDIRECTED = True + + root_logger = logging.getLogger() + root_logger.setLevel(logging.INFO) + if not root_logger.handlers: + logging.basicConfig( + level=logging.INFO, + stream=sys.stdout, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + else: + for handler in root_logger.handlers: + if not isinstance(handler, logging.StreamHandler): + continue + stream = getattr(handler, "stream", None) + if stream is _ORIGINAL_STDOUT: + handler.setStream(sys.stdout) + elif stream is _ORIGINAL_STDERR: + handler.setStream(sys.stderr) + + append_app_event( + component=component, + event="process_logging_configured", + level="INFO", + message=f"{component} logging configured", + log_path=str(get_app_event_log_path()), + configured_at_ms=round(time.time() * 1000), + ) + _CONFIGURED_COMPONENTS.add(component) + return get_app_event_log_path() diff --git a/pyproject.toml b/pyproject.toml index 283f740d..d4bc1bd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "uvicorn==0.38.0", "zarr>=2.18", "connectomics", + "langgraph>=1.0.0", ] [tool.uv.sources] diff --git a/scripts/dev.sh b/scripts/dev.sh index 455ed869..60398630 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -4,8 +4,9 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CLIENT_DIR="${ROOT_DIR}/client" -DEFAULT_OLLAMA_BASE_URL="http://cscigpu08.bc.edu:4443" -DEFAULT_OLLAMA_MODEL="llama3.1:8b" +DEFAULT_OLLAMA_BASE_URL="http://127.0.0.1:11434" +DEFAULT_OLLAMA_MODEL="llama3.2:1b" +DEFAULT_OLLAMA_EMBED_MODEL="nomic-embed-text:latest" if ! command -v uv >/dev/null 2>&1; then echo "uv is required. Run scripts/bootstrap.sh first." >&2 @@ -44,6 +45,7 @@ DATA_SERVER_PID=$! echo "Starting API server (port 4242)..." OLLAMA_BASE_URL="${OLLAMA_BASE_URL:-${DEFAULT_OLLAMA_BASE_URL}}" \ OLLAMA_MODEL="${OLLAMA_MODEL:-${DEFAULT_OLLAMA_MODEL}}" \ + OLLAMA_EMBED_MODEL="${OLLAMA_EMBED_MODEL:-${DEFAULT_OLLAMA_EMBED_MODEL}}" \ PYTHONDONTWRITEBYTECODE=1 uv run --directory "${ROOT_DIR}" python -m server_api.main & API_PID=$! diff --git a/scripts/run_closed_loop_smoke.py b/scripts/run_closed_loop_smoke.py new file mode 100644 index 00000000..ed810274 --- /dev/null +++ b/scripts/run_closed_loop_smoke.py @@ -0,0 +1,831 @@ +#!/usr/bin/env python3 +"""Run a local closed-loop workflow smoke against the FastAPI workflow API. + +This harness is deliberately honest about scope: it exercises workflow +materialization, metrics, bundle export, and researcher-readiness gates with +synthetic TIFF artifacts. It does not run PyTC training, PyTC inference, or the +browser proofreading editor. +""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import sys +import tempfile +from pathlib import Path +from typing import Any, Dict, Optional + +import numpy as np +import tifffile + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from server_api.workflows.volume_io import ( # noqa: E402 + SUPPORTED_VOLUME_FORMATS, + load_volume, + parse_crop, +) + + +def _write_json(path: Path, payload: Dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def _request_json(response, label: str) -> Dict[str, Any]: + if response.status_code >= 400: + raise RuntimeError(f"{label} failed: {response.status_code} {response.text}") + return response.json() + + +def _post_event( + client, + *, + workflow_id: int, + event_type: str, + stage: str, + summary: str, + payload: Optional[Dict[str, Any]] = None, + actor: str = "system", +) -> Dict[str, Any]: + return _request_json( + client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": actor, + "event_type": event_type, + "stage": stage, + "summary": summary, + "payload": payload or {}, + }, + ), + f"post {event_type}", + ) + + +def generate_smoke_artifacts(output_dir: Path) -> Dict[str, str]: + artifact_dir = output_dir / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + + volume_shape = (4, 32, 32) + z, y, x = np.indices(volume_shape) + image = ((x * 5 + y * 3 + z * 17) % 255).astype(np.uint8) + + ground_truth = np.zeros(volume_shape, dtype=np.uint8) + ground_truth[:, 8:22, 9:23] = 1 + ground_truth[1:3, 18:27, 4:13] = 2 + + baseline = np.zeros_like(ground_truth) + baseline[:, 10:20, 11:21] = 1 + baseline[1:3, 20:25, 6:11] = 2 + + corrected = ground_truth.copy() + candidate = ground_truth.copy() + + paths = { + "image": artifact_dir / "image.tif", + "initial_mask": artifact_dir / "initial-mask.tif", + "corrected_mask": artifact_dir / "corrected-mask.tif", + "ground_truth": artifact_dir / "ground-truth.tif", + "baseline_prediction": artifact_dir / "baseline-prediction.tif", + "candidate_prediction": artifact_dir / "candidate-prediction.tif", + "training_output": artifact_dir / "training-output", + "checkpoint": artifact_dir / "checkpoint-smoke.pth", + "evaluation_report": output_dir / "evaluation-report.json", + } + + tifffile.imwrite(str(paths["image"]), image, photometric="minisblack") + tifffile.imwrite(str(paths["initial_mask"]), baseline, photometric="minisblack") + tifffile.imwrite(str(paths["corrected_mask"]), corrected, photometric="minisblack") + tifffile.imwrite(str(paths["ground_truth"]), ground_truth, photometric="minisblack") + tifffile.imwrite( + str(paths["baseline_prediction"]), baseline, photometric="minisblack" + ) + tifffile.imwrite( + str(paths["candidate_prediction"]), candidate, photometric="minisblack" + ) + paths["training_output"].mkdir(parents=True, exist_ok=True) + paths["checkpoint"].write_text("synthetic checkpoint placeholder\n", encoding="utf-8") + + return {key: str(value) for key, value in paths.items()} + + +def _write_tiff(path: Path, volume: np.ndarray) -> str: + tifffile.imwrite(str(path), np.asarray(volume), photometric="minisblack") + return str(path) + + +def _apply_crop_to_array(array: np.ndarray, crop: Optional[str]) -> np.ndarray: + crop_slices = parse_crop(crop) + if crop_slices is None: + return np.asarray(array) + if len(crop_slices) > array.ndim: + raise ValueError( + f"Crop has {len(crop_slices)} dimensions but prediction array has " + f"{array.ndim}" + ) + key = tuple(list(crop_slices) + [slice(None)] * (array.ndim - len(crop_slices))) + return np.asarray(array[key]) + + +def _select_prediction_channel( + array: np.ndarray, + *, + channel: Optional[int], + reference_ndim: int, + label: str, +) -> np.ndarray: + volume = np.asarray(array) + if volume.ndim == reference_ndim: + if channel is not None: + raise ValueError( + f"{label} prediction is already {reference_ndim}D; " + "do not pass a channel selector for this artifact." + ) + return volume + if volume.ndim != reference_ndim + 1: + return volume + + if channel is not None and channel < 0: + raise ValueError(f"{label} channel must be non-negative") + + if channel is None: + if volume.shape[0] <= 16: + axis = 0 + elif volume.shape[-1] <= 16: + axis = -1 + else: + raise ValueError( + f"{label} prediction has shape {volume.shape}; pass an explicit " + "channel because no small channel axis could be inferred." + ) + channel = 0 + elif volume.shape[0] > channel and volume.shape[0] <= 16: + axis = 0 + elif volume.shape[-1] > channel and volume.shape[-1] <= 16: + axis = -1 + elif volume.shape[0] > channel: + axis = 0 + elif volume.shape[-1] > channel: + axis = -1 + else: + raise ValueError( + f"{label} channel {channel} is out of bounds for prediction shape " + f"{volume.shape}" + ) + + return np.take(volume, channel, axis=axis) + + +def _load_prediction_volume( + path: str, + *, + dataset_key: Optional[str], + crop: Optional[str], + channel: Optional[int], + reference_ndim: int, + label: str, +) -> np.ndarray: + raw = load_volume(str(path), dataset_key=dataset_key) + selected = _select_prediction_channel( + raw, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + return _apply_crop_to_array(selected, crop) + + +def _derived_baseline_from_mask(mask: np.ndarray) -> np.ndarray: + baseline = np.asarray(mask).copy() + if baseline.size == 0: + return baseline + last_axis = baseline.ndim - 1 + cutoff = max(1, baseline.shape[last_axis] // 4) + index = [slice(None)] * baseline.ndim + index[last_axis] = slice(0, cutoff) + baseline[tuple(index)] = 0 + return baseline + + +def generate_real_pair_artifacts( + output_dir: Path, + *, + image_path: str, + mask_path: str, + image_dataset: Optional[str] = None, + mask_dataset: Optional[str] = None, + baseline_prediction_path: Optional[str] = None, + candidate_prediction_path: Optional[str] = None, + baseline_dataset: Optional[str] = None, + candidate_dataset: Optional[str] = None, + baseline_channel: Optional[int] = None, + candidate_channel: Optional[int] = None, + ground_truth_path: Optional[str] = None, + ground_truth_dataset: Optional[str] = None, + crop: Optional[str] = None, +) -> Dict[str, str]: + artifact_dir = output_dir / "artifacts" + artifact_dir.mkdir(parents=True, exist_ok=True) + + image = load_volume(image_path, dataset_key=image_dataset, crop=crop) + ground_truth_source = ground_truth_path or mask_path + ground_truth_key = ground_truth_dataset or mask_dataset + ground_truth = load_volume( + ground_truth_source, + dataset_key=ground_truth_key, + crop=crop, + ) + if image.shape != ground_truth.shape: + raise ValueError( + "Real image/ground-truth pair must resolve to the same shape after crop; " + f"got image {image.shape} and ground truth {ground_truth.shape}" + ) + + using_real_predictions = bool(baseline_prediction_path or candidate_prediction_path) + if using_real_predictions and not ( + baseline_prediction_path and candidate_prediction_path + ): + raise ValueError( + "Real-prediction mode requires both baseline_prediction_path and " + "candidate_prediction_path" + ) + + if using_real_predictions: + baseline = _load_prediction_volume( + str(baseline_prediction_path), + dataset_key=baseline_dataset, + crop=crop, + channel=baseline_channel, + reference_ndim=ground_truth.ndim, + label="baseline", + ) + candidate = _load_prediction_volume( + str(candidate_prediction_path), + dataset_key=candidate_dataset, + crop=crop, + channel=candidate_channel, + reference_ndim=ground_truth.ndim, + label="candidate", + ) + shapes = { + "ground_truth": ground_truth.shape, + "baseline_prediction": baseline.shape, + "candidate_prediction": candidate.shape, + } + if len(set(shapes.values())) != 1: + raise ValueError( + "Real prediction volumes must match ground truth shape after crop; " + f"got {shapes}" + ) + else: + baseline = _derived_baseline_from_mask(ground_truth) + candidate = np.asarray(ground_truth).copy() + + paths = { + "source_image": str(Path(image_path).expanduser()), + "source_mask": str(Path(mask_path).expanduser()), + "source_ground_truth": str(Path(ground_truth_source).expanduser()), + "image": _write_tiff(artifact_dir / "image-crop.tif", image), + "initial_mask": _write_tiff(artifact_dir / "initial-mask-derived.tif", baseline), + "corrected_mask": _write_tiff( + artifact_dir / "corrected-mask-crop.tif", ground_truth + ), + "ground_truth": _write_tiff( + artifact_dir / "ground-truth-crop.tif", ground_truth + ), + "baseline_prediction": _write_tiff( + artifact_dir + / ( + "baseline-prediction-real.tif" + if using_real_predictions + else "baseline-prediction-derived.tif" + ), + baseline, + ), + "candidate_prediction": _write_tiff( + artifact_dir + / ( + "candidate-prediction-real.tif" + if using_real_predictions + else "candidate-prediction-derived.tif" + ), + candidate, + ), + "training_output": str(artifact_dir / "training-output"), + "checkpoint": str(artifact_dir / "checkpoint-smoke.pth"), + "evaluation_report": str(output_dir / "evaluation-report.json"), + } + if baseline_prediction_path: + paths["source_baseline_prediction"] = str( + Path(baseline_prediction_path).expanduser() + ) + if baseline_channel is not None: + paths["source_baseline_channel"] = str(baseline_channel) + if candidate_prediction_path: + paths["source_candidate_prediction"] = str( + Path(candidate_prediction_path).expanduser() + ) + if candidate_channel is not None: + paths["source_candidate_channel"] = str(candidate_channel) + + Path(paths["training_output"]).mkdir(parents=True, exist_ok=True) + Path(paths["checkpoint"]).write_text( + "real-pair smoke checkpoint placeholder\n", encoding="utf-8" + ) + return paths + + +def run_closed_loop_smoke( + output_dir: Path, + *, + include_research_plan: bool = True, + image_path: Optional[str] = None, + mask_path: Optional[str] = None, + image_dataset: Optional[str] = None, + mask_dataset: Optional[str] = None, + baseline_prediction_path: Optional[str] = None, + candidate_prediction_path: Optional[str] = None, + baseline_dataset: Optional[str] = None, + candidate_dataset: Optional[str] = None, + baseline_channel: Optional[int] = None, + candidate_channel: Optional[int] = None, + ground_truth_path: Optional[str] = None, + ground_truth_dataset: Optional[str] = None, + crop: Optional[str] = None, +) -> Dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + if image_path or mask_path: + if not image_path or not mask_path: + raise ValueError("Real-artifact mode requires both image_path and mask_path") + has_prediction_inputs = bool(baseline_prediction_path or candidate_prediction_path) + if has_prediction_inputs and not ( + baseline_prediction_path and candidate_prediction_path + ): + raise ValueError( + "Real-prediction mode requires both baseline_prediction_path and " + "candidate_prediction_path" + ) + artifact_mode = ( + "real_pair_real_predictions" + if has_prediction_inputs + else "real_pair_derived_predictions" + ) + artifacts = generate_real_pair_artifacts( + output_dir, + image_path=image_path, + mask_path=mask_path, + image_dataset=image_dataset, + mask_dataset=mask_dataset, + baseline_prediction_path=baseline_prediction_path, + candidate_prediction_path=candidate_prediction_path, + baseline_dataset=baseline_dataset, + candidate_dataset=candidate_dataset, + baseline_channel=baseline_channel, + candidate_channel=candidate_channel, + ground_truth_path=ground_truth_path, + ground_truth_dataset=ground_truth_dataset, + crop=crop, + ) + source_data = { + "image_path": image_path, + "mask_path": mask_path, + "image_dataset": image_dataset, + "mask_dataset": mask_dataset, + "ground_truth_path": ground_truth_path or mask_path, + "ground_truth_dataset": ground_truth_dataset or mask_dataset, + "baseline_prediction_path": baseline_prediction_path, + "candidate_prediction_path": candidate_prediction_path, + "baseline_dataset": baseline_dataset, + "candidate_dataset": candidate_dataset, + "baseline_channel": baseline_channel, + "candidate_channel": candidate_channel, + "crop": crop, + } + else: + artifact_mode = "synthetic" + artifacts = generate_smoke_artifacts(output_dir) + source_data = {"synthetic": True} + + from fastapi.testclient import TestClient + from sqlalchemy import create_engine + from sqlalchemy.orm import sessionmaker + + from server_api.auth import database as auth_database + from server_api.auth import models + + with contextlib.redirect_stdout(sys.stderr): + from server_api.main import app as server_api_app + + db_path = output_dir / "workflow-smoke.db" + engine = create_engine( + f"sqlite:///{db_path}", connect_args={"check_same_thread": False} + ) + session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine) + models.Base.metadata.create_all(bind=engine) + + def override_get_db(): + db = session_local() + try: + yield db + finally: + db.close() + + previous_override = server_api_app.dependency_overrides.get(auth_database.get_db) + server_api_app.dependency_overrides[auth_database.get_db] = override_get_db + prediction_runtime = ( + "external_artifact" + if artifact_mode == "real_pair_real_predictions" + else "simulated" + ) + baseline_summary = ( + "Recorded external baseline prediction artifact." + if artifact_mode == "real_pair_real_predictions" + else "Recorded synthetic baseline inference output." + ) + candidate_summary = ( + "Recorded external candidate prediction artifact." + if artifact_mode == "real_pair_real_predictions" + else "Recorded synthetic candidate inference output." + ) + + try: + client = TestClient(server_api_app) + current = _request_json(client.get("/api/workflows/current"), "get workflow") + workflow_id = current["workflow"]["id"] + + _post_event( + client, + workflow_id=workflow_id, + actor="user", + event_type="dataset.loaded", + stage="visualization", + summary="Loaded synthetic smoke-test image and initial mask.", + payload={ + "image_path": artifacts.get("source_image") or artifacts["image"], + "mask_path": artifacts["initial_mask"], + "source_mask_path": artifacts.get("source_mask"), + "source_ground_truth_path": artifacts.get("source_ground_truth"), + "image_dataset": image_dataset, + "mask_dataset": mask_dataset, + "ground_truth_dataset": ground_truth_dataset, + "crop": crop, + "artifact_mode": artifact_mode, + }, + ) + _post_event( + client, + workflow_id=workflow_id, + event_type="inference.completed", + stage="inference", + summary=baseline_summary, + payload={ + "outputPath": artifacts["baseline_prediction"], + "runtime": prediction_runtime, + "artifact_mode": artifact_mode, + "source_prediction_path": artifacts.get("source_baseline_prediction"), + "prediction_dataset": baseline_dataset, + "prediction_channel": baseline_channel, + }, + ) + _post_event( + client, + workflow_id=workflow_id, + actor="user", + event_type="proofreading.instance_classified", + stage="proofreading", + summary="Marked the smoke hotspot as incorrect.", + payload={"region_id": "z:1", "classification": "incorrect"}, + ) + _post_event( + client, + workflow_id=workflow_id, + actor="user", + event_type="proofreading.mask_saved", + stage="proofreading", + summary="Saved a synthetic corrected mask for the hotspot.", + payload={"region_id": "z:1", "instance_id": 42}, + ) + _post_event( + client, + workflow_id=workflow_id, + event_type="proofreading.masks_exported", + stage="proofreading", + summary="Exported synthetic corrected masks.", + payload={ + "written_path": artifacts["corrected_mask"], + "region_id": "z:1", + "runtime": "simulated", + "artifact_mode": artifact_mode, + }, + ) + + hotspots = _request_json( + client.get(f"/api/workflows/{workflow_id}/hotspots"), + "compute hotspots", + ) + proposal = _request_json( + client.post( + f"/api/workflows/{workflow_id}/agent-actions", + json={ + "action": "stage_retraining_from_corrections", + "summary": "Stage smoke corrections for retraining.", + "payload": {"corrected_mask_path": artifacts["corrected_mask"]}, + }, + ), + "create agent proposal", + ) + _request_json( + client.post( + f"/api/workflows/{workflow_id}/agent-actions/{proposal['id']}/approve" + ), + "approve agent proposal", + ) + _post_event( + client, + workflow_id=workflow_id, + event_type="training.completed", + stage="evaluation", + summary="Recorded synthetic retraining completion.", + payload={ + "outputPath": artifacts["training_output"], + "checkpointPath": artifacts["checkpoint"], + "checkpointName": "checkpoint-smoke.pth", + "runtime": "simulated", + "artifact_mode": artifact_mode, + }, + ) + _post_event( + client, + workflow_id=workflow_id, + event_type="inference.completed", + stage="evaluation", + summary=candidate_summary, + payload={ + "outputPath": artifacts["candidate_prediction"], + "checkpointPath": artifacts["checkpoint"], + "runtime": prediction_runtime, + "artifact_mode": artifact_mode, + "source_prediction_path": artifacts.get("source_candidate_prediction"), + "prediction_dataset": candidate_dataset, + "prediction_channel": candidate_channel, + }, + ) + + research_plan: Optional[Dict[str, Any]] = None + if include_research_plan: + research_plan = _request_json( + client.post( + f"/api/workflows/{workflow_id}/agent-plans", + json={ + "title": "Researcher-only smoke readiness plan", + "metadata": {"researcher_only": True, "synthetic": True}, + }, + ), + "create researcher-only agent plan", + ) + _request_json( + client.post( + f"/api/workflows/{workflow_id}/agent-plans/{research_plan['id']}/approve" + ), + "approve researcher-only agent plan", + ) + + evaluation = _request_json( + client.post( + f"/api/workflows/{workflow_id}/evaluation-results/compute", + json={ + "name": "synthetic-closed-loop-smoke", + "ground_truth_path": artifacts["ground_truth"], + "baseline_prediction_path": artifacts["baseline_prediction"], + "candidate_prediction_path": artifacts["candidate_prediction"], + "report_path": artifacts["evaluation_report"], + "metadata": { + "artifact_mode": artifact_mode, + "source_data": source_data, + }, + }, + ), + "compute evaluation", + ) + readiness = _request_json( + client.get(f"/api/workflows/{workflow_id}/case-study-readiness"), + "get researcher readiness", + ) + bundle = _request_json( + client.post(f"/api/workflows/{workflow_id}/export-bundle"), + "export bundle", + ) + + _write_json(output_dir / "workflow-bundle.json", bundle) + _write_json(output_dir / "readiness.json", readiness) + + report = { + "workflow_id": workflow_id, + "artifact_mode": artifact_mode, + "source_data": source_data, + "output_dir": str(output_dir), + "db_path": str(db_path), + "ready_for_case_study": readiness["ready_for_case_study"], + "readiness": { + "completed_count": readiness["completed_count"], + "total_count": readiness["total_count"], + "next_required_items": readiness["next_required_items"], + }, + "metric_summary": evaluation["metrics"]["summary"], + "bundle_counts": { + "events": len(bundle.get("events", [])), + "artifacts": len(bundle.get("artifacts", [])), + "model_runs": len(bundle.get("model_runs", [])), + "model_versions": len(bundle.get("model_versions", [])), + "correction_sets": len(bundle.get("correction_sets", [])), + "evaluation_results": len(bundle.get("evaluation_results", [])), + "persisted_hotspots": len(bundle.get("persisted_hotspots", [])), + "agent_plans": len(bundle.get("agent_plans", [])), + }, + "top_hotspot": hotspots.get("hotspots", [{}])[0], + "real_checks_exercised": [ + "FastAPI workflow routes", + "SQLite workflow persistence with typed records", + "artifact materialization from workflow events", + "correction-set materialization", + "model-run and model-version materialization", + "hotspot ranking from workflow event evidence", + "TIFF before/after segmentation metrics", + *( + ["real prediction artifacts supplied by caller"] + if artifact_mode == "real_pair_real_predictions" + else [] + ), + "workflow evidence bundle export", + ], + "simulated_or_not_exercised": [ + "actual PyTC training subprocess", + "actual PyTC inference subprocess", + "browser-level proofreading editor interaction", + *( + [] + if artifact_mode.startswith("real_pair_") + else ["real biomedical/connectomics sample data"] + ), + *( + [ + "baseline/candidate predictions derived from real segmentation labels, not model outputs" + ] + if artifact_mode == "real_pair_derived_predictions" + else [] + ), + "long-running job queue, cancellation, retry, and failure recovery", + ], + "supported_volume_formats": list(SUPPORTED_VOLUME_FORMATS), + "researcher_only_checks": [ + "case-study readiness gate", + "bounded agent plan preview and approval audit", + ] + if include_research_plan + else ["case-study readiness gate without plan approval"], + "artifacts": artifacts, + } + _write_json(output_dir / "smoke-report.json", report) + return report + finally: + if previous_override is None: + server_api_app.dependency_overrides.pop(auth_database.get_db, None) + else: + server_api_app.dependency_overrides[ + auth_database.get_db + ] = previous_override + engine.dispose() + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run a synthetic closed-loop workflow smoke for PyTC workflow evidence." + ) + parser.add_argument( + "--output-dir", + type=Path, + default=None, + help="Directory for generated smoke artifacts. Defaults to a temp directory.", + ) + parser.add_argument( + "--no-research-plan", + action="store_true", + help="Skip the researcher-only agent-plan readiness gate.", + ) + parser.add_argument( + "--image-path", + type=str, + default=None, + help="Optional real raw image volume path. Enables real-artifact mode.", + ) + parser.add_argument( + "--mask-path", + type=str, + default=None, + help="Optional real label/segmentation volume path. Enables real-artifact mode.", + ) + parser.add_argument( + "--image-dataset", + type=str, + default=None, + help="Dataset key for HDF5/Zarr/NPZ image volumes, e.g. main.", + ) + parser.add_argument( + "--mask-dataset", + type=str, + default=None, + help="Dataset key for HDF5/Zarr/NPZ mask volumes, e.g. data.", + ) + parser.add_argument( + "--ground-truth-path", + type=str, + default=None, + help="Optional ground-truth path for evaluation. Defaults to --mask-path.", + ) + parser.add_argument( + "--ground-truth-dataset", + type=str, + default=None, + help="Dataset key for the ground-truth volume. Defaults to --mask-dataset.", + ) + parser.add_argument( + "--baseline-prediction-path", + type=str, + default=None, + help="Optional real baseline prediction path.", + ) + parser.add_argument( + "--candidate-prediction-path", + type=str, + default=None, + help="Optional real candidate prediction path.", + ) + parser.add_argument( + "--baseline-dataset", + type=str, + default=None, + help="Dataset key for the baseline prediction volume.", + ) + parser.add_argument( + "--candidate-dataset", + type=str, + default=None, + help="Dataset key for the candidate prediction volume.", + ) + parser.add_argument( + "--baseline-channel", + type=int, + default=None, + help=( + "Optional channel index for channel-first/channel-last baseline " + "prediction volumes such as PyTC result_xy.h5." + ), + ) + parser.add_argument( + "--candidate-channel", + type=int, + default=None, + help=( + "Optional channel index for channel-first/channel-last candidate " + "prediction volumes such as PyTC result_xy.h5." + ), + ) + parser.add_argument( + "--crop", + type=str, + default=None, + help="Optional crop like '0:8,0:256,0:256'. Use this for large volumes.", + ) + args = parser.parse_args() + + output_dir = args.output_dir or Path( + tempfile.mkdtemp(prefix="pytc-closed-loop-smoke-") + ) + report = run_closed_loop_smoke( + output_dir=output_dir, + include_research_plan=not args.no_research_plan, + image_path=args.image_path, + mask_path=args.mask_path, + image_dataset=args.image_dataset, + mask_dataset=args.mask_dataset, + baseline_prediction_path=args.baseline_prediction_path, + candidate_prediction_path=args.candidate_prediction_path, + baseline_dataset=args.baseline_dataset, + candidate_dataset=args.candidate_dataset, + baseline_channel=args.baseline_channel, + candidate_channel=args.candidate_channel, + ground_truth_path=args.ground_truth_path, + ground_truth_dataset=args.ground_truth_dataset, + crop=args.crop, + ) + + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start.sh b/scripts/start.sh index 2311eae6..80bc6aa1 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,11 +2,28 @@ set -euo pipefail -# Prefer Homebrew's Node over nvm to avoid version conflicts -export PATH="/opt/homebrew/bin:$PATH" -export OLLAMA_BASE_URL="http://cscigpu08.bc.edu:4443" -export OLLAMA_MODEL="gpt-oss:20b" -export OLLAMA_EMBED_MODEL="qwen3-embedding:8b" +# Prefer system/Homebrew Node over shell-specific managers to avoid version conflicts. +append_path_if_dir() { + local dir="$1" + if [[ -d "${dir}" && ":${PATH}:" != *":${dir}:"* ]]; then + PATH="${dir}:${PATH}" + fi +} + +append_path_if_dir "/opt/homebrew/bin" +append_path_if_dir "/usr/local/bin" + +if command -v brew >/dev/null 2>&1; then + BREW_PREFIX="$(brew --prefix 2>/dev/null || true)" + if [[ -n "${BREW_PREFIX}" ]]; then + append_path_if_dir "${BREW_PREFIX}/bin" + fi +fi + +export PATH +export OLLAMA_BASE_URL="${OLLAMA_BASE_URL:-http://127.0.0.1:11434}" +export OLLAMA_MODEL="${OLLAMA_MODEL:-llama3.2:1b}" +export OLLAMA_EMBED_MODEL="${OLLAMA_EMBED_MODEL:-nomic-embed-text:latest}" ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" CLIENT_DIR="${ROOT_DIR}/client" @@ -16,9 +33,10 @@ if ! command -v uv >/dev/null 2>&1; then echo "uv is required. Install it from https://docs.astral.sh/uv/." >&2 exit 1 fi +UV_BIN="$(command -v uv)" -if ! command -v npm >/dev/null 2>&1; then - echo "npm is required to run the Electron client." >&2 +if ! NPM_BIN="$(command -v npm)"; then + echo "npm is required to run the Electron client. Current PATH: ${PATH}" >&2 exit 1 fi @@ -108,27 +126,45 @@ start_service \ 8000 \ "http://localhost:8000/" \ "${LOG_DIR}/data-server.log" \ - uv run --directory "${ROOT_DIR}" python server_api/scripts/serve_data.py + "${UV_BIN}" run --directory "${ROOT_DIR}" python server_api/scripts/serve_data.py start_service \ "API server" \ 4242 \ "http://localhost:4242/health" \ "${LOG_DIR}/api-server.log" \ - env PYTHONDONTWRITEBYTECODE=1 uv run --directory "${ROOT_DIR}" python -m server_api.main + env PYTHONDONTWRITEBYTECODE=1 "${UV_BIN}" run --directory "${ROOT_DIR}" python -m server_api.main start_service \ "PyTC server" \ 4243 \ "http://localhost:4243/hello" \ "${LOG_DIR}/pytc-server.log" \ - uv run --directory "${ROOT_DIR}" python -m server_pytc.main + "${UV_BIN}" run --directory "${ROOT_DIR}" python -m server_pytc.main + +ensure_client_dependencies() { + if [[ -x "${CLIENT_DIR}/node_modules/.bin/react-scripts" && -x "${CLIENT_DIR}/node_modules/.bin/cross-env" ]]; then + return 0 + fi + + local install_log="${LOG_DIR}/npm-install.log" + echo "Client dependencies missing; installing with npm (log: $(relative_log_path "${install_log}"))..." + : >"${install_log}" + if "${NPM_BIN}" install >"${install_log}" 2>&1; then + echo "Client dependencies installed" + else + echo "ERROR: npm install failed. Recent log output:" >&2 + tail -n 60 "${install_log}" >&2 || true + exit 1 + fi +} pushd "${CLIENT_DIR}" >/dev/null +ensure_client_dependencies if [[ "${SKIP_CLIENT_BUILD:-0}" != "1" ]]; then BUILD_LOG="${LOG_DIR}/react-build.log" echo "Building React client (log: $(relative_log_path "${BUILD_LOG}"))..." - if npm run build >"${BUILD_LOG}" 2>&1; then + if "${NPM_BIN}" run build >"${BUILD_LOG}" 2>&1; then echo "React build complete" else echo "ERROR: React build failed. Recent log output:" >&2 @@ -148,7 +184,7 @@ if port_is_listening 3000; then else : >"${REACT_LOG}" echo "React server starting on :3000 (log: $(relative_log_path "${REACT_LOG}"))" - BROWSER=none npm start >"${REACT_LOG}" 2>&1 & + BROWSER=none "${NPM_BIN}" start >"${REACT_LOG}" 2>&1 & STARTED_PIDS+=("$!") fi @@ -171,7 +207,7 @@ wait_for_react() { if wait_for_react; then echo "Startup logs are under $(relative_log_path "${LOG_DIR}")" echo "Starting Electron client..." - ENVIRONMENT=development npm run electron + ENVIRONMENT=development "${NPM_BIN}" run electron else echo "Failed to start React server" >&2 exit 1 diff --git a/scripts/summarize_app_log.py b/scripts/summarize_app_log.py new file mode 100644 index 00000000..618d31d7 --- /dev/null +++ b/scripts/summarize_app_log.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +from collections import Counter, defaultdict +from pathlib import Path + + +DEFAULT_LOG = Path(__file__).resolve().parents[1] / ".logs" / "app" / "app-events.jsonl" + + +def load_rows(path: Path): + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + rows.append(json.loads(line)) + except json.JSONDecodeError: + continue + return rows + + +def summarize(rows, *, session_id: str | None = None, last: int = 12): + if session_id: + rows = [row for row in rows if row.get("session_id") == session_id] + + if not rows: + print("No matching log rows.") + return + + components = Counter(row.get("component", "?") for row in rows) + events = Counter(row.get("event", "?") for row in rows) + levels = Counter(row.get("level", "?") for row in rows) + sessions = Counter(row.get("session_id") for row in rows if row.get("session_id")) + + print(f"rows: {len(rows)}") + print(f"first: {rows[0].get('timestamp')}") + print(f"last: {rows[-1].get('timestamp')}") + if sessions and not session_id: + print(f"sessions: {dict(sessions)}") + + print("\ncomponents:") + for name, count in components.most_common(): + print(f" {name}: {count}") + + print("\nlevels:") + for name, count in levels.most_common(): + print(f" {name}: {count}") + + print("\ntop events:") + for name, count in events.most_common(12): + print(f" {name}: {count}") + + print("\npossible issues:") + issue_rows = [ + row + for row in rows + if row.get("level") == "ERROR" + or row.get("event") in {"api_response_error", "window_error", "unhandled_rejection"} + ] + if not issue_rows: + print(" none") + else: + for row in issue_rows[:12]: + print( + f" {row.get('timestamp')} {row.get('component')} " + f"{row.get('event')}: {row.get('message')}" + ) + + print("\nrecent:") + for row in rows[-last:]: + print( + f" {row.get('timestamp')} {row.get('component')} " + f"{row.get('event')}: {row.get('message')}" + ) + + +def main(): + parser = argparse.ArgumentParser(description="Summarize PyTC app event logs") + parser.add_argument("--log", default=str(DEFAULT_LOG), help="Path to app-events.jsonl") + parser.add_argument("--session", default=None, help="Optional client session_id filter") + parser.add_argument("--last", type=int, default=12, help="Recent rows to print") + args = parser.parse_args() + + log_path = Path(args.log).expanduser().resolve(strict=False) + if not log_path.is_file(): + raise SystemExit(f"Log file not found: {log_path}") + + summarize(load_rows(log_path), session_id=args.session, last=args.last) + + +if __name__ == "__main__": + main() diff --git a/server_api/main.py b/server_api/main.py index 736ba3a0..a56ab901 100644 --- a/server_api/main.py +++ b/server_api/main.py @@ -1,4 +1,5 @@ import json +import logging import pathlib import re import shutil @@ -6,15 +7,22 @@ import traceback import time from concurrent.futures import ThreadPoolExecutor, TimeoutError -from typing import List, Optional +from datetime import datetime, timezone +from typing import Any, List, Optional from urllib.parse import urlsplit, urlunsplit import requests import uvicorn from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel from sqlalchemy import inspect, text from sqlalchemy.orm import Session +from app_event_logger import ( + append_app_event, + configure_process_logging, + get_app_event_log_path, +) from runtime_settings import ( get_allowed_origins, get_neuroglancer_public_base, @@ -30,8 +38,10 @@ request_id_from_request, ) from server_api.workflows import router as workflow_router +from server_api.workflows.db_models import WorkflowEvent from server_api.workflows.service import ( append_event_for_workflow_if_present, + decode_json, get_user_workflow_or_404, update_workflow_fields, ) @@ -45,7 +55,9 @@ from server_api.chatbot.chatbot import ( build_chain, build_helper_chain, + _compact_agent_response, _format_admin_llm_error, + _sanitize_agent_response, ) except Exception as exc: # pragma: no cover - exercised indirectly via endpoints build_chain = None @@ -58,6 +70,13 @@ def _format_admin_llm_error(error): "Please contact your system administrator with this error: " f"{str(error).strip() or error.__class__.__name__}" ) + + def _compact_agent_response(response, max_words=120): + return str(response or "") + + def _sanitize_agent_response(response): + return str(response or "").strip() + else: _chatbot_error = None @@ -118,9 +137,7 @@ def _invoke_with_progress(invoke_fn, *, label: str, request_id: str, poll_second try: result = future.result(timeout=poll_seconds) elapsed = time.perf_counter() - start_time - print( - f"[CHATBOT][{request_id}] {label} completed in {elapsed:.2f}s" - ) + print(f"[CHATBOT][{request_id}] {label} completed in {elapsed:.2f}s") return result except TimeoutError: elapsed = time.perf_counter() - start_time @@ -128,6 +145,8 @@ def _invoke_with_progress(invoke_fn, *, label: str, request_id: str, poll_second f"[CHATBOT][{request_id}] {label} still running " f"after {elapsed:.2f}s..." ) + + REACT_APP_SERVER_PROTOCOL = "http" REACT_APP_SERVER_URL = "localhost:4243" @@ -148,6 +167,10 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None: _ensure_sqlite_column("ehtool_sessions", "workflow_id", "workflow_id INTEGER") +_ensure_sqlite_column("chat_messages", "source", "source VARCHAR") +_ensure_sqlite_column("chat_messages", "actions_json", "actions_json TEXT") +_ensure_sqlite_column("chat_messages", "commands_json", "commands_json TEXT") +_ensure_sqlite_column("chat_messages", "proposals_json", "proposals_json TEXT") app = FastAPI() @@ -167,12 +190,232 @@ def _ensure_sqlite_column(table_name: str, column_name: str, ddl: str) -> None: allow_headers=["*"], ) +logger = logging.getLogger(__name__) + + +class ClientAppLogEvent(BaseModel): + event: str + level: str = "INFO" + message: Optional[str] = None + source: Optional[str] = None + sessionId: Optional[str] = None + url: Optional[str] = None + data: Optional[dict[str, Any]] = None + + +class WorkflowInferenceRuntimeSyncRequest(BaseModel): + stage: Optional[str] = None + + +_PREDICTION_OUTPUT_SUFFIXES = ( + ".h5", + ".hdf5", + ".hdf", + ".tif", + ".tiff", + ".ome.tif", + ".ome.tiff", + ".zarr", + ".n5", + ".npy", + ".npz", + ".nii", + ".nii.gz", + ".mrc", + ".map", + ".rec", +) + + +def _path_has_prediction_suffix(path: pathlib.Path) -> bool: + lower_name = path.name.lower() + lower_path = str(path).lower() + return lower_name.endswith(_PREDICTION_OUTPUT_SUFFIXES) or lower_path.endswith( + (".ome.tif", ".ome.tiff", ".nii.gz") + ) + + +def _is_prediction_output_candidate(path: pathlib.Path) -> bool: + if path.name.startswith("."): + return False + if not _path_has_prediction_suffix(path): + return False + if path.is_dir(): + return path.name.lower().endswith((".zarr", ".n5")) + return path.is_file() + + +def _prediction_output_priority(path: pathlib.Path) -> tuple[float, int, str]: + name = path.name.lower() + preferred = 0 + if name.startswith("result_xy"): + preferred = 3 + elif name.startswith("result"): + preferred = 2 + elif "prediction" in name or "pred" in name: + preferred = 1 + try: + mtime = path.stat().st_mtime + except OSError: + mtime = 0.0 + return (mtime, preferred, path.name) + + +def _discover_prediction_output(path_value: Optional[str]) -> Optional[str]: + if not path_value or not str(path_value).strip(): + return None + + candidate = pathlib.Path(str(path_value).strip()).expanduser() + if not candidate.is_absolute(): + candidate = (pathlib.Path.cwd() / candidate).resolve() + else: + candidate = candidate.resolve() + if _is_prediction_output_candidate(candidate): + return str(candidate) + + if candidate.is_dir(): + output_dir = candidate + elif candidate.parent.exists(): + output_dir = candidate.parent + else: + return None + + candidates = [ + child + for child in output_dir.rglob("*") + if _is_prediction_output_candidate(child) + ] + if not candidates: + return None + return str(max(candidates, key=_prediction_output_priority).resolve()) + + +def _metadata_path(metadata: dict[str, Any], *keys: str) -> Optional[str]: + for key in keys: + value = metadata.get(key) + if isinstance(value, str) and value.strip(): + return value + return None + + +def _find_synced_inference_event( + db: Session, + *, + workflow_id: int, + event_type: str, + output_path: Optional[str] = None, + ended_at: Optional[str] = None, +) -> Optional[WorkflowEvent]: + events = ( + db.query(WorkflowEvent) + .filter( + WorkflowEvent.workflow_id == workflow_id, + WorkflowEvent.event_type == event_type, + ) + .order_by(WorkflowEvent.id.desc()) + .all() + ) + for event in events: + payload = decode_json(event.payload_json) + if payload.get("source") != "runtime_sync": + continue + if output_path and payload.get("outputPath") == output_path: + return event + if ended_at and payload.get("runtimeEndedAt") == ended_at: + return event + return None + + +@app.on_event("startup") +async def configure_app_event_logging(): + log_path = configure_process_logging("server_api") + logger.info("App event logging enabled at %s", log_path) + + +@app.middleware("http") +async def log_http_requests(request: Request, call_next): + request_id = request_id_from_request(request) + start_time = time.perf_counter() + path = request.url.path + method = request.method + client_host = request.client.host if request.client else None + is_client_log_endpoint = path == "/app/log-event" + if not is_client_log_endpoint: + append_app_event( + component="server_api", + event="http_request_started", + level="INFO", + message=f"{method} {path}", + request_id=request_id, + method=method, + path=path, + query=str(request.url.query or ""), + client_host=client_host, + ) + try: + response = await call_next(request) + except Exception as exc: + append_app_event( + component="server_api", + event="http_request_failed", + level="ERROR", + message=f"{method} {path} failed", + request_id=request_id, + method=method, + path=path, + client_host=client_host, + latency_ms=round((time.perf_counter() - start_time) * 1000, 2), + error_type=exc.__class__.__name__, + error=str(exc), + ) + raise + + response.headers.setdefault("x-request-id", request_id) + latency_ms = round((time.perf_counter() - start_time) * 1000, 2) + if not is_client_log_endpoint or response.status_code >= 400 or latency_ms >= 1000: + append_app_event( + component="server_api", + event="http_request_completed", + level="INFO", + message=f"{method} {path} -> {response.status_code}", + request_id=request_id, + method=method, + path=path, + client_host=client_host, + status_code=response.status_code, + latency_ms=latency_ms, + ) + return response + @app.get("/health") def health(): return {"status": "ok"} +@app.get("/app/log-path") +def app_log_path(): + return {"path": str(get_app_event_log_path())} + + +@app.post("/app/log-event") +async def app_log_event(payload: ClientAppLogEvent, request: Request): + request_id = request_id_from_request(request) + append_app_event( + component="client", + event=payload.event, + level=payload.level, + message=payload.message, + source=payload.source, + session_id=payload.sessionId, + url=payload.url, + data=payload.data or {}, + request_id=request_id, + client_host=request.client.host if request.client else None, + ) + return {"status": "ok"} + + def _worker_url(path: str) -> str: return f"{REACT_APP_SERVER_PROTOCOL}://{REACT_APP_SERVER_URL}{path}" @@ -357,22 +600,22 @@ def _resolve_requested_config(path: str) -> Optional[pathlib.Path]: def _read_model_architectures() -> List[str]: - # Prefer runtime registry from the installed connectomics package. + # Prefer the runtime model map from the installed connectomics package. try: - from connectomics.models.arch import list_architectures + from connectomics.model.build import MODEL_MAP - architectures = list_architectures() + architectures = sorted(set(MODEL_MAP.keys())) if architectures: - return sorted(set(architectures)) + return architectures except Exception: pass - # Fallback: parse decorator registrations from source files. - pattern = re.compile(r"""@register_architecture\(\s*['"]([^'"]+)['"]\s*\)""") + # Fallback: parse the static model map from source. + pattern = re.compile(r"""['"]([^'"]+)['"]\s*:\s*[A-Za-z_][A-Za-z0-9_]*""") architectures = [] - arch_root = PYTC_ROOT / "connectomics" / "models" / "arch" - for py_file in arch_root.rglob("*.py"): - text = py_file.read_text(encoding="utf-8", errors="ignore") + build_file = PYTC_ROOT / "connectomics" / "model" / "build.py" + if build_file.is_file(): + text = build_file.read_text(encoding="utf-8", errors="ignore") architectures.extend(pattern.findall(text)) return sorted(set(architectures)) @@ -443,6 +686,98 @@ def _is_probable_label_volume(image_array) -> bool: return False +def _smallest_unsigned_dtype_for_max(max_value: int): + import numpy as np + + if max_value <= np.iinfo(np.uint8).max: + return np.uint8 + if max_value <= np.iinfo(np.uint16).max: + return np.uint16 + if max_value <= np.iinfo(np.uint32).max: + return np.uint32 + return np.uint64 + + +def _normalize_segmentation_volume_for_neuroglancer(volume): + import numpy as np + + array = np.asarray(volume) + if array.ndim == 4: + if array.shape[0] <= 16: + channel_axis = 0 + elif array.shape[-1] <= 16: + channel_axis = -1 + else: + raise ValueError( + "4D label volumes must use a small channel axis to be visualized." + ) + + if channel_axis != 0: + array = np.moveaxis(array, channel_axis, 0) + + channel_count = int(array.shape[0]) + if channel_count == 1: + array = array[0] + elif channel_count == 2: + try: + from connectomics.utils.process import bc_watershed + + array = bc_watershed(array, thres_small=1, seed_thres=1) + except Exception as exc: + raise ValueError( + "Failed to derive a 3D segmentation preview from the " + f"2-channel prediction volume: {exc}" + ) from exc + else: + array = np.argmax(array, axis=0) + + if array.size == 0: + return array.astype(np.uint8, copy=False) + + if np.issubdtype(array.dtype, np.bool_): + return array.astype(np.uint8, copy=False) + + if np.issubdtype(array.dtype, np.floating): + if not np.all(np.isfinite(array)): + raise ValueError( + "Segmentation volumes must not contain NaN or infinite values." + ) + rounded = np.rint(array) + if not np.allclose(array, rounded): + raise ValueError("Segmentation volumes must use integer-valued labels.") + array = rounded.astype(np.int64, copy=False) + + if not np.issubdtype(array.dtype, np.integer): + raise ValueError(f"Segmentation volume dtype {array.dtype} is not supported.") + + min_value = int(array.min()) + if min_value < 0: + raise ValueError("Segmentation volumes must contain non-negative label ids.") + + max_value = int(array.max()) + target_dtype = _smallest_unsigned_dtype_for_max(max_value) + if array.dtype == np.dtype(target_dtype): + return array + return array.astype(target_dtype, copy=False) + + +def _raise_missing_volume_error(path: pathlib.Path, role: str) -> None: + target = str(path) + role_name = "image" if role == "image" else "label" + if "/uploads/" in target or target.startswith("uploads/"): + raise HTTPException( + status_code=400, + detail=( + f"The selected {role_name} file is no longer present in app uploads. " + f"Please re-select or re-upload it." + ), + ) + raise HTTPException( + status_code=400, + detail=f"The selected {role_name} file does not exist on disk: {target}", + ) + + @app.post("/neuroglancer") async def neuroglancer( req: Request, @@ -492,6 +827,10 @@ async def neuroglancer( raise HTTPException( status_code=400, detail="Image path or file is required." ) + if not pathlib.Path(image).exists(): + _raise_missing_volume_error(pathlib.Path(image), "image") + if label is not None and not pathlib.Path(label).exists(): + _raise_missing_volume_error(pathlib.Path(label), "label") # neuroglancer setting -- bind to this to make accessible outside of container ip = "0.0.0.0" @@ -504,16 +843,29 @@ async def neuroglancer( ) try: im = readVol(image, image_type="im") - gt = readVol(label, image_type="im") if label else None except Exception as e: raise HTTPException( status_code=400, detail=f"Failed to read image volume: {str(e)}" ) + try: + gt = readVol(label, image_type="seg") if label else None + if gt is not None: + gt = _normalize_segmentation_volume_for_neuroglancer(gt) + except Exception as e: + raise HTTPException( + status_code=400, detail=f"Failed to prepare label volume: {str(e)}" + ) def ngLayer(data, res, oo=[0, 0, 0], tt="segmentation"): - return neuroglancer.LocalVolume( - data, dimensions=res, volume_type=tt, voxel_offset=oo - ) + try: + return neuroglancer.LocalVolume( + data, dimensions=res, volume_type=tt, voxel_offset=oo + ) + except Exception as exc: + raise HTTPException( + status_code=400, + detail=f"Failed to prepare Neuroglancer {tt} layer: {exc}", + ) from exc with viewer.txn() as s: s.layers.append(name="im", layer=ngLayer(im, res, tt="image")) @@ -569,6 +921,20 @@ async def start_model_training( db: Session = Depends(get_db), ): body = await req.json() + append_app_event( + component="server_api", + event="training_request_received", + level="INFO", + message="Training request received by API", + source="api_endpoint", + payload_keys=sorted(body.keys()), + config_origin_path=body.get("configOriginPath"), + output_path=body.get("outputPath"), + log_path=body.get("logPath"), + training_config_length=len(body.get("trainingConfig") or ""), + workflow_id=body.get("workflow_id") or body.get("workflowId"), + user_id=current_user.id, + ) workflow_id = body.get("workflow_id") or body.get("workflowId") if workflow_id: workflow = get_user_workflow_or_404( @@ -635,6 +1001,21 @@ async def start_model_inference( db: Session = Depends(get_db), ): body = await req.json() + append_app_event( + component="server_api", + event="inference_request_received", + level="INFO", + message="Inference request received by API", + source="api_endpoint", + payload_keys=sorted(body.keys()), + config_origin_path=body.get("configOriginPath"), + output_path=body.get("outputPath"), + checkpoint_path=(body.get("arguments") or {}).get("checkpoint") + or body.get("checkpointPath"), + inference_config_length=len(body.get("inferenceConfig") or ""), + workflow_id=body.get("workflow_id") or body.get("workflowId"), + user_id=current_user.id, + ) workflow_id = body.get("workflow_id") or body.get("workflowId") if workflow_id: workflow = get_user_workflow_or_404( @@ -696,6 +1077,156 @@ async def get_inference_logs(): return _proxy_to_worker("get", "/inference_logs", timeout=5) +@app.post("/api/workflows/{workflow_id}/sync-inference-runtime") +async def sync_workflow_inference_runtime( + workflow_id: int, + body: Optional[WorkflowInferenceRuntimeSyncRequest] = None, + current_user: models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404( + db, workflow_id=int(workflow_id), user_id=current_user.id + ) + runtime = _proxy_to_worker("get", "/inference_logs", timeout=5) or {} + metadata = runtime.get("metadata") if isinstance(runtime, dict) else {} + metadata = metadata if isinstance(metadata, dict) else {} + phase = runtime.get("phase") if isinstance(runtime, dict) else None + checkpoint_path = _metadata_path( + metadata, + "checkpointPath", + "latestCheckpointPath", + "checkpoint", + ) + output_directory = _metadata_path(metadata, "outputPath", "output_path") + prediction_path = _metadata_path( + metadata, + "predictionPath", + "latestPredictionPath", + "outputPredictionPath", + ) or _discover_prediction_output(output_directory) + + if phase not in {"finished", "failed"}: + return { + "synced": False, + "phase": phase or "unknown", + "reason": "runtime_not_terminal", + "metadata": metadata, + } + + runtime_payload = { + "source": "runtime_sync", + "runtimePhase": phase, + "runtimePid": runtime.get("pid") if isinstance(runtime, dict) else None, + "runtimeExitCode": ( + runtime.get("exitCode") if isinstance(runtime, dict) else None + ), + "runtimeStartedAt": ( + runtime.get("startedAt") if isinstance(runtime, dict) else None + ), + "runtimeEndedAt": runtime.get("endedAt") if isinstance(runtime, dict) else None, + "runtimeLineCount": ( + runtime.get("lineCount") if isinstance(runtime, dict) else None + ), + "outputDirectory": output_directory, + "checkpointPath": checkpoint_path, + "configPath": runtime.get("configPath") if isinstance(runtime, dict) else None, + "configOriginPath": ( + runtime.get("configOriginPath") if isinstance(runtime, dict) else None + ) + or metadata.get("configOriginPath"), + "workflowId": metadata.get("workflowId") or workflow_id, + "predictionName": ( + pathlib.Path(prediction_path).name if prediction_path else None + ), + } + + if phase == "failed": + ended_at = runtime_payload.get("runtimeEndedAt") + existing = _find_synced_inference_event( + db, + workflow_id=workflow.id, + event_type="inference.failed", + ended_at=ended_at, + ) + if existing is None: + append_event_for_workflow_if_present( + db, + workflow_id=workflow.id, + actor="system", + event_type="inference.failed", + stage="inference", + summary="Synchronized failed PyTC inference runtime.", + payload={ + **runtime_payload, + "lastError": ( + runtime.get("lastError") if isinstance(runtime, dict) else None + ), + }, + ) + return { + "synced": True, + "phase": phase, + "event_type": "inference.failed", + "deduplicated": existing is not None, + } + + if not prediction_path: + return { + "synced": False, + "phase": phase, + "reason": "prediction_artifact_not_found", + "metadata": metadata, + } + + requested_stage = body.stage if body else None + target_stage = requested_stage or ( + "evaluation" + if workflow.stage in {"retraining_staged", "evaluation"} + else "inference" + ) + update_payload = { + "stage": target_stage, + "inference_output_path": prediction_path, + } + if checkpoint_path: + update_payload["checkpoint_path"] = checkpoint_path + update_workflow_fields(db, workflow, update_payload, commit=True) + + existing = _find_synced_inference_event( + db, + workflow_id=workflow.id, + event_type="inference.completed", + output_path=prediction_path, + ended_at=runtime_payload.get("runtimeEndedAt"), + ) + event = existing + if event is None: + event = append_event_for_workflow_if_present( + db, + workflow_id=workflow.id, + actor="system", + event_type="inference.completed", + stage=target_stage, + summary="Synchronized completed PyTC inference output.", + payload={ + **runtime_payload, + "outputPath": prediction_path, + "latestPredictionPath": prediction_path, + }, + ) + + return { + "synced": True, + "phase": phase, + "event_type": "inference.completed", + "event_id": event.id if event else None, + "deduplicated": existing is not None, + "outputPath": prediction_path, + "checkpointPath": checkpoint_path, + "stage": target_stage, + } + + @app.get("/start_tensorboard") async def start_tensorboard(logPath: Optional[str] = None): return _proxy_to_worker( @@ -711,6 +1242,11 @@ async def get_tensorboard_url(): return _proxy_to_worker("get", "/get_tensorboard_url", timeout=5) +@app.get("/get_tensorboard_status") +async def get_tensorboard_status(): + return _proxy_to_worker("get", "/get_tensorboard_status", timeout=5) + + # TODO: Improve on this: basic idea: labels are binary -- black or white? # Check the unique values: Assume that the label should have 0 or 255 # This is temporarily ditched in favor of allowing users to specify whether or not a file is a label or image. @@ -863,7 +1399,11 @@ def update_conversation( if not convo: raise HTTPException(status_code=404, detail="Conversation not found") if "title" in req_body: - convo.title = req_body["title"][:120] # cap at 120 chars + title = str(req_body["title"] or "").strip() + if not title: + raise HTTPException(status_code=400, detail="Title must be non-empty") + convo.title = title[:120] # cap at 120 chars + convo.updated_at = datetime.now(timezone.utc) db.commit() db.refresh(convo) return convo @@ -959,10 +1499,7 @@ async def chat_query( ) except Exception as exc: _chatbot_error = exc - print( - "[CHATBOT] LLM request failed: " - f"{exc.__class__.__name__}: {exc!r}" - ) + print("[CHATBOT] LLM request failed: " f"{exc.__class__.__name__}: {exc!r}") traceback.print_exc() log_request_summary( request_id=request_id, @@ -976,6 +1513,8 @@ async def chat_query( ) from exc messages = result.get("messages", []) response = messages[-1].content if messages else "No response generated" + response = _compact_agent_response(response) + response = _sanitize_agent_response(response) print( f"[CHATBOT][{request_id}] Chain returned messages={len(messages)}, " f"response_len={len(response) if isinstance(response, str) else 0}" @@ -990,6 +1529,7 @@ async def chat_query( # Auto-title: first user message becomes the title (truncated) if convo.title == "New Chat": convo.title = query[:120].strip() or "New Chat" + convo.updated_at = datetime.now(timezone.utc) db.commit() @@ -1042,6 +1582,114 @@ async def chat_status(): # Helper chat endpoints (inline "?" popovers — RAG only, no training/inference) # --------------------------------------------------------------------------- +_INLINE_HELP_DOCS_CACHE: Optional[dict[str, str]] = None + + +def _inline_helper_mode() -> str: + return os.getenv("PYTC_INLINE_HELP_MODE", "docs").strip().lower() + + +def _load_inline_help_docs() -> dict[str, str]: + global _INLINE_HELP_DOCS_CACHE + if _INLINE_HELP_DOCS_CACHE is not None: + return _INLINE_HELP_DOCS_CACHE + + docs_dir = BASE_DIR / "server_api" / "chatbot" / "file_summaries" + docs: dict[str, str] = {} + if docs_dir.is_dir(): + for path in docs_dir.rglob("*.md"): + docs[path.name] = path.read_text(encoding="utf-8", errors="ignore") + _INLINE_HELP_DOCS_CACHE = docs + return docs + + +def _helper_tokens(*values: Optional[str]) -> set[str]: + text = " ".join(value or "" for value in values).lower() + return { + token + for token in re.findall(r"[a-z0-9_]+", text) + if len(token) >= 3 and token not in {"the", "and", "for", "this", "that"} + } + + +def _extract_helper_snippet( + content: str, tokens: set[str], *, max_lines: int = 7 +) -> str: + lines = [line.strip() for line in content.splitlines() if line.strip()] + matching = [ + line + for line in lines + if any(token in line.lower() for token in tokens) + and not line.lower().startswith("image:") + ] + selected = matching[:max_lines] or lines[:max_lines] + return "\n".join(selected) + + +def _direct_inline_help(query: str, field_context: str) -> Optional[str]: + normalized = f"{field_context} {query}".lower() + if "aug_num" in normalized or "augment" in normalized: + return ( + "`INFERENCE.AUG_NUM` controls how many test-time transformed predictions are " + "averaged for inference. Use `4` for quick smoke runs; use `8` or `16` " + "when you can spend more time for a more stable output. Higher values " + "increase runtime substantially." + ) + if "samples_per_batch" in normalized or "batch size" in normalized: + return ( + "Batch size controls how many patches are processed at once. Keep it at " + "`1` for large 3D volumes or memory-constrained runs; raise it only after " + "a successful small run proves there is headroom." + ) + if "blending" in normalized: + return ( + "Blending controls how overlapping inference patches are combined. " + "`gaussian` is usually the safer default because it softens patch-edge " + "artifacts; use simpler blending only for quick debugging." + ) + if "do_eval" in normalized or "eval mode" in normalized: + return ( + "Eval mode asks the inference config to run evaluation-style behavior " + "when matching labels/metrics are available. Leave it off for a plain " + "prediction export unless you intentionally configured evaluation inputs." + ) + return None + + +def _docs_only_helper_response(*, task_key: str, query: str, field_context: str) -> str: + direct = _direct_inline_help(query, field_context) + docs = _load_inline_help_docs() + tokens = _helper_tokens(task_key, query, field_context) + scored: list[tuple[int, str, str]] = [] + for filename, content in docs.items(): + haystack = f"{filename}\n{content}".lower() + score = sum( + 3 if token in filename.lower() else 1 + for token in tokens + if token in haystack + ) + if score > 0: + scored.append((score, filename, content)) + scored.sort(key=lambda item: item[0], reverse=True) + + snippets = [] + for _, filename, content in scored[:2]: + snippet = _extract_helper_snippet(content, tokens) + if snippet: + snippets.append(f"From `{filename}`:\n{snippet}") + + if direct and snippets: + return f"{direct}\n\nRelevant local docs:\n\n" + "\n\n".join(snippets) + if direct: + return direct + if snippets: + return "Relevant local docs:\n\n" + "\n\n".join(snippets) + return ( + "I could not find a precise local-doc match for this field. Use the visible " + "YAML key and current value as the source of truth, and prefer a small smoke " + "run before increasing runtime-heavy settings." + ) + def _ensure_helper_chat(task_key: str): """Lazily build a helper agent for *task_key*, reusing it on subsequent calls.""" @@ -1059,9 +1707,7 @@ def _ensure_helper_chat(task_key: str): _helper_chains[task_key] = (agent, reset_fn) _helper_histories[task_key] = [] elapsed = time.perf_counter() - start_time - print( - f"[CHATBOT] Helper chain ready for task_key={task_key} in {elapsed:.2f}s" - ) + print(f"[CHATBOT] Helper chain ready for task_key={task_key} in {elapsed:.2f}s") return True except Exception as exc: _chatbot_error = exc @@ -1091,8 +1737,24 @@ async def chat_helper_query(req: Request): f"task_key={task_key} query_len={len(query.strip())}" ) + if _inline_helper_mode() != "agent": + response = _docs_only_helper_response( + task_key=task_key, + query=query, + field_context=field_context, + ) + log_request_summary( + request_id=request_id, + endpoint="/chat/helper/query", + start_time=request_start, + status="ok", + ) + return {"response": response, "mode": "docs"} + if not _ensure_helper_chat(task_key): - print(f"[CHATBOT][{request_id}] Helper chain unavailable for task_key={task_key}") + print( + f"[CHATBOT][{request_id}] Helper chain unavailable for task_key={task_key}" + ) log_request_summary( request_id=request_id, endpoint="/chat/helper/query", @@ -1127,8 +1789,7 @@ async def chat_helper_query(req: Request): ) except Exception as exc: print( - "[CHATBOT] Helper LLM request failed: " - f"{exc.__class__.__name__}: {exc!r}" + "[CHATBOT] Helper LLM request failed: " f"{exc.__class__.__name__}: {exc!r}" ) traceback.print_exc() log_request_summary( @@ -1143,6 +1804,8 @@ async def chat_helper_query(req: Request): ) from exc messages = result.get("messages", []) response = messages[-1].content if messages else "No response generated" + response = _compact_agent_response(response, max_words=80) + response = _sanitize_agent_response(response) history.append({"role": "user", "content": user_content}) history.append({"role": "assistant", "content": response}) total_elapsed = time.perf_counter() - request_start @@ -1169,6 +1832,8 @@ async def chat_helper_clear(req: Request): def run(): + log_path = configure_process_logging("server_api") + print(f"[APP LOG] Writing app events to {log_path}") uvicorn.run( app, host="0.0.0.0", diff --git a/server_api/utils/utils.py b/server_api/utils/utils.py index 48426e0e..ebfe2342 100644 --- a/server_api/utils/utils.py +++ b/server_api/utils/utils.py @@ -1,10 +1,65 @@ import pathlib +MOUNT_VARIANT_SUFFIXES = ( + "_im", + "_image", + "_img", + "_seg", + "_mask", + "_label", + "_gt", +) + + +def _normalized_variant_name(name): + candidate = pathlib.Path(name) + stem = candidate.stem.lower() + suffix = candidate.suffix.lower() + for token in MOUNT_VARIANT_SUFFIXES: + if stem.endswith(token): + stem = stem[: -len(token)] + break + return stem, suffix + + +def resolve_existing_path(path): + if not path: + return None + + candidate = pathlib.Path(path).expanduser() + if candidate.exists(): + return candidate + + parent = candidate.parent + if not parent.exists() or not parent.is_dir(): + return candidate + + target_is_file = candidate.suffix != "" + normalized_stem, normalized_suffix = _normalized_variant_name(candidate.name) + matches = [] + + for child in parent.iterdir(): + if target_is_file and not child.is_file(): + continue + if not target_is_file and not child.is_dir(): + continue + + child_stem, child_suffix = _normalized_variant_name(child.name) + if target_is_file and child_suffix != normalized_suffix: + continue + if child_stem == normalized_stem: + matches.append(child) + + if len(matches) == 1: + return matches[0] + + return candidate + def process_path(path): if not path: return None candidate = pathlib.Path(path).expanduser() if candidate.is_absolute(): - return candidate - return candidate.resolve(strict=False) + return resolve_existing_path(candidate) + return resolve_existing_path(candidate.resolve(strict=False)) diff --git a/server_api/workflows/agent_plan.py b/server_api/workflows/agent_plan.py new file mode 100644 index 00000000..3fdfe678 --- /dev/null +++ b/server_api/workflows/agent_plan.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional, TypedDict + +from .db_models import WorkflowEvent, WorkflowSession +from .service import decode_json + +try: # LangGraph is optional at runtime for now, but the plan is graph-shaped. + from langgraph.graph import END, START, StateGraph + + LANGGRAPH_AVAILABLE = True +except Exception as exc: # pragma: no cover - exercised only without LangGraph. + END = "__end__" + START = "__start__" + StateGraph = None + LANGGRAPH_AVAILABLE = False + LANGGRAPH_IMPORT_ERROR = str(exc) +else: + LANGGRAPH_IMPORT_ERROR = None + + +class PlanGraphState(TypedDict, total=False): + step_index: int + interrupted: bool + approval_required: bool + + +def _latest_exported_mask_path(events: List[WorkflowEvent]) -> Optional[str]: + for event in reversed(events): + if event.event_type != "proofreading.masks_exported": + continue + payload = decode_json(event.payload_json) + value = ( + payload.get("corrected_mask_path") + or payload.get("written_path") + or payload.get("output_path") + ) + if value: + return str(value) + return None + + +def _langgraph_runtime_summary() -> Dict[str, Any]: + if not LANGGRAPH_AVAILABLE: + return { + "status": "unavailable", + "import_error": LANGGRAPH_IMPORT_ERROR, + } + + try: + graph = StateGraph(PlanGraphState) + + def enter(state: PlanGraphState) -> PlanGraphState: + return {"step_index": state.get("step_index", 0)} + + def approval_gate(state: PlanGraphState) -> PlanGraphState: + return {"approval_required": bool(state.get("approval_required"))} + + graph.add_node("enter_plan", enter) + graph.add_node("approval_gate", approval_gate) + graph.add_edge(START, "enter_plan") + graph.add_edge("enter_plan", "approval_gate") + graph.add_edge("approval_gate", END) + graph.compile() + except Exception as exc: # pragma: no cover - defensive compatibility guard. + return { + "status": "available_compile_failed", + "import_error": None, + "compile_error": str(exc), + } + + return { + "status": "available_not_executing", + "state_graph": "StateGraph", + "interrupt_strategy": "persisted human approval gates", + "checkpointer": "sqlalchemy workflow_agent_plans/workflow_agent_steps", + } + + +def _gate_lookup(gates: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + return {str(gate.get("id")): gate for gate in gates} + + +def _gate_complete(gates_by_id: Dict[str, Dict[str, Any]], gate_id: str) -> bool: + return bool(gates_by_id.get(gate_id, {}).get("complete")) + + +def _node_status( + *, + action: str, + gate_id: Optional[str], + dependencies: List[str], + gates_by_id: Dict[str, Dict[str, Any]], + has_hotspots: bool, +) -> str: + if action == "review_failure_hotspots" and has_hotspots: + return "completed" + if gate_id and _gate_complete(gates_by_id, gate_id): + return "completed" + if all(_gate_complete(gates_by_id, dependency) for dependency in dependencies): + return "ready" + return "blocked" + + +def build_case_study_plan_graph( + *, + workflow: WorkflowSession, + events: List[WorkflowEvent], + gates: List[Dict[str, Any]], + title: Optional[str] = None, + goal: Optional[str] = None, +) -> Dict[str, Any]: + gates_by_id = _gate_lookup(gates) + corrected_mask_path = workflow.corrected_mask_path or _latest_exported_mask_path( + events + ) + has_hotspots = bool(getattr(workflow, "region_hotspots", []) or []) + latest_checkpoint = workflow.checkpoint_path + + step_specs: List[Dict[str, Any]] = [ + { + "action": "verify_data_context", + "title": "Verify data context", + "summary": "Confirm image, label/mask, config, and workflow metadata are linked.", + "stage": "setup", + "gate_id": "data_loaded", + "dependencies": ["workflow_context"], + "requires_approval": False, + "client_effects": {"navigate_to": "files"}, + "required_for": ["CS1", "CS7"], + }, + { + "action": "run_baseline_inference", + "title": "Run baseline inference", + "summary": "Produce or register the initial prediction artifact before proofreading.", + "stage": "inference", + "gate_id": "baseline_inference", + "dependencies": ["data_loaded"], + "requires_approval": True, + "client_effects": { + "navigate_to": "inference", + "runtime_action": {"kind": "start_inference"}, + }, + "required_for": ["CS1", "CS4"], + }, + { + "action": "review_failure_hotspots", + "title": "Review failure hotspots", + "summary": "Rank likely failure regions and route the top region into proofreading.", + "stage": "proofreading", + "gate_id": None, + "dependencies": ["baseline_inference"], + "requires_approval": False, + "client_effects": { + "navigate_to": "mask-proofreading", + "refresh_insights": True, + }, + "required_for": ["CS2"], + }, + { + "action": "export_corrections", + "title": "Export corrections", + "summary": "Persist proofread masks as a correction set for retraining.", + "stage": "proofreading", + "gate_id": "proofreading_corrections", + "dependencies": ["baseline_inference"], + "requires_approval": False, + "client_effects": {"navigate_to": "mask-proofreading"}, + "required_for": ["CS2", "CS3"], + }, + { + "action": "stage_retraining_from_corrections", + "title": "Stage retraining", + "summary": "Attach the correction set to the next training configuration.", + "stage": "retraining_staged", + "gate_id": "retraining_handoff", + "dependencies": ["proofreading_corrections"], + "requires_approval": True, + "client_effects": { + "navigate_to": "training", + "set_training_label_path": corrected_mask_path or "", + }, + "required_for": ["CS3", "CS5"], + }, + { + "action": "launch_retraining", + "title": "Launch retraining", + "summary": "Run the candidate training job from staged corrections.", + "stage": "retraining_staged", + "gate_id": "training_completion", + "dependencies": ["retraining_handoff"], + "requires_approval": True, + "client_effects": { + "navigate_to": "training", + "set_training_label_path": corrected_mask_path or "", + "runtime_action": {"kind": "start_training"}, + }, + "required_for": ["CS3", "CS6"], + }, + { + "action": "register_model_version", + "title": "Register model version", + "summary": "Link the produced checkpoint to a versioned candidate model.", + "stage": "evaluation", + "gate_id": "model_version", + "dependencies": ["training_completion"], + "requires_approval": False, + "client_effects": {"refresh_insights": True}, + "required_for": ["CS3", "CS4"], + }, + { + "action": "run_post_retraining_inference", + "title": "Run post-retraining inference", + "summary": "Run inference with the candidate checkpoint for before/after comparison.", + "stage": "evaluation", + "gate_id": "post_retraining_inference", + "dependencies": ["model_version"], + "requires_approval": True, + "client_effects": { + "navigate_to": "inference", + "set_inference_checkpoint_path": latest_checkpoint or "", + "runtime_action": {"kind": "start_inference"}, + }, + "required_for": ["CS4"], + }, + { + "action": "compute_before_after_evaluation", + "title": "Compute before/after evaluation", + "summary": "Compare baseline and candidate predictions against held-out labels.", + "stage": "evaluation", + "gate_id": "evaluation_result", + "dependencies": ["post_retraining_inference"], + "requires_approval": False, + "client_effects": {"refresh_insights": True}, + "required_for": ["CS4", "CS7"], + }, + { + "action": "audit_agent_control", + "title": "Audit agent control", + "summary": "Record human decisions for plan approval, interruption, resume, or rejection.", + "stage": workflow.stage, + "gate_id": "agent_audit", + "dependencies": ["proofreading_corrections"], + "requires_approval": True, + "client_effects": {}, + "required_for": ["CS5", "CS6"], + }, + { + "action": "export_case_study_bundle", + "title": "Export case-study bundle", + "summary": "Export typed artifacts, events, metrics, plans, and evaluation evidence.", + "stage": "evaluation", + "gate_id": None, + "dependencies": [ + "evaluation_result", + "agent_audit", + "agent_plan_preview", + ], + "requires_approval": False, + "client_effects": {"refresh_insights": True}, + "required_for": ["CS7"], + }, + ] + + nodes: List[Dict[str, Any]] = [] + for index, spec in enumerate(step_specs): + dependencies = list(spec["dependencies"]) + status = _node_status( + action=spec["action"], + gate_id=spec["gate_id"], + dependencies=dependencies, + gates_by_id=gates_by_id, + has_hotspots=has_hotspots, + ) + nodes.append( + { + "id": f"step_{index + 1:02d}", + "index": index, + **spec, + "status": status, + } + ) + + edges: List[Dict[str, str]] = [] + action_to_node = {node["action"]: node["id"] for node in nodes} + gate_to_node = { + str(node["gate_id"]): node["id"] for node in nodes if node.get("gate_id") + } + for node in nodes: + for dependency in node["dependencies"]: + source = gate_to_node.get(dependency) or action_to_node.get(dependency) + if source: + edges.append({"source": source, "target": node["id"]}) + + completed_count = len([gate for gate in gates if gate.get("complete")]) + total_count = len(gates) + return { + "graph_spec_version": "case-study-agent-plan/v1", + "workflow_id": workflow.id, + "title": title or "Closed-loop case-study plan", + "goal": goal + or "Drive the biomedical segmentation workflow through triage, proofreading, retraining, evaluation, and evidence export.", + "execution_model": { + "mode": "bounded_human_approved_plan_preview", + "mutating_steps_require_approval": True, + "durability": "workflow_agent_plans and workflow_agent_steps tables", + "langgraph": _langgraph_runtime_summary(), + }, + "readiness_snapshot": { + "completed_count": completed_count, + "total_count": total_count, + "ready_for_case_study": completed_count == total_count, + }, + "nodes": nodes, + "edges": edges, + } diff --git a/server_api/workflows/bundle_export.py b/server_api/workflows/bundle_export.py index 1abe8be1..d953b816 100644 --- a/server_api/workflows/bundle_export.py +++ b/server_api/workflows/bundle_export.py @@ -5,7 +5,17 @@ from typing import Any, Dict, Iterable, List, Tuple from .db_models import WorkflowEvent, WorkflowSession -from .service import event_to_dict, workflow_to_dict +from .service import ( + agent_plan_to_dict, + artifact_to_dict, + correction_set_to_dict, + evaluation_result_to_dict, + event_to_dict, + model_run_to_dict, + model_version_to_dict, + region_hotspot_to_dict, + workflow_to_dict, +) def _parse_timestamp(value: Any) -> datetime: @@ -59,8 +69,52 @@ def build_export_bundle( discovered = set(_collect_paths(session_snapshot)) for event in ordered_events: discovered.update(_collect_paths(event)) + typed_artifacts = [ + _normalize_value(artifact_to_dict(artifact)) + for artifact in getattr(workflow, "artifacts", []) + ] + model_runs = [ + _normalize_value(model_run_to_dict(run)) + for run in getattr(workflow, "model_runs", []) + ] + model_versions = [ + _normalize_value(model_version_to_dict(version)) + for version in getattr(workflow, "model_versions", []) + ] + correction_sets = [ + _normalize_value(correction_set_to_dict(correction_set)) + for correction_set in getattr(workflow, "correction_sets", []) + ] + evaluation_results = [ + _normalize_value(evaluation_result_to_dict(result)) + for result in getattr(workflow, "evaluation_results", []) + ] + persisted_hotspots = [ + _normalize_value(region_hotspot_to_dict(hotspot)) + for hotspot in getattr(workflow, "region_hotspots", []) + ] + agent_plans = [ + _normalize_value(agent_plan_to_dict(plan)) + for plan in getattr(workflow, "agent_plans", []) + ] + + for artifact in typed_artifacts: + discovered.update(_collect_paths(artifact)) + for run in model_runs: + discovered.update(_collect_paths(run)) + for version in model_versions: + discovered.update(_collect_paths(version)) + for correction_set in correction_sets: + discovered.update(_collect_paths(correction_set)) + for result in evaluation_results: + discovered.update(_collect_paths(result)) + for plan in agent_plans: + discovered.update(_collect_paths(plan)) + artifact_paths = sorted(path for path in discovered if path) - artifacts = [{"path": path, "exists": os.path.exists(path)} for path in artifact_paths] + artifacts = [ + {"path": path, "exists": os.path.exists(path)} for path in artifact_paths + ] return { "schema_version": "workflow-export-bundle/v1", @@ -68,5 +122,12 @@ def build_export_bundle( "workflow_id": workflow.id, "session_snapshot": session_snapshot, "events": ordered_events, + "artifacts": typed_artifacts, + "model_runs": model_runs, + "model_versions": model_versions, + "correction_sets": correction_sets, + "evaluation_results": evaluation_results, + "persisted_hotspots": persisted_hotspots, + "agent_plans": agent_plans, "artifact_paths": artifacts, } diff --git a/server_api/workflows/db_models.py b/server_api/workflows/db_models.py index f7d2b56b..a6066786 100644 --- a/server_api/workflows/db_models.py +++ b/server_api/workflows/db_models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import Boolean, Column, DateTime, Float, ForeignKey, Integer, String, Text from sqlalchemy.orm import relationship from sqlalchemy.sql import func @@ -34,6 +34,48 @@ class WorkflowSession(Base): cascade="all, delete-orphan", order_by="WorkflowEvent.created_at", ) + artifacts = relationship( + "WorkflowArtifact", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowArtifact.created_at", + ) + model_runs = relationship( + "WorkflowModelRun", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowModelRun.created_at", + ) + model_versions = relationship( + "WorkflowModelVersion", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowModelVersion.created_at", + ) + correction_sets = relationship( + "WorkflowCorrectionSet", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowCorrectionSet.created_at", + ) + evaluation_results = relationship( + "WorkflowEvaluationResult", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowEvaluationResult.created_at", + ) + region_hotspots = relationship( + "WorkflowRegionHotspot", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowRegionHotspot.updated_at", + ) + agent_plans = relationship( + "WorkflowAgentPlan", + back_populates="workflow", + cascade="all, delete-orphan", + order_by="WorkflowAgentPlan.updated_at", + ) class WorkflowEvent(Base): @@ -53,3 +95,212 @@ class WorkflowEvent(Base): workflow = relationship("WorkflowSession", back_populates="events") + +class WorkflowArtifact(Base): + __tablename__ = "workflow_artifacts" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + artifact_type = Column(String, nullable=False, index=True) + role = Column(String, nullable=True, index=True) + name = Column(String, nullable=True) + path = Column(Text, nullable=True) + uri = Column(Text, nullable=True) + checksum = Column(String, nullable=True) + size_bytes = Column(Integer, nullable=True) + source_event_id = Column(Integer, ForeignKey("workflow_events.id"), nullable=True) + metadata_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + workflow = relationship("WorkflowSession", back_populates="artifacts") + source_event = relationship("WorkflowEvent") + + +class WorkflowModelRun(Base): + __tablename__ = "workflow_model_runs" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + run_type = Column(String, nullable=False, index=True) + status = Column(String, default="pending", index=True) + name = Column(String, nullable=True) + config_path = Column(Text, nullable=True) + log_path = Column(Text, nullable=True) + output_path = Column(Text, nullable=True) + checkpoint_path = Column(Text, nullable=True) + input_artifact_id = Column( + Integer, ForeignKey("workflow_artifacts.id"), nullable=True + ) + output_artifact_id = Column( + Integer, ForeignKey("workflow_artifacts.id"), nullable=True + ) + source_event_id = Column(Integer, ForeignKey("workflow_events.id"), nullable=True) + metrics_json = Column(Text, nullable=True) + metadata_json = Column(Text, nullable=True) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workflow = relationship("WorkflowSession", back_populates="model_runs") + input_artifact = relationship( + "WorkflowArtifact", foreign_keys=[input_artifact_id] + ) + output_artifact = relationship( + "WorkflowArtifact", foreign_keys=[output_artifact_id] + ) + source_event = relationship("WorkflowEvent") + + +class WorkflowModelVersion(Base): + __tablename__ = "workflow_model_versions" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + version_label = Column(String, nullable=False, index=True) + status = Column(String, default="candidate", index=True) + checkpoint_path = Column(Text, nullable=True) + training_run_id = Column(Integer, ForeignKey("workflow_model_runs.id"), nullable=True) + checkpoint_artifact_id = Column( + Integer, ForeignKey("workflow_artifacts.id"), nullable=True + ) + correction_set_id = Column( + Integer, ForeignKey("workflow_correction_sets.id"), nullable=True + ) + metrics_json = Column(Text, nullable=True) + metadata_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + workflow = relationship("WorkflowSession", back_populates="model_versions") + training_run = relationship("WorkflowModelRun") + checkpoint_artifact = relationship("WorkflowArtifact") + + +class WorkflowCorrectionSet(Base): + __tablename__ = "workflow_correction_sets" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + artifact_id = Column(Integer, ForeignKey("workflow_artifacts.id"), nullable=True) + corrected_mask_path = Column(Text, nullable=False) + source_mask_path = Column(Text, nullable=True) + proofreading_session_id = Column(Integer, nullable=True, index=True) + edit_count = Column(Integer, default=0) + region_count = Column(Integer, default=0) + source_event_id = Column(Integer, ForeignKey("workflow_events.id"), nullable=True) + metadata_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + workflow = relationship("WorkflowSession", back_populates="correction_sets") + artifact = relationship("WorkflowArtifact") + source_event = relationship("WorkflowEvent") + + +class WorkflowEvaluationResult(Base): + __tablename__ = "workflow_evaluation_results" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + name = Column(String, nullable=True) + baseline_run_id = Column(Integer, ForeignKey("workflow_model_runs.id"), nullable=True) + candidate_run_id = Column(Integer, ForeignKey("workflow_model_runs.id"), nullable=True) + model_version_id = Column( + Integer, ForeignKey("workflow_model_versions.id"), nullable=True + ) + report_artifact_id = Column( + Integer, ForeignKey("workflow_artifacts.id"), nullable=True + ) + report_path = Column(Text, nullable=True) + summary = Column(Text, nullable=True) + metrics_json = Column(Text, nullable=True) + metadata_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + workflow = relationship("WorkflowSession", back_populates="evaluation_results") + baseline_run = relationship("WorkflowModelRun", foreign_keys=[baseline_run_id]) + candidate_run = relationship("WorkflowModelRun", foreign_keys=[candidate_run_id]) + model_version = relationship("WorkflowModelVersion") + report_artifact = relationship("WorkflowArtifact") + + +class WorkflowRegionHotspot(Base): + __tablename__ = "workflow_region_hotspots" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + region_key = Column(String, nullable=False, index=True) + score = Column(Float, default=0.0, index=True) + severity = Column(String, default="low", index=True) + status = Column(String, default="open", index=True) + source = Column(String, default="event_heuristic", index=True) + evidence_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workflow = relationship("WorkflowSession", back_populates="region_hotspots") + + +class WorkflowAgentPlan(Base): + __tablename__ = "workflow_agent_plans" + + id = Column(Integer, primary_key=True, index=True) + workflow_id = Column( + Integer, ForeignKey("workflow_sessions.id"), nullable=False, index=True + ) + title = Column(String, nullable=False) + status = Column(String, default="draft", index=True) + risk_level = Column(String, default="medium", index=True) + approval_status = Column(String, default="pending", index=True) + goal = Column(Text, nullable=True) + graph_json = Column(Text, nullable=True) + metadata_json = Column(Text, nullable=True) + source_event_id = Column(Integer, ForeignKey("workflow_events.id"), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + source_event = relationship("WorkflowEvent") + workflow = relationship("WorkflowSession", back_populates="agent_plans") + steps = relationship( + "WorkflowAgentStep", + back_populates="plan", + cascade="all, delete-orphan", + order_by="WorkflowAgentStep.step_index", + ) + + +class WorkflowAgentStep(Base): + __tablename__ = "workflow_agent_steps" + + id = Column(Integer, primary_key=True, index=True) + plan_id = Column(Integer, ForeignKey("workflow_agent_plans.id"), nullable=False) + step_index = Column(Integer, nullable=False) + action = Column(String, nullable=False, index=True) + status = Column(String, default="pending", index=True) + requires_approval = Column(Boolean, default=True, index=True) + summary = Column(Text, nullable=True) + params_json = Column(Text, nullable=True) + result_json = Column(Text, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + plan = relationship("WorkflowAgentPlan", back_populates="steps") diff --git a/server_api/workflows/evaluation.py b/server_api/workflows/evaluation.py new file mode 100644 index 00000000..217a1006 --- /dev/null +++ b/server_api/workflows/evaluation.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, Optional + +import numpy as np + +from .volume_io import load_volume + + +def _read_volume( + path: str, + *, + dataset_key: Optional[str] = None, + crop: Optional[str] = None, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + return np.asarray( + load_volume( + path, + dataset_key=dataset_key, + crop=crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + ) + + +def _read_eval_volume( + path: str, + *, + dataset_key: Optional[str] = None, + crop: Optional[str] = None, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str, +) -> np.ndarray: + return _read_volume( + path, + dataset_key=dataset_key, + crop=crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + +def _validate_same_shape(*arrays: np.ndarray) -> None: + shapes = {array.shape for array in arrays} + if len(shapes) != 1: + raise ValueError(f"Evaluation arrays must have identical shape; got {shapes}") + + +def _safe_ratio(numerator: float, denominator: float, empty_value: float = 1.0) -> float: + if denominator == 0: + return empty_value + return float(numerator / denominator) + + +def _segmentation_metrics(prediction: np.ndarray, ground_truth: np.ndarray) -> Dict[str, Any]: + prediction_binary = prediction > 0 + truth_binary = ground_truth > 0 + intersection = np.logical_and(prediction_binary, truth_binary).sum() + prediction_sum = prediction_binary.sum() + truth_sum = truth_binary.sum() + union = np.logical_or(prediction_binary, truth_binary).sum() + + metrics: Dict[str, Any] = { + "dice": round(_safe_ratio(2 * intersection, prediction_sum + truth_sum), 6), + "iou": round(_safe_ratio(intersection, union), 6), + "voxel_accuracy": round(float(np.mean(prediction == ground_truth)), 6), + "foreground_voxels": int(prediction_sum), + "truth_foreground_voxels": int(truth_sum), + } + + try: + from skimage.metrics import adapted_rand_error, variation_of_information + + are, precision, recall = adapted_rand_error(ground_truth, prediction) + split_vi, merge_vi = variation_of_information(ground_truth, prediction) + metrics.update( + { + "adapted_rand_error": round(float(are), 6), + "adapted_rand_precision": round(float(precision), 6), + "adapted_rand_recall": round(float(recall), 6), + "vi_split": round(float(split_vi), 6), + "vi_merge": round(float(merge_vi), 6), + "vi_total": round(float(split_vi + merge_vi), 6), + } + ) + except Exception: + metrics["advanced_metrics_unavailable"] = True + + return metrics + + +def compute_before_after_evaluation( + *, + baseline_prediction_path: str, + candidate_prediction_path: str, + ground_truth_path: str, + baseline_dataset: Optional[str] = None, + candidate_dataset: Optional[str] = None, + ground_truth_dataset: Optional[str] = None, + crop: Optional[str] = None, + baseline_channel: Optional[int] = None, + candidate_channel: Optional[int] = None, + ground_truth_channel: Optional[int] = None, +) -> Dict[str, Any]: + ground_truth = _read_eval_volume( + ground_truth_path, + dataset_key=ground_truth_dataset, + crop=crop, + channel=ground_truth_channel, + label="ground truth", + ) + baseline = _read_eval_volume( + baseline_prediction_path, + dataset_key=baseline_dataset, + crop=crop, + channel=baseline_channel, + reference_ndim=ground_truth.ndim, + label="baseline prediction", + ) + candidate = _read_eval_volume( + candidate_prediction_path, + dataset_key=candidate_dataset, + crop=crop, + channel=candidate_channel, + reference_ndim=ground_truth.ndim, + label="candidate prediction", + ) + _validate_same_shape(baseline, candidate, ground_truth) + + baseline_metrics = _segmentation_metrics(baseline, ground_truth) + candidate_metrics = _segmentation_metrics(candidate, ground_truth) + deltas: Dict[str, float] = {} + for key, candidate_value in candidate_metrics.items(): + baseline_value = baseline_metrics.get(key) + if isinstance(candidate_value, (int, float)) and isinstance( + baseline_value, (int, float) + ): + deltas[key] = round(float(candidate_value - baseline_value), 6) + + return { + "baseline": baseline_metrics, + "candidate": candidate_metrics, + "delta": deltas, + "summary": { + "dice_delta": deltas.get("dice"), + "iou_delta": deltas.get("iou"), + "voxel_accuracy_delta": deltas.get("voxel_accuracy"), + "candidate_improved_dice": bool( + candidate_metrics.get("dice", 0) >= baseline_metrics.get("dice", 0) + ), + }, + } + + +def write_evaluation_report(path: str, payload: Dict[str, Any]) -> str: + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + return str(target) diff --git a/server_api/workflows/router.py b/server_api/workflows/router.py index 5bcfe801..5eb0fbca 100644 --- a/server_api/workflows/router.py +++ b/server_api/workflows/router.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from datetime import datetime, timezone from typing import Any, Dict, List, Optional @@ -7,22 +8,46 @@ from pydantic import BaseModel, Field from sqlalchemy.orm import Session +from app_event_logger import append_app_event from server_api.auth import models as auth_models from server_api.auth.database import get_db from server_api.auth.router import get_current_user -from .db_models import WorkflowEvent, WorkflowSession +from .db_models import ( + WorkflowAgentPlan, + WorkflowAgentStep, + WorkflowArtifact, + WorkflowCorrectionSet, + WorkflowEvaluationResult, + WorkflowEvent, + WorkflowModelRun, + WorkflowModelVersion, + WorkflowRegionHotspot, + WorkflowSession, +) from .service import ( + agent_plan_to_dict, + agent_step_to_dict, append_workflow_event, + artifact_to_dict, + correction_set_to_dict, + create_workflow_artifact, decode_json, + encode_json, + evaluation_result_to_dict, event_to_dict, get_current_or_create_workflow, get_user_workflow_or_404, + model_run_to_dict, + model_version_to_dict, + region_hotspot_to_dict, update_workflow_fields, validate_stage, workflow_to_dict, ) from .bundle_export import build_export_bundle +from .agent_plan import build_case_study_plan_graph +from .evaluation import compute_before_after_evaluation, write_evaluation_report from .metrics import compute_workflow_metrics router = APIRouter() @@ -100,11 +125,51 @@ class AgentActionCreateRequest(BaseModel): class AgentQueryRequest(BaseModel): query: str + conversation_id: Optional[int] = None + conversationId: Optional[int] = None + + +class AgentChatAction(BaseModel): + id: str + label: str + description: str + variant: str = "default" + risk_level: str = "read_only" + requires_approval: bool = False + disabled_reason: Optional[str] = None + client_effects: Dict[str, Any] = Field(default_factory=dict) + + +class AgentCommandBlock(BaseModel): + id: str + title: str + description: str + command: str + run_label: str = "Execute" + risk_level: str = "read_only" + requires_approval: bool = False + client_effects: Dict[str, Any] = Field(default_factory=dict) + + +class AgentTaskItem(BaseModel): + id: str + label: str + status: str + detail: str + priority: str = "normal" class AgentQueryResponse(BaseModel): response: str + source: str = "workflow_orchestrator" + intent: str = "recommendation" + permission_mode: str = "approval_required_for_runtime" + conversation_id: Optional[int] = None + conversationId: Optional[int] = None proposals: List[WorkflowEventResponse] = Field(default_factory=list) + actions: List[AgentChatAction] = Field(default_factory=list) + commands: List[AgentCommandBlock] = Field(default_factory=list) + tasks: List[AgentTaskItem] = Field(default_factory=list) class AgentActionResult(BaseModel): @@ -143,6 +208,31 @@ class WorkflowImpactPreviewResponse(BaseModel): next_actions: List[str] = Field(default_factory=list) +class WorkflowReadinessItem(BaseModel): + id: str + label: str + complete: bool + detail: str + severity: str = "default" + + +class WorkflowAgentRecommendationResponse(BaseModel): + workflow_id: int + generated_at: str + stage: str + decision: str + rationale: str + confidence: str + next_stage: str + can_act: bool = True + blockers: List[str] = Field(default_factory=list) + readiness: List[WorkflowReadinessItem] = Field(default_factory=list) + top_hotspot: Optional[WorkflowHotspotItem] = None + impact_preview: Optional[WorkflowImpactPreviewResponse] = None + actions: List[AgentChatAction] = Field(default_factory=list) + commands: List[AgentCommandBlock] = Field(default_factory=list) + + class WorkflowMetricsResponse(BaseModel): workflow_id: int metrics: Dict[str, Any] = Field(default_factory=dict) @@ -154,9 +244,238 @@ class WorkflowExportBundleResponse(BaseModel): workflow_id: int session_snapshot: Dict[str, Any] = Field(default_factory=dict) events: List[Dict[str, Any]] = Field(default_factory=list) + artifacts: List[Dict[str, Any]] = Field(default_factory=list) + model_runs: List[Dict[str, Any]] = Field(default_factory=list) + model_versions: List[Dict[str, Any]] = Field(default_factory=list) + correction_sets: List[Dict[str, Any]] = Field(default_factory=list) + evaluation_results: List[Dict[str, Any]] = Field(default_factory=list) + persisted_hotspots: List[Dict[str, Any]] = Field(default_factory=list) + agent_plans: List[Dict[str, Any]] = Field(default_factory=list) artifact_paths: List[Dict[str, Any]] = Field(default_factory=list) +class WorkflowArtifactCreateRequest(BaseModel): + artifact_type: str + role: Optional[str] = None + path: Optional[str] = None + uri: Optional[str] = None + name: Optional[str] = None + checksum: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowArtifactResponse(BaseModel): + id: int + workflow_id: int + artifact_type: str + role: Optional[str] = None + name: Optional[str] = None + path: Optional[str] = None + uri: Optional[str] = None + checksum: Optional[str] = None + size_bytes: Optional[int] = None + source_event_id: Optional[int] = None + metadata_json: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + created_at: Any + exists: bool = False + + +class WorkflowModelRunCreateRequest(BaseModel): + run_type: str + status: str = "pending" + name: Optional[str] = None + config_path: Optional[str] = None + log_path: Optional[str] = None + output_path: Optional[str] = None + checkpoint_path: Optional[str] = None + input_artifact_id: Optional[int] = None + output_artifact_id: Optional[int] = None + metrics: Dict[str, Any] = Field(default_factory=dict) + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowModelRunResponse(BaseModel): + id: int + workflow_id: int + run_type: str + status: str + name: Optional[str] = None + config_path: Optional[str] = None + log_path: Optional[str] = None + output_path: Optional[str] = None + checkpoint_path: Optional[str] = None + input_artifact_id: Optional[int] = None + output_artifact_id: Optional[int] = None + source_event_id: Optional[int] = None + metrics_json: Optional[str] = None + metrics: Dict[str, Any] = Field(default_factory=dict) + metadata_json: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + started_at: Any = None + completed_at: Any = None + created_at: Any + updated_at: Any + + +class WorkflowModelVersionCreateRequest(BaseModel): + version_label: str + status: str = "candidate" + checkpoint_path: Optional[str] = None + training_run_id: Optional[int] = None + checkpoint_artifact_id: Optional[int] = None + correction_set_id: Optional[int] = None + metrics: Dict[str, Any] = Field(default_factory=dict) + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowModelVersionResponse(BaseModel): + id: int + workflow_id: int + version_label: str + status: str + checkpoint_path: Optional[str] = None + training_run_id: Optional[int] = None + checkpoint_artifact_id: Optional[int] = None + correction_set_id: Optional[int] = None + metrics_json: Optional[str] = None + metrics: Dict[str, Any] = Field(default_factory=dict) + metadata_json: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + created_at: Any + + +class WorkflowCorrectionSetResponse(BaseModel): + id: int + workflow_id: int + artifact_id: Optional[int] = None + corrected_mask_path: str + source_mask_path: Optional[str] = None + proofreading_session_id: Optional[int] = None + edit_count: int = 0 + region_count: int = 0 + source_event_id: Optional[int] = None + metadata_json: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + created_at: Any + + +class WorkflowEvaluationCreateRequest(BaseModel): + name: Optional[str] = None + baseline_run_id: Optional[int] = None + candidate_run_id: Optional[int] = None + model_version_id: Optional[int] = None + report_path: Optional[str] = None + summary: Optional[str] = None + metrics: Dict[str, Any] = Field(default_factory=dict) + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowEvaluationComputeRequest(BaseModel): + baseline_prediction_path: str + candidate_prediction_path: str + ground_truth_path: str + baseline_dataset: Optional[str] = None + candidate_dataset: Optional[str] = None + ground_truth_dataset: Optional[str] = None + crop: Optional[str] = None + baseline_channel: Optional[int] = None + candidate_channel: Optional[int] = None + ground_truth_channel: Optional[int] = None + name: Optional[str] = None + baseline_run_id: Optional[int] = None + candidate_run_id: Optional[int] = None + model_version_id: Optional[int] = None + report_path: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowEvaluationResponse(BaseModel): + id: int + workflow_id: int + name: Optional[str] = None + baseline_run_id: Optional[int] = None + candidate_run_id: Optional[int] = None + model_version_id: Optional[int] = None + report_artifact_id: Optional[int] = None + report_path: Optional[str] = None + summary: Optional[str] = None + metrics_json: Optional[str] = None + metrics: Dict[str, Any] = Field(default_factory=dict) + metadata_json: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + created_at: Any + + +class WorkflowRegionHotspotResponse(BaseModel): + id: int + workflow_id: int + region_key: str + score: float + severity: str + status: str + source: str + evidence_json: Optional[str] = None + evidence: Dict[str, Any] = Field(default_factory=dict) + created_at: Any + updated_at: Any + + +class WorkflowAgentStepResponse(BaseModel): + id: int + plan_id: int + step_index: int + action: str + status: str + requires_approval: bool + summary: Optional[str] = None + params_json: Optional[str] = None + params: Dict[str, Any] = Field(default_factory=dict) + result_json: Optional[str] = None + result: Dict[str, Any] = Field(default_factory=dict) + created_at: Any + updated_at: Any + + +class WorkflowAgentPlanResponse(BaseModel): + id: int + workflow_id: int + title: str + status: str + risk_level: str + approval_status: str + goal: Optional[str] = None + graph_json: Optional[str] = None + graph: Dict[str, Any] = Field(default_factory=dict) + metadata_json: Optional[str] = None + metadata: Dict[str, Any] = Field(default_factory=dict) + source_event_id: Optional[int] = None + created_at: Any + updated_at: Any + steps: List[WorkflowAgentStepResponse] = Field(default_factory=list) + + +class WorkflowAgentPlanCreateRequest(BaseModel): + title: Optional[str] = None + goal: Optional[str] = None + plan_type: str = "case_study_closed_loop" + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowAgentStepCompleteRequest(BaseModel): + status: str = "completed" + result: Dict[str, Any] = Field(default_factory=dict) + + +class WorkflowReadinessResponse(BaseModel): + workflow_id: int + ready_for_case_study: bool + completed_count: int + total_count: int + gates: List[Dict[str, Any]] = Field(default_factory=list) + next_required_items: List[str] = Field(default_factory=list) + + def _workflow_response(workflow: WorkflowSession) -> WorkflowResponse: return WorkflowResponse(**workflow_to_dict(workflow)) @@ -165,6 +484,46 @@ def _event_response(event: WorkflowEvent) -> WorkflowEventResponse: return WorkflowEventResponse(**event_to_dict(event)) +def _artifact_response(artifact: WorkflowArtifact) -> WorkflowArtifactResponse: + return WorkflowArtifactResponse(**artifact_to_dict(artifact)) + + +def _model_run_response(run: WorkflowModelRun) -> WorkflowModelRunResponse: + return WorkflowModelRunResponse(**model_run_to_dict(run)) + + +def _model_version_response( + version: WorkflowModelVersion, +) -> WorkflowModelVersionResponse: + return WorkflowModelVersionResponse(**model_version_to_dict(version)) + + +def _correction_set_response( + correction_set: WorkflowCorrectionSet, +) -> WorkflowCorrectionSetResponse: + return WorkflowCorrectionSetResponse(**correction_set_to_dict(correction_set)) + + +def _evaluation_response( + result: WorkflowEvaluationResult, +) -> WorkflowEvaluationResponse: + return WorkflowEvaluationResponse(**evaluation_result_to_dict(result)) + + +def _region_hotspot_response( + hotspot: WorkflowRegionHotspot, +) -> WorkflowRegionHotspotResponse: + return WorkflowRegionHotspotResponse(**region_hotspot_to_dict(hotspot)) + + +def _agent_step_response(step: WorkflowAgentStep) -> WorkflowAgentStepResponse: + return WorkflowAgentStepResponse(**agent_step_to_dict(step)) + + +def _agent_plan_response(plan: WorkflowAgentPlan) -> WorkflowAgentPlanResponse: + return WorkflowAgentPlanResponse(**agent_plan_to_dict(plan)) + + def _event_list(db: Session, workflow_id: int) -> List[WorkflowEventResponse]: events = ( db.query(WorkflowEvent) @@ -203,6 +562,38 @@ def _get_pending_proposal_or_404( return event +def _get_agent_plan_or_404( + db: Session, *, workflow_id: int, plan_id: int +) -> WorkflowAgentPlan: + plan = ( + db.query(WorkflowAgentPlan) + .filter( + WorkflowAgentPlan.id == plan_id, + WorkflowAgentPlan.workflow_id == workflow_id, + ) + .first() + ) + if not plan: + raise HTTPException(status_code=404, detail="Agent plan not found") + return plan + + +def _get_agent_step_or_404( + db: Session, *, plan_id: int, step_id: int +) -> WorkflowAgentStep: + step = ( + db.query(WorkflowAgentStep) + .filter( + WorkflowAgentStep.id == step_id, + WorkflowAgentStep.plan_id == plan_id, + ) + .first() + ) + if not step: + raise HTTPException(status_code=404, detail="Agent plan step not found") + return step + + def _latest_exported_mask_path(db: Session, workflow_id: int) -> Optional[str]: event = ( db.query(WorkflowEvent) @@ -272,12 +663,12 @@ def _hotspot_severity(score: float) -> str: def _default_region_action(workflow: WorkflowSession, severity: str) -> str: if workflow.stage == "proofreading": - return "Open this region in proofreading and apply mask corrections." + return "Proofread this likely mistake." if severity == "high": - return "Prioritize this region for proofreading before the next training iteration." + return "Proofread this region before training again." if workflow.stage in {"visualization", "inference"}: - return "Inspect this region in visualization and route it into proofreading." - return "Inspect this region and log whether model output is acceptable." + return "Inspect this region, then proofread it if the mask looks wrong." + return "Check this region and mark whether the mask is usable." def _compute_hotspots( @@ -365,9 +756,7 @@ def ensure_region(region_key: str) -> Dict[str, Any]: ) recommended_action = _default_region_action(workflow, severity) if stat["exports"] > 0 and workflow.stage != "retraining_staged": - recommended_action = ( - "Corrections already exist; stage this region's masks for retraining." - ) + recommended_action = "Use these saved edits for training." ranked.append( WorkflowHotspotItem( @@ -393,6 +782,46 @@ def ensure_region(region_key: str) -> Dict[str, Any]: ] +def _persist_computed_hotspots( + db: Session, + *, + workflow_id: int, + hotspots: List[WorkflowHotspotItem], +) -> None: + for item in hotspots: + existing = ( + db.query(WorkflowRegionHotspot) + .filter( + WorkflowRegionHotspot.workflow_id == workflow_id, + WorkflowRegionHotspot.region_key == item.region_key, + ) + .first() + ) + evidence = { + **(item.evidence or {}), + "summary": item.summary, + "recommended_action": item.recommended_action, + "rank": item.rank, + } + if existing: + existing.score = item.score + existing.severity = item.severity + existing.evidence_json = encode_json(evidence) + existing.source = "event_heuristic" + else: + db.add( + WorkflowRegionHotspot( + workflow_id=workflow_id, + region_key=item.region_key, + score=item.score, + severity=item.severity, + evidence_json=encode_json(evidence), + source="event_heuristic", + ) + ) + db.commit() + + def _compute_impact_preview( workflow: WorkflowSession, events: List[WorkflowEvent], @@ -403,6 +832,8 @@ def _compute_impact_preview( "inference_started": 0, "inference_completed": 0, "inference_failed": 0, + "training_completed": 0, + "training_failed": 0, "proofreading_classified": 0, "proofreading_mask_saved": 0, "proofreading_masks_exported": 0, @@ -415,6 +846,10 @@ def _compute_impact_preview( signals["inference_completed"] += 1 if event.event_type == "inference.failed": signals["inference_failed"] += 1 + if event.event_type == "training.completed": + signals["training_completed"] += 1 + if event.event_type == "training.failed": + signals["training_failed"] += 1 if event.event_type == "proofreading.instance_classified": signals["proofreading_classified"] += 1 if event.event_type == "proofreading.mask_saved": @@ -468,24 +903,31 @@ def _compute_impact_preview( "Export corrected masks from proofreading to create a retraining artifact." ) elif can_stage_retraining: - next_actions.append("Approve or trigger retraining staging from corrected masks.") + next_actions.append( + "Approve or trigger retraining staging from corrected masks." + ) if workflow.stage == "retraining_staged": next_actions.append( "Open Model Training and launch the next experiment using staged labels." ) - - if confidence == "low": - summary = ( - "Correction evidence is still sparse; prioritize proofreading edits before retraining." + if workflow.stage == "evaluation": + next_actions.append( + "Run inference with the latest trained checkpoint to evaluate the new model iteration." ) - elif can_stage_retraining: - summary = ( - "Corrections are substantial enough to justify retraining staging for the next model iteration." + next_actions.append( + "Open TensorBoard if you want to inspect the completed training run before inference." ) + + if workflow.stage == "evaluation" and signals["training_completed"] > 0: + summary = "A new model iteration is ready. Run inference with the latest checkpoint and compare it against the prior result." + elif workflow.stage == "evaluation" and signals["training_failed"] > 0: + summary = "The last training run failed. Review the runtime log before launching another iteration." + elif confidence == "low": + summary = "Correction evidence is still sparse; prioritize proofreading edits before retraining." + elif can_stage_retraining: + summary = "Corrections are substantial enough to justify retraining staging for the next model iteration." else: - summary = ( - "Correction evidence is accumulating; compare outcomes after the next staged loop." - ) + summary = "Correction evidence is accumulating; compare outcomes after the next staged loop." return WorkflowImpactPreviewResponse( workflow_id=workflow.id, @@ -508,119 +950,1957 @@ def _recommendation_for_workflow( if workflow.stage == "setup": return "Start by loading an image volume and, if available, the current mask or label volume." if workflow.stage == "visualization": - return "The next useful step is to run inference or open proofreading on the current result." + return "Proofread the mask if it is ready; otherwise run the model to make one." if workflow.stage == "inference": - return "Review the inference output and send likely failure regions into proofreading." + return "Run the model, then proofread the result." if workflow.stage == "proofreading": has_export = any( event.event_type == "proofreading.masks_exported" for event in events ) if has_export or workflow.corrected_mask_path: - return "Corrected masks are available. Stage them for retraining so the next model iteration is linked to the edits." - return "Continue classifying instances and save or export corrected masks before retraining." + return "Saved edits are available. Use them for the next training run." + return "Keep reviewing instances, then save or export edits before training." if workflow.stage == "retraining_staged": - return "The corrected masks are staged. Review the training configuration before launching retraining." + return "The saved edits are linked. Check training settings, then train." + if workflow.stage == "evaluation": + return "Training finished successfully. Use the latest checkpoint to run inference, and open TensorBoard only if you want to inspect the run." return "Review the workflow timeline and compare results before starting another iteration." -@router.get("/current", response_model=WorkflowDetailResponse) -def get_current_workflow( - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - workflow = get_current_or_create_workflow(db, user_id=user.id) - return { - "workflow": _workflow_response(workflow), - "events": _event_list(db, workflow.id), - } +def _client_effects_to_command(client_effects: Dict[str, Any]) -> str: + lines: List[str] = [] + navigate_to = client_effects.get("navigate_to") + if navigate_to: + lines.append(f"app open {navigate_to}") + training_label_path = client_effects.get("set_training_label_path") + if training_label_path: + lines.append(f"app training labels set {json.dumps(str(training_label_path))}") -@router.patch("/{workflow_id}", response_model=WorkflowResponse) -async def update_workflow( - workflow_id: int, - body: WorkflowUpdateRequest, - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) - updates = body.model_dump(exclude_unset=True) - workflow = update_workflow_fields(db, workflow, updates, commit=True) - return _workflow_response(workflow) + training_output_path = client_effects.get("set_training_output_path") + if training_output_path: + lines.append(f"app training output set {json.dumps(str(training_output_path))}") + training_log_path = client_effects.get("set_training_log_path") + if training_log_path: + lines.append(f"app training logs set {json.dumps(str(training_log_path))}") -@router.get("/{workflow_id}/events", response_model=List[WorkflowEventResponse]) -def list_workflow_events( - workflow_id: int, - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) - return _event_list(db, workflow_id) + inference_output_path = client_effects.get("set_inference_output_path") + if inference_output_path: + lines.append( + f"app inference output set {json.dumps(str(inference_output_path))}" + ) + inference_checkpoint_path = client_effects.get("set_inference_checkpoint_path") + if inference_checkpoint_path: + lines.append( + f"app inference checkpoint set {json.dumps(str(inference_checkpoint_path))}" + ) -@router.get("/{workflow_id}/hotspots", response_model=WorkflowHotspotsResponse) -def get_workflow_hotspots( - workflow_id: int, - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) - events = _event_rows(db, workflow.id) - hotspots = _compute_hotspots(workflow, events) - return WorkflowHotspotsResponse( - workflow_id=workflow.id, - generated_at=datetime.now(timezone.utc).isoformat(), - hotspots=hotspots, - ) + runtime_action = client_effects.get("runtime_action") or {} + runtime_kind = runtime_action.get("kind") + if runtime_kind == "start_inference": + lines.append("app inference run") + elif runtime_kind == "start_training": + lines.append("app training run") + elif runtime_kind == "start_proofreading": + lines.append("app proofreading start") + + workflow_action = client_effects.get("workflow_action") or {} + workflow_action_kind = workflow_action.get("kind") + if workflow_action_kind == "compute_evaluation": + lines.append("workflow metrics compute") + elif workflow_action_kind == "export_bundle": + lines.append("workflow evidence export") + elif workflow_action_kind == "propose_retraining_stage": + lines.append("workflow retraining handoff propose") + + if client_effects.get("show_workflow_context"): + lines.append("assistant status show") + + if client_effects.get("refresh_insights"): + lines.append("workflow insights refresh") + + return "\n".join(lines) or "# No app command is available for this action." + + +def _infer_action_risk(client_effects: Optional[Dict[str, Any]]) -> str: + effects = client_effects or {} + runtime_kind = (effects.get("runtime_action") or {}).get("kind") + workflow_action_kind = (effects.get("workflow_action") or {}).get("kind") + if runtime_kind in {"start_inference", "start_training"}: + return "runs_job" + if runtime_kind == "start_proofreading": + return "loads_editor" + if workflow_action_kind == "export_bundle": + return "exports_evidence" + if workflow_action_kind in {"compute_evaluation", "propose_retraining_stage"}: + return "writes_workflow_record" + if any(key.startswith("set_") for key in effects): + return "prefills_form" + if effects.get("navigate_to") or effects.get("show_workflow_context"): + return "read_only" + if effects.get("refresh_insights"): + return "read_only" + return "read_only" + + +def _requires_action_approval(client_effects: Optional[Dict[str, Any]]) -> bool: + risk = _infer_action_risk(client_effects) + return risk in { + "runs_job", + "loads_editor", + "exports_evidence", + "writes_workflow_record", + } -@router.get( - "/{workflow_id}/impact-preview", - response_model=WorkflowImpactPreviewResponse, -) -def get_workflow_impact_preview( - workflow_id: int, - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) - events = _event_rows(db, workflow.id) - hotspots = _compute_hotspots(workflow, events) - corrected_mask_path = workflow.corrected_mask_path or _latest_exported_mask_path( - db, workflow.id +def _build_agent_chat_action( + action_id: str, + label: str, + description: str, + *, + variant: str = "default", + client_effects: Optional[Dict[str, Any]] = None, + risk_level: Optional[str] = None, + requires_approval: Optional[bool] = None, + disabled_reason: Optional[str] = None, +) -> AgentChatAction: + effects = client_effects or {} + inferred_risk = risk_level or _infer_action_risk(effects) + return AgentChatAction( + id=action_id, + label=label, + description=description, + variant=variant, + risk_level=inferred_risk, + requires_approval=( + _requires_action_approval(effects) + if requires_approval is None + else requires_approval + ), + disabled_reason=disabled_reason, + client_effects=effects, ) - return _compute_impact_preview(workflow, events, hotspots, corrected_mask_path) -@router.get("/{workflow_id}/metrics", response_model=WorkflowMetricsResponse) -def get_workflow_metrics( - workflow_id: int, - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) - events = _event_rows(db, workflow.id) - return WorkflowMetricsResponse( - workflow_id=workflow.id, - metrics=compute_workflow_metrics(events), +def _build_agent_command_block( + command_id: str, + title: str, + description: str, + client_effects: Dict[str, Any], + *, + run_label: str = "Run in app", +) -> AgentCommandBlock: + risk = _infer_action_risk(client_effects) + return AgentCommandBlock( + id=command_id, + title=title, + description=description, + command=_client_effects_to_command(client_effects), + run_label=run_label, + risk_level=risk, + requires_approval=_requires_action_approval(client_effects), + client_effects=client_effects, ) -@router.post( - "/{workflow_id}/export-bundle", - response_model=WorkflowExportBundleResponse, -) -def export_workflow_bundle( - workflow_id: int, - user: auth_models.User = Depends(get_current_user), - db: Session = Depends(get_db), -): - workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) - events = _event_rows(db, workflow.id) - return WorkflowExportBundleResponse(**build_export_bundle(workflow, events)) +def _workflow_stage_to_tab(stage: Optional[str]) -> str: + return { + "setup": "files", + "visualization": "visualization", + "inference": "inference", + "proofreading": "mask-proofreading", + "retraining_staged": "training", + "evaluation": "inference", + }.get(stage or "", "files") + + +def _build_start_inference_effects(workflow: WorkflowSession) -> Dict[str, Any]: + effects: Dict[str, Any] = { + "navigate_to": "inference", + "runtime_action": {"kind": "start_inference"}, + } + if workflow.inference_output_path: + effects["set_inference_output_path"] = workflow.inference_output_path + if workflow.checkpoint_path: + effects["set_inference_checkpoint_path"] = workflow.checkpoint_path + return effects + + +def _build_start_training_effects( + workflow: WorkflowSession, corrected_mask_path: Optional[str] +) -> Dict[str, Any]: + effects: Dict[str, Any] = { + "navigate_to": "training", + "runtime_action": {"kind": "start_training"}, + } + if corrected_mask_path: + effects["set_training_label_path"] = corrected_mask_path + if workflow.training_output_path: + effects["set_training_output_path"] = workflow.training_output_path + return effects + + +def _build_start_proofreading_effects(workflow: WorkflowSession) -> Dict[str, Any]: + dataset_path = ( + workflow.image_path + or workflow.dataset_path + or workflow.inference_output_path + or "" + ) + mask_path = ( + workflow.mask_path + or workflow.inference_output_path + or workflow.corrected_mask_path + or workflow.label_path + or "" + ) + effects: Dict[str, Any] = { + "navigate_to": "mask-proofreading", + "runtime_action": {"kind": "start_proofreading"}, + "refresh_insights": True, + } + if dataset_path: + effects["set_proofreading_dataset_path"] = dataset_path + if mask_path: + effects["set_proofreading_mask_path"] = mask_path + if workflow.title: + effects["set_proofreading_project_name"] = workflow.title + return effects -@router.post("/{workflow_id}/events", response_model=WorkflowEventResponse) +def _build_default_agent_actions( + workflow: WorkflowSession, + corrected_mask_path: Optional[str], +) -> List[AgentChatAction]: + if workflow.stage == "setup": + return [ + _build_agent_chat_action( + "open-files", + "Choose data", + "Pick the image and mask for this segmentation loop.", + variant="primary", + client_effects={"navigate_to": "files"}, + ), + _build_agent_chat_action( + "open-visualization", + "View data", + "Open the image viewer after data is mounted.", + client_effects={"navigate_to": "visualization"}, + ), + _build_agent_chat_action( + "open-inference", + "Run model", + "Open model inference setup.", + client_effects={"navigate_to": "inference"}, + ), + ] + + if workflow.stage == "visualization": + can_start_proofreading = bool( + workflow.mask_path or workflow.label_path or workflow.inference_output_path + ) + return [ + _build_agent_chat_action( + "start-proofreading", + "Proofread this data", + "Open the image and mask in the proofreading workbench.", + variant="primary" if can_start_proofreading else "default", + client_effects=_build_start_proofreading_effects(workflow), + ), + _build_agent_chat_action( + "open-inference", + "Run model", + "Make a prediction for the current data.", + variant="default" if can_start_proofreading else "primary", + client_effects={"navigate_to": "inference"}, + ), + ] + + if workflow.stage == "inference": + if not workflow.inference_output_path: + return [ + _build_agent_chat_action( + "start-inference", + "Run model", + "Start the model run with the current settings.", + variant="primary", + client_effects=_build_start_inference_effects(workflow), + ), + _build_agent_chat_action( + "open-inference", + "Check settings", + "Review the checkpoint and output path before running.", + client_effects={"navigate_to": "inference"}, + ), + ] + return [ + _build_agent_chat_action( + "start-proofreading", + "Proofread this result", + "Open the image and mask in the proofreading workbench.", + variant="primary", + client_effects=_build_start_proofreading_effects(workflow), + ), + _build_agent_chat_action( + "refresh-insights", + "Refresh", + "Update the next-step recommendation.", + client_effects={"refresh_insights": True}, + ), + ] + + if workflow.stage == "proofreading": + actions = [ + _build_agent_chat_action( + "start-proofreading", + "Proofread this data", + "Open the image and mask in the proofreading workbench.", + variant=( + "primary" if not workflow.proofreading_session_id else "default" + ), + client_effects=_build_start_proofreading_effects(workflow), + ), + _build_agent_chat_action( + "refresh-insights", + "Refresh", + "Update the recommendation using the latest edits.", + client_effects={"refresh_insights": True}, + ), + ] + if corrected_mask_path: + actions.insert( + 1, + _build_agent_chat_action( + "propose-retraining-handoff", + "Use edits for training", + "Ask for approval before linking the saved edits to training.", + variant="primary", + client_effects={ + "workflow_action": { + "kind": "propose_retraining_stage", + "corrected_mask_path": corrected_mask_path, + }, + "refresh_insights": True, + }, + ), + ) + actions.insert( + 2, + _build_agent_chat_action( + "prime-training", + "Set up training", + "Open training with the saved edits already selected.", + client_effects={ + "navigate_to": "training", + "set_training_label_path": corrected_mask_path, + }, + ), + ) + return actions + + if workflow.stage == "retraining_staged": + training_effects = _build_start_training_effects(workflow, corrected_mask_path) + return [ + _build_agent_chat_action( + "start-training", + "Train on edits", + "Start training with the saved mask edits.", + variant="primary", + client_effects=training_effects, + ), + _build_agent_chat_action( + "refresh-insights", + "Refresh", + "Update the recommendation before training.", + client_effects={"refresh_insights": True}, + ), + ] + + if workflow.stage == "evaluation": + inference_effects = ( + _build_start_inference_effects(workflow) + if workflow.checkpoint_path + else {"navigate_to": "inference"} + ) + inference_label = "Run model" if workflow.checkpoint_path else "Check inference" + inference_description = ( + "Run the model with the latest checkpoint." + if workflow.checkpoint_path + else "Open inference and review the latest checkpoint." + ) + return [ + _build_agent_chat_action( + "open-inference-ready-model", + inference_label, + inference_description, + variant="primary", + client_effects=inference_effects, + ), + _build_agent_chat_action( + "open-monitoring", + "View training log", + "Open the training monitor.", + client_effects={"navigate_to": "monitoring"}, + ), + ] + + return [ + _build_agent_chat_action( + "open-workflow-stage", + "Go to next step", + "Open the active workflow screen.", + variant="primary", + client_effects={"navigate_to": _workflow_stage_to_tab(workflow.stage)}, + ) + ] + + +def _query_has(lower_query: str, terms: List[str]) -> bool: + return any(term in lower_query for term in terms) + + +def _is_greeting_query(lower_query: str) -> bool: + stripped = lower_query.strip(" \t\n\r.!?,") + return stripped in {"hi", "hello", "hey", "yo", "sup", "hiya"} + + +WORKFLOW_AGENT_COMMAND_ALIASES = { + "/status": "status", + "/next": "next step", + "/help": "what can the agent do", + "/infer": "run model", + "/inference": "run model", + "/segment": "run model to segment this volume", + "/proofread": "proofread this data", + "/train": "start training", + "/compare": "compare results and compute metrics", + "/metrics": "compare results and compute metrics", + "/export": "export evidence bundle", +} + + +def _normalize_workflow_agent_query(query: str) -> tuple[str, Optional[str]]: + stripped = query.strip() + if not stripped.startswith("/"): + return stripped, None + command, _, tail = stripped.partition(" ") + alias = WORKFLOW_AGENT_COMMAND_ALIASES.get(command.lower()) + if not alias: + return stripped, None + normalized = f"{alias}: {tail.strip()}" if tail.strip() else alias + return normalized, command.lower() + + +def _is_incomplete_work_intent(lower_query: str) -> bool: + stripped = lower_query.strip().lower().rstrip(". ") + return stripped in { + "i want", + "i wanna", + "i would like", + "i need", + "help", + "help me", + "what should i", + } + + +def _build_navigation_action( + tab_key: str, label: str, description: str +) -> AgentChatAction: + return _build_agent_chat_action( + f"open-{tab_key}", + label, + description, + variant="primary", + client_effects={"navigate_to": tab_key}, + ) + + +def _target_tab_from_query(lower_query: str) -> Optional[Dict[str, str]]: + targets = [ + ( + [ + "files", + "file management", + "mount", + "load data", + "choose data", + "project", + ], + { + "tab": "files", + "label": "Open Files", + "description": "Open project files and mounted datasets.", + }, + ), + ( + ["visualize", "visualization", "viewer", "view data"], + { + "tab": "visualization", + "label": "Open Visualize", + "description": "Open the image/label viewer.", + }, + ), + ( + ["infer", "inference", "prediction", "predict", "run model", "segment"], + { + "tab": "inference", + "label": "Open Infer", + "description": "Open model inference setup.", + }, + ), + ( + ["train", "training", "retrain"], + { + "tab": "training", + "label": "Open Train", + "description": "Open model training setup.", + }, + ), + ( + ["monitor", "tensorboard", "log"], + { + "tab": "monitoring", + "label": "Open Monitor", + "description": "Open runtime and training monitoring.", + }, + ), + ( + ["proofread", "proofreading", "fix mask", "review mask"], + { + "tab": "mask-proofreading", + "label": "Open Proofread", + "description": "Open the mask proofreading workbench.", + }, + ), + ] + navigation_words = ["go to", "open", "show", "take me", "move me", "switch"] + if not _query_has(lower_query, navigation_words): + return None + for terms, target in targets: + if _query_has(lower_query, terms): + return target + return None + + +def _latest_completed_inference_runs( + db: Session, workflow_id: int +) -> List[WorkflowModelRun]: + return ( + db.query(WorkflowModelRun) + .filter( + WorkflowModelRun.workflow_id == workflow_id, + WorkflowModelRun.run_type == "inference", + WorkflowModelRun.status == "completed", + WorkflowModelRun.output_path.isnot(None), + ) + .order_by(WorkflowModelRun.created_at.asc(), WorkflowModelRun.id.asc()) + .all() + ) + + +def _build_compute_evaluation_effects( + db: Session, workflow: WorkflowSession +) -> tuple[Dict[str, Any], List[str]]: + inference_runs = _latest_completed_inference_runs(db, workflow.id) + baseline_run = inference_runs[0] if inference_runs else None + candidate_run = inference_runs[-1] if len(inference_runs) > 1 else None + baseline_path = baseline_run.output_path if baseline_run else None + candidate_path = ( + candidate_run.output_path + if candidate_run + else ( + workflow.inference_output_path + if workflow.inference_output_path != baseline_path + else None + ) + ) + ground_truth_path = workflow.label_path or workflow.mask_path + missing = [] + if not baseline_path: + missing.append("previous result") + if not candidate_path: + missing.append("new result") + if not ground_truth_path: + missing.append("reference mask") + + effect = { + "show_workflow_context": True, + "workflow_action": { + "kind": "compute_evaluation", + "name": "workflow-before-after-evaluation", + "baseline_prediction_path": baseline_path, + "candidate_prediction_path": candidate_path, + "ground_truth_path": ground_truth_path, + "baseline_run_id": baseline_run.id if baseline_run else None, + "candidate_run_id": candidate_run.id if candidate_run else None, + "metadata": {"source": "workflow_agent"}, + }, + "refresh_insights": True, + } + return effect, missing + + +def _build_export_bundle_effects() -> Dict[str, Any]: + return { + "show_workflow_context": True, + "workflow_action": {"kind": "export_bundle"}, + "refresh_insights": True, + } + + +def _format_greeting_response( + recommendation: WorkflowAgentRecommendationResponse, +) -> str: + return ( + f"Hi. You are in {recommendation.stage.replace('_', ' ')}.\n" + f"Next: {recommendation.decision}\n" + "Tell me the job when you want me to act." + ) + + +def _readiness_item( + item_id: str, + label: str, + complete: bool, + detail: str, + *, + severity: str = "default", +) -> WorkflowReadinessItem: + return WorkflowReadinessItem( + id=item_id, + label=label, + complete=complete, + detail=detail, + severity=severity, + ) + + +def _event_count(events: List[WorkflowEvent], event_type: str) -> int: + return sum(1 for event in events if event.event_type == event_type) + + +def _build_workflow_readiness( + workflow: WorkflowSession, + events: List[WorkflowEvent], + corrected_mask_path: Optional[str], +) -> List[WorkflowReadinessItem]: + has_dataset = bool( + workflow.dataset_path + or workflow.image_path + or _event_count(events, "dataset.loaded") > 0 + ) + has_inference = bool( + workflow.inference_output_path + or _event_count(events, "inference.completed") > 0 + ) + has_proofreading = bool( + workflow.proofreading_session_id + or _event_count(events, "proofreading.session_loaded") > 0 + ) + has_edits = bool( + _event_count(events, "proofreading.mask_saved") > 0 + or _event_count(events, "proofreading.instance_classified") > 0 + ) + has_corrections = bool( + corrected_mask_path or _event_count(events, "proofreading.masks_exported") > 0 + ) + has_training = bool( + workflow.training_output_path + or workflow.checkpoint_path + or _event_count(events, "training.completed") > 0 + ) + has_evaluation = bool(_event_count(events, "evaluation.completed") > 0) + + return [ + _readiness_item( + "dataset", + "Data mounted", + has_dataset, + workflow.dataset_path + or workflow.image_path + or "No source volume recorded.", + severity="warning", + ), + _readiness_item( + "inference", + "Prediction artifact", + has_inference, + workflow.inference_output_path or "Run inference or register a prediction.", + severity="warning", + ), + _readiness_item( + "proofreading", + "Proofreading session", + has_proofreading, + f"{_event_count(events, 'proofreading.instance_classified')} classifications logged.", + ), + _readiness_item( + "edits", + "Correction evidence", + has_edits, + f"{_event_count(events, 'proofreading.mask_saved')} mask saves logged.", + ), + _readiness_item( + "corrections", + "Corrected masks exported", + has_corrections, + corrected_mask_path or "Export masks before retraining.", + severity="warning", + ), + _readiness_item( + "training", + "Candidate model", + has_training, + workflow.checkpoint_path + or workflow.training_output_path + or "Launch retraining after corrections are staged.", + ), + _readiness_item( + "evaluation", + "Before/after evidence", + has_evaluation, + ( + "Evaluation report recorded." + if has_evaluation + else "Run candidate inference and compare metrics." + ), + ), + ] + + +def _workflow_agent_decision( + workflow: WorkflowSession, + readiness: List[WorkflowReadinessItem], + impact: WorkflowImpactPreviewResponse, + hotspots: List[WorkflowHotspotItem], + corrected_mask_path: Optional[str], +) -> Dict[str, Any]: + incomplete = {item.id for item in readiness if not item.complete} + top_hotspot = hotspots[0] if hotspots else None + + if "dataset" in incomplete and workflow.stage in {"setup", "visualization"}: + return { + "decision": "Choose the image and mask first.", + "rationale": "I need the data pair before I can route the next step.", + "next_stage": "setup", + "confidence": "high", + } + if workflow.stage in {"setup", "visualization"}: + return { + "decision": "Proofread this data if the mask is ready.", + "rationale": "A human review pass is the fastest way to create useful edits.", + "next_stage": "proofreading", + "confidence": "medium", + } + if workflow.stage == "inference": + if "inference" in incomplete: + return { + "decision": "Run the model on this data.", + "rationale": "No prediction is recorded yet.", + "next_stage": "inference", + "confidence": "medium", + } + return { + "decision": "Proofread the model result.", + "rationale": "A prediction exists, so the useful next step is to review and fix it.", + "next_stage": "proofreading", + "confidence": "medium", + } + if workflow.stage == "proofreading": + if corrected_mask_path or impact.can_stage_retraining: + return { + "decision": "Use your saved edits for training.", + "rationale": "I can prepare the training step, but you approve it first.", + "next_stage": "retraining_staged", + "confidence": impact.confidence, + } + if top_hotspot: + return { + "decision": "Keep proofreading likely mistakes.", + "rationale": top_hotspot.summary, + "next_stage": "proofreading", + "confidence": impact.confidence, + } + return { + "decision": "Save or export edits before training.", + "rationale": "Training needs saved mask edits, not just a viewed slice.", + "next_stage": "proofreading", + "confidence": "low", + } + if workflow.stage == "retraining_staged": + return { + "decision": "Train on the saved edits.", + "rationale": "The edited masks are linked to the training screen.", + "next_stage": "evaluation", + "confidence": "medium", + } + if workflow.stage == "evaluation": + return { + "decision": "Compare the new result.", + "rationale": "The new checkpoint only matters if its prediction improves.", + "next_stage": "inference", + "confidence": "medium" if workflow.checkpoint_path else "low", + } + return { + "decision": "Check what is ready and choose the next step.", + "rationale": impact.summary, + "next_stage": workflow.stage, + "confidence": impact.confidence, + } + + +def _build_workflow_agent_recommendation( + db: Session, + workflow: WorkflowSession, +) -> WorkflowAgentRecommendationResponse: + event_rows = _event_rows(db, workflow.id) + event_responses = _event_list(db, workflow.id) + corrected_mask_path = workflow.corrected_mask_path or _latest_exported_mask_path( + db, workflow.id + ) + hotspots = _compute_hotspots(workflow, event_rows) + impact = _compute_impact_preview( + workflow, + event_rows, + hotspots, + corrected_mask_path, + ) + readiness = _build_workflow_readiness(workflow, event_rows, corrected_mask_path) + decision = _workflow_agent_decision( + workflow, + readiness, + impact, + hotspots, + corrected_mask_path, + ) + actions = _build_default_agent_actions(workflow, corrected_mask_path) + primary_action = next( + ( + action + for action in actions + if action.client_effects and action.variant == "primary" + ), + next((action for action in actions if action.client_effects), None), + ) + commands = [] + if primary_action: + commands.append( + _build_agent_command_block( + f"{primary_action.id}-command", + primary_action.label, + primary_action.description, + primary_action.client_effects, + ) + ) + + blockers = [ + item.detail + for item in readiness + if not item.complete and item.severity == "warning" + ] + if workflow.stage == "proofreading" and corrected_mask_path: + blockers = [] + if workflow.stage == "retraining_staged": + blockers = [] + + return WorkflowAgentRecommendationResponse( + workflow_id=workflow.id, + generated_at=datetime.now(timezone.utc).isoformat(), + stage=workflow.stage, + decision=decision["decision"], + rationale=decision["rationale"], + confidence=decision["confidence"], + next_stage=decision["next_stage"], + can_act=bool(actions), + blockers=blockers[:3], + readiness=readiness, + top_hotspot=hotspots[0] if hotspots else None, + impact_preview=impact, + actions=actions, + commands=commands, + ) + + +def _format_workflow_agent_response( + recommendation: WorkflowAgentRecommendationResponse, +) -> str: + ready_count = sum(1 for item in recommendation.readiness if item.complete) + total_count = len(recommendation.readiness) + lines = [ + f"Do this: {recommendation.decision}", + f"Why: {recommendation.rationale}", + ] + if total_count: + lines.append(f"Ready: {ready_count}/{total_count} loop checks pass.") + if recommendation.blockers: + lines.append(f"Watch out: {recommendation.blockers[0]}") + return "\n".join(lines) + + +def _short_path_label(path: Optional[str]) -> str: + if not path: + return "not set" + parts = str(path).rstrip("/").split("/") + return parts[-1] or str(path) + + +def _format_project_context_response( + workflow: WorkflowSession, + recommendation: WorkflowAgentRecommendationResponse, +) -> str: + project_name = workflow.title or f"Workflow #{workflow.id}" + image_label = _short_path_label(workflow.image_path or workflow.dataset_path) + mask_label = _short_path_label( + workflow.mask_path or workflow.label_path or workflow.inference_output_path + ) + return "\n".join( + [ + f"Project: {project_name}.", + f"Stage: {workflow.stage.replace('_', ' ')}.", + f"Image: {image_label}. Mask/result: {mask_label}.", + f"Next: {recommendation.decision}", + ] + ) + + +def _format_needed_from_user_response( + recommendation: WorkflowAgentRecommendationResponse, +) -> str: + if recommendation.stage == "proofreading": + gap = recommendation.blockers[0] if recommendation.blockers else "Save edits." + return "\n".join( + [ + "I need your mask judgment.", + "Do this: proofread likely mistakes, save fixes, then export masks.", + f"Current gap: {gap}", + ] + ) + + blocker = recommendation.blockers[0] if recommendation.blockers else None + if blocker: + return "\n".join( + [ + "I need one missing workflow input.", + f"Do this: {recommendation.decision}", + f"Current gap: {blocker}", + ] + ) + return "\n".join( + [ + "I need your approval before changing artifacts.", + f"Do this: {recommendation.decision}", + "I can run the in-app step when you approve it.", + ] + ) + + +def _workflow_agent_tasks_from_readiness( + readiness: List[WorkflowReadinessItem], +) -> List[AgentTaskItem]: + tasks: List[AgentTaskItem] = [] + for index, item in enumerate(readiness, start=1): + status = "done" if item.complete else "blocked" + if not item.complete and item.severity != "warning": + status = "pending" + tasks.append( + AgentTaskItem( + id=item.id, + label=item.label, + status=status, + detail=item.detail, + priority="high" if index <= 2 and not item.complete else "normal", + ) + ) + return tasks + + +@router.get("/current", response_model=WorkflowDetailResponse) +def get_current_workflow( + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_current_or_create_workflow(db, user_id=user.id) + return { + "workflow": _workflow_response(workflow), + "events": _event_list(db, workflow.id), + } + + +@router.patch("/{workflow_id}", response_model=WorkflowResponse) +async def update_workflow( + workflow_id: int, + body: WorkflowUpdateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + updates = body.model_dump(exclude_unset=True) + workflow = update_workflow_fields(db, workflow, updates, commit=True) + return _workflow_response(workflow) + + +@router.get("/{workflow_id}/events", response_model=List[WorkflowEventResponse]) +def list_workflow_events( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + return _event_list(db, workflow_id) + + +@router.get("/{workflow_id}/hotspots", response_model=WorkflowHotspotsResponse) +def get_workflow_hotspots( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + events = _event_rows(db, workflow.id) + hotspots = _compute_hotspots(workflow, events) + _persist_computed_hotspots(db, workflow_id=workflow.id, hotspots=hotspots) + return WorkflowHotspotsResponse( + workflow_id=workflow.id, + generated_at=datetime.now(timezone.utc).isoformat(), + hotspots=hotspots, + ) + + +@router.get( + "/{workflow_id}/impact-preview", + response_model=WorkflowImpactPreviewResponse, +) +def get_workflow_impact_preview( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + events = _event_rows(db, workflow.id) + hotspots = _compute_hotspots(workflow, events) + corrected_mask_path = workflow.corrected_mask_path or _latest_exported_mask_path( + db, workflow.id + ) + return _compute_impact_preview(workflow, events, hotspots, corrected_mask_path) + + +@router.get("/{workflow_id}/metrics", response_model=WorkflowMetricsResponse) +def get_workflow_metrics( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + events = _event_rows(db, workflow.id) + return WorkflowMetricsResponse( + workflow_id=workflow.id, + metrics=compute_workflow_metrics(events), + ) + + +@router.get( + "/{workflow_id}/agent/recommendation", + response_model=WorkflowAgentRecommendationResponse, +) +def get_workflow_agent_recommendation( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + return _build_workflow_agent_recommendation(db, workflow) + + +@router.get( + "/{workflow_id}/artifacts", + response_model=List[WorkflowArtifactResponse], +) +def list_workflow_artifacts( + workflow_id: int, + artifact_type: Optional[str] = None, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + query = db.query(WorkflowArtifact).filter( + WorkflowArtifact.workflow_id == workflow_id + ) + if artifact_type: + query = query.filter(WorkflowArtifact.artifact_type == artifact_type) + rows = query.order_by( + WorkflowArtifact.created_at.asc(), WorkflowArtifact.id.asc() + ).all() + return [_artifact_response(row) for row in rows] + + +@router.post( + "/{workflow_id}/artifacts", + response_model=WorkflowArtifactResponse, +) +def create_artifact( + workflow_id: int, + body: WorkflowArtifactCreateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type=body.artifact_type, + role=body.role, + path=body.path, + uri=body.uri, + name=body.name, + checksum=body.checksum, + metadata=body.metadata, + commit=True, + ) + return _artifact_response(artifact) + + +@router.get( + "/{workflow_id}/model-runs", + response_model=List[WorkflowModelRunResponse], +) +def list_model_runs( + workflow_id: int, + run_type: Optional[str] = None, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + query = db.query(WorkflowModelRun).filter( + WorkflowModelRun.workflow_id == workflow_id + ) + if run_type: + query = query.filter(WorkflowModelRun.run_type == run_type) + rows = query.order_by( + WorkflowModelRun.created_at.asc(), WorkflowModelRun.id.asc() + ).all() + return [_model_run_response(row) for row in rows] + + +@router.post( + "/{workflow_id}/model-runs", + response_model=WorkflowModelRunResponse, +) +def create_model_run( + workflow_id: int, + body: WorkflowModelRunCreateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + run = WorkflowModelRun( + workflow_id=workflow.id, + run_type=body.run_type, + status=body.status, + name=body.name, + config_path=body.config_path, + log_path=body.log_path, + output_path=body.output_path, + checkpoint_path=body.checkpoint_path, + input_artifact_id=body.input_artifact_id, + output_artifact_id=body.output_artifact_id, + metrics_json=encode_json(body.metrics), + metadata_json=encode_json(body.metadata), + ) + db.add(run) + db.commit() + db.refresh(run) + return _model_run_response(run) + + +@router.get( + "/{workflow_id}/model-versions", + response_model=List[WorkflowModelVersionResponse], +) +def list_model_versions( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + rows = ( + db.query(WorkflowModelVersion) + .filter(WorkflowModelVersion.workflow_id == workflow_id) + .order_by(WorkflowModelVersion.created_at.asc(), WorkflowModelVersion.id.asc()) + .all() + ) + return [_model_version_response(row) for row in rows] + + +@router.post( + "/{workflow_id}/model-versions", + response_model=WorkflowModelVersionResponse, +) +def create_model_version( + workflow_id: int, + body: WorkflowModelVersionCreateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + version = WorkflowModelVersion( + workflow_id=workflow.id, + version_label=body.version_label, + status=body.status, + checkpoint_path=body.checkpoint_path, + training_run_id=body.training_run_id, + checkpoint_artifact_id=body.checkpoint_artifact_id, + correction_set_id=body.correction_set_id, + metrics_json=encode_json(body.metrics), + metadata_json=encode_json(body.metadata), + ) + db.add(version) + db.commit() + db.refresh(version) + return _model_version_response(version) + + +@router.get( + "/{workflow_id}/correction-sets", + response_model=List[WorkflowCorrectionSetResponse], +) +def list_correction_sets( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + rows = ( + db.query(WorkflowCorrectionSet) + .filter(WorkflowCorrectionSet.workflow_id == workflow_id) + .order_by( + WorkflowCorrectionSet.created_at.asc(), WorkflowCorrectionSet.id.asc() + ) + .all() + ) + return [_correction_set_response(row) for row in rows] + + +@router.get( + "/{workflow_id}/evaluation-results", + response_model=List[WorkflowEvaluationResponse], +) +def list_evaluation_results( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + rows = ( + db.query(WorkflowEvaluationResult) + .filter(WorkflowEvaluationResult.workflow_id == workflow_id) + .order_by( + WorkflowEvaluationResult.created_at.asc(), + WorkflowEvaluationResult.id.asc(), + ) + .all() + ) + return [_evaluation_response(row) for row in rows] + + +@router.post( + "/{workflow_id}/evaluation-results", + response_model=WorkflowEvaluationResponse, +) +def create_evaluation_result( + workflow_id: int, + body: WorkflowEvaluationCreateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + report_artifact = None + if body.report_path: + report_artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type="evaluation_report", + role="case_study_evidence", + path=body.report_path, + metadata={"source": "manual_evaluation_result"}, + commit=False, + ) + result = WorkflowEvaluationResult( + workflow_id=workflow.id, + name=body.name, + baseline_run_id=body.baseline_run_id, + candidate_run_id=body.candidate_run_id, + model_version_id=body.model_version_id, + report_artifact_id=report_artifact.id if report_artifact else None, + report_path=body.report_path, + summary=body.summary, + metrics_json=encode_json(body.metrics), + metadata_json=encode_json(body.metadata), + ) + db.add(result) + db.commit() + db.refresh(result) + return _evaluation_response(result) + + +@router.post( + "/{workflow_id}/evaluation-results/compute", + response_model=WorkflowEvaluationResponse, +) +def compute_evaluation_result( + workflow_id: int, + body: WorkflowEvaluationComputeRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + try: + metrics = compute_before_after_evaluation( + baseline_prediction_path=body.baseline_prediction_path, + candidate_prediction_path=body.candidate_prediction_path, + ground_truth_path=body.ground_truth_path, + baseline_dataset=body.baseline_dataset, + candidate_dataset=body.candidate_dataset, + ground_truth_dataset=body.ground_truth_dataset, + crop=body.crop, + baseline_channel=body.baseline_channel, + candidate_channel=body.candidate_channel, + ground_truth_channel=body.ground_truth_channel, + ) + except FileNotFoundError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=500, detail=str(exc)) from exc + + metadata = { + **body.metadata, + "baseline_prediction_path": body.baseline_prediction_path, + "candidate_prediction_path": body.candidate_prediction_path, + "ground_truth_path": body.ground_truth_path, + "baseline_dataset": body.baseline_dataset, + "candidate_dataset": body.candidate_dataset, + "ground_truth_dataset": body.ground_truth_dataset, + "crop": body.crop, + "baseline_channel": body.baseline_channel, + "candidate_channel": body.candidate_channel, + "ground_truth_channel": body.ground_truth_channel, + } + summary = ( + "Before/after evaluation computed. " + f"Dice delta: {metrics.get('summary', {}).get('dice_delta')}." + ) + report_path = body.report_path + if report_path: + report_payload = { + "workflow_id": workflow.id, + "name": body.name, + "summary": summary, + "metrics": metrics, + "metadata": metadata, + } + report_path = write_evaluation_report(report_path, report_payload) + + report_artifact = None + if report_path: + report_artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type="evaluation_report", + role="case_study_evidence", + path=report_path, + metadata={"source": "computed_evaluation_result"}, + commit=False, + ) + + result = WorkflowEvaluationResult( + workflow_id=workflow.id, + name=body.name or "before-after-evaluation", + baseline_run_id=body.baseline_run_id, + candidate_run_id=body.candidate_run_id, + model_version_id=body.model_version_id, + report_artifact_id=report_artifact.id if report_artifact else None, + report_path=report_path, + summary=summary, + metrics_json=encode_json(metrics), + metadata_json=encode_json(metadata), + ) + db.add(result) + db.commit() + db.refresh(result) + return _evaluation_response(result) + + +@router.get( + "/{workflow_id}/persisted-hotspots", + response_model=List[WorkflowRegionHotspotResponse], +) +def list_persisted_hotspots( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + rows = ( + db.query(WorkflowRegionHotspot) + .filter(WorkflowRegionHotspot.workflow_id == workflow_id) + .order_by(WorkflowRegionHotspot.score.desc(), WorkflowRegionHotspot.id.asc()) + .all() + ) + return [_region_hotspot_response(row) for row in rows] + + +def _has_event(events: List[WorkflowEvent], event_type: str) -> bool: + return any(event.event_type == event_type for event in events) + + +def _case_study_readiness_gates( + workflow: WorkflowSession, events: List[WorkflowEvent] +) -> List[Dict[str, Any]]: + artifacts = list(getattr(workflow, "artifacts", []) or []) + runs = list(getattr(workflow, "model_runs", []) or []) + versions = list(getattr(workflow, "model_versions", []) or []) + correction_sets = list(getattr(workflow, "correction_sets", []) or []) + evaluations = list(getattr(workflow, "evaluation_results", []) or []) + agent_plans = list(getattr(workflow, "agent_plans", []) or []) + inference_completed = [ + run for run in runs if run.run_type == "inference" and run.status == "completed" + ] + proposal_audit_complete = bool( + _has_event(events, "agent.proposal_created") + and ( + _has_event(events, "agent.proposal_approved") + or _has_event(events, "agent.proposal_rejected") + ) + ) + agent_plan_created = bool(agent_plans or _has_event(events, "agent.plan_created")) + agent_plan_decisioned = bool( + any( + plan.approval_status in {"approved", "rejected"} + or plan.status in {"approved", "interrupted", "rejected"} + for plan in agent_plans + ) + or _has_event(events, "agent.plan_approved") + or _has_event(events, "agent.plan_rejected") + or _has_event(events, "agent.plan_interrupted") + or _has_event(events, "agent.plan_resumed") + ) + + gates = [ + { + "id": "workflow_context", + "label": "Workflow context exists", + "complete": bool(workflow.id and _has_event(events, "workflow.created")), + "required_for": ["CS1", "CS7"], + }, + { + "id": "data_loaded", + "label": "Dataset/image artifacts are linked", + "complete": bool( + workflow.image_path + or any( + a.artifact_type in {"dataset", "image_volume"} for a in artifacts + ) + or _has_event(events, "dataset.loaded") + or _has_event(events, "viewer.created") + ), + "required_for": ["CS1"], + }, + { + "id": "baseline_inference", + "label": "Baseline inference run/output is captured", + "complete": bool( + workflow.inference_output_path + or any(run.run_type == "inference" for run in runs) + or _has_event(events, "inference.completed") + ), + "required_for": ["CS1", "CS4"], + }, + { + "id": "proofreading_corrections", + "label": "Proofreading corrections are saved/exported", + "complete": bool( + workflow.corrected_mask_path + or correction_sets + or _has_event(events, "proofreading.masks_exported") + ), + "required_for": ["CS2", "CS3"], + }, + { + "id": "retraining_handoff", + "label": "Corrections can be staged for retraining", + "complete": bool( + workflow.stage in {"retraining_staged", "evaluation"} + or _has_event(events, "retraining.staged") + ), + "required_for": ["CS3", "CS5"], + }, + { + "id": "training_completion", + "label": "Retraining produces a terminal run record", + "complete": bool( + any( + run.run_type == "training" and run.status in {"completed", "failed"} + for run in runs + ) + or _has_event(events, "training.completed") + or _has_event(events, "training.failed") + ), + "required_for": ["CS3", "CS6"], + }, + { + "id": "model_version", + "label": "A versioned candidate model exists", + "complete": bool(versions or workflow.checkpoint_path), + "required_for": ["CS3", "CS4"], + }, + { + "id": "post_retraining_inference", + "label": "Post-retraining inference is captured", + "complete": len(inference_completed) >= 2, + "required_for": ["CS4"], + }, + { + "id": "evaluation_result", + "label": "Before/after evaluation evidence exists", + "complete": bool(evaluations), + "required_for": ["CS4", "CS7"], + }, + { + "id": "agent_plan_preview", + "label": "Bounded agent plan is previewed and decisioned", + "complete": bool(agent_plan_created and agent_plan_decisioned), + "required_for": ["CS5", "CS6"], + }, + { + "id": "agent_audit", + "label": "Agent proposal decisions are auditable", + "complete": bool( + proposal_audit_complete + or (agent_plan_created and agent_plan_decisioned) + ), + "required_for": ["CS5", "CS6"], + }, + ] + return gates + + +@router.get( + "/{workflow_id}/case-study-readiness", + response_model=WorkflowReadinessResponse, +) +def get_case_study_readiness( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + events = _event_rows(db, workflow.id) + gates = _case_study_readiness_gates(workflow, events) + completed_count = len([gate for gate in gates if gate["complete"]]) + next_required_items = [gate["label"] for gate in gates if not gate["complete"]][:4] + return WorkflowReadinessResponse( + workflow_id=workflow.id, + ready_for_case_study=completed_count == len(gates), + completed_count=completed_count, + total_count=len(gates), + gates=gates, + next_required_items=next_required_items, + ) + + +def _create_agent_plan_preview( + db: Session, + *, + workflow: WorkflowSession, + events: List[WorkflowEvent], + body: WorkflowAgentPlanCreateRequest, +) -> WorkflowAgentPlan: + gates = _case_study_readiness_gates(workflow, events) + graph = build_case_study_plan_graph( + workflow=workflow, + events=events, + gates=gates, + title=body.title, + goal=body.goal, + ) + plan = WorkflowAgentPlan( + workflow_id=workflow.id, + title=graph["title"], + status="draft", + risk_level="medium", + approval_status="pending", + goal=graph["goal"], + graph_json=encode_json(graph), + metadata_json=encode_json( + { + **body.metadata, + "plan_type": body.plan_type, + "source": "case_study_plan_preview", + } + ), + ) + db.add(plan) + db.flush() + + for node in graph.get("nodes", []): + step = WorkflowAgentStep( + plan_id=plan.id, + step_index=int(node.get("index", 0)), + action=str(node.get("action")), + status=str(node.get("status") or "blocked"), + requires_approval=bool(node.get("requires_approval")), + summary=node.get("summary"), + params_json=encode_json( + { + "title": node.get("title"), + "stage": node.get("stage"), + "gate_id": node.get("gate_id"), + "dependencies": node.get("dependencies") or [], + "client_effects": node.get("client_effects") or {}, + "required_for": node.get("required_for") or [], + } + ), + ) + db.add(step) + + db.flush() + event = append_workflow_event( + db, + workflow_id=workflow.id, + actor="agent", + event_type="agent.plan_created", + stage=workflow.stage, + summary=f"Created bounded agent plan: {plan.title}", + payload={ + "plan_id": plan.id, + "plan_type": body.plan_type, + "ready_steps": len( + [ + node + for node in graph.get("nodes", []) + if node.get("status") == "ready" + ] + ), + "blocked_steps": len( + [ + node + for node in graph.get("nodes", []) + if node.get("status") == "blocked" + ] + ), + "graph_spec_version": graph.get("graph_spec_version"), + }, + approval_status="pending", + commit=False, + ) + plan.source_event_id = event.id if event else None + db.commit() + db.refresh(plan) + return plan + + +@router.get( + "/{workflow_id}/agent-plans", + response_model=List[WorkflowAgentPlanResponse], +) +def list_agent_plans( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + rows = ( + db.query(WorkflowAgentPlan) + .filter(WorkflowAgentPlan.workflow_id == workflow_id) + .order_by(WorkflowAgentPlan.created_at.asc(), WorkflowAgentPlan.id.asc()) + .all() + ) + return [_agent_plan_response(row) for row in rows] + + +@router.post( + "/{workflow_id}/agent-plans", + response_model=WorkflowAgentPlanResponse, +) +def create_agent_plan( + workflow_id: int, + body: WorkflowAgentPlanCreateRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + events = _event_rows(db, workflow.id) + plan = _create_agent_plan_preview( + db, + workflow=workflow, + events=events, + body=body, + ) + return _agent_plan_response(plan) + + +@router.get( + "/{workflow_id}/agent-plans/{plan_id}", + response_model=WorkflowAgentPlanResponse, +) +def get_agent_plan( + workflow_id: int, + plan_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow_id, plan_id=plan_id) + return _agent_plan_response(plan) + + +@router.post( + "/{workflow_id}/agent-plans/{plan_id}/approve", + response_model=WorkflowAgentPlanResponse, +) +def approve_agent_plan( + workflow_id: int, + plan_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow.id, plan_id=plan_id) + if plan.approval_status == "rejected": + raise HTTPException(status_code=400, detail="Rejected plans cannot be approved") + plan.approval_status = "approved" + plan.status = "approved" + append_workflow_event( + db, + workflow_id=workflow.id, + actor="user", + event_type="agent.plan_approved", + stage=workflow.stage, + summary=f"Approved bounded agent plan: {plan.title}", + payload={"plan_id": plan.id}, + commit=False, + ) + db.commit() + db.refresh(plan) + return _agent_plan_response(plan) + + +@router.post( + "/{workflow_id}/agent-plans/{plan_id}/reject", + response_model=WorkflowAgentPlanResponse, +) +def reject_agent_plan( + workflow_id: int, + plan_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow.id, plan_id=plan_id) + plan.approval_status = "rejected" + plan.status = "rejected" + append_workflow_event( + db, + workflow_id=workflow.id, + actor="user", + event_type="agent.plan_rejected", + stage=workflow.stage, + summary=f"Rejected bounded agent plan: {plan.title}", + payload={"plan_id": plan.id}, + commit=False, + ) + db.commit() + db.refresh(plan) + return _agent_plan_response(plan) + + +@router.post( + "/{workflow_id}/agent-plans/{plan_id}/interrupt", + response_model=WorkflowAgentPlanResponse, +) +def interrupt_agent_plan( + workflow_id: int, + plan_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow.id, plan_id=plan_id) + if plan.status == "rejected": + raise HTTPException( + status_code=400, detail="Rejected plans cannot be interrupted" + ) + plan.status = "interrupted" + append_workflow_event( + db, + workflow_id=workflow.id, + actor="user", + event_type="agent.plan_interrupted", + stage=workflow.stage, + summary=f"Interrupted bounded agent plan: {plan.title}", + payload={"plan_id": plan.id}, + commit=False, + ) + db.commit() + db.refresh(plan) + return _agent_plan_response(plan) + + +@router.post( + "/{workflow_id}/agent-plans/{plan_id}/resume", + response_model=WorkflowAgentPlanResponse, +) +def resume_agent_plan( + workflow_id: int, + plan_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow.id, plan_id=plan_id) + if plan.status != "interrupted": + raise HTTPException( + status_code=400, detail="Only interrupted plans can be resumed" + ) + plan.status = "approved" if plan.approval_status == "approved" else "draft" + append_workflow_event( + db, + workflow_id=workflow.id, + actor="user", + event_type="agent.plan_resumed", + stage=workflow.stage, + summary=f"Resumed bounded agent plan: {plan.title}", + payload={"plan_id": plan.id}, + commit=False, + ) + db.commit() + db.refresh(plan) + return _agent_plan_response(plan) + + +@router.post( + "/{workflow_id}/agent-plans/{plan_id}/steps/{step_id}/approve", + response_model=WorkflowAgentStepResponse, +) +def approve_agent_plan_step( + workflow_id: int, + plan_id: int, + step_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow.id, plan_id=plan_id) + step = _get_agent_step_or_404(db, plan_id=plan.id, step_id=step_id) + if not step.requires_approval: + raise HTTPException( + status_code=400, detail="This step does not require approval" + ) + if step.status == "completed": + raise HTTPException( + status_code=400, detail="Completed steps cannot be approved" + ) + step.status = "approved" + append_workflow_event( + db, + workflow_id=workflow.id, + actor="user", + event_type="agent.plan_step_approved", + stage=workflow.stage, + summary=f"Approved plan step: {step.action}", + payload={"plan_id": plan.id, "step_id": step.id, "action": step.action}, + commit=False, + ) + db.commit() + db.refresh(step) + return _agent_step_response(step) + + +@router.post( + "/{workflow_id}/agent-plans/{plan_id}/steps/{step_id}/complete", + response_model=WorkflowAgentStepResponse, +) +def complete_agent_plan_step( + workflow_id: int, + plan_id: int, + step_id: int, + body: WorkflowAgentStepCompleteRequest, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + plan = _get_agent_plan_or_404(db, workflow_id=workflow.id, plan_id=plan_id) + step = _get_agent_step_or_404(db, plan_id=plan.id, step_id=step_id) + if body.status not in {"completed", "failed", "skipped"}: + raise HTTPException( + status_code=400, + detail="status must be one of: completed, failed, skipped", + ) + step.status = body.status + step.result_json = encode_json(body.result) + append_workflow_event( + db, + workflow_id=workflow.id, + actor="system", + event_type=f"agent.plan_step_{body.status}", + stage=workflow.stage, + summary=f"Plan step {body.status}: {step.action}", + payload={ + "plan_id": plan.id, + "step_id": step.id, + "action": step.action, + "result": body.result, + }, + commit=False, + ) + db.commit() + db.refresh(step) + return _agent_step_response(step) + + +@router.post( + "/{workflow_id}/export-bundle", + response_model=WorkflowExportBundleResponse, +) +def export_workflow_bundle( + workflow_id: int, + user: auth_models.User = Depends(get_current_user), + db: Session = Depends(get_db), +): + workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + events = _event_rows(db, workflow.id) + bundle = build_export_bundle(workflow, events) + append_workflow_event( + db, + workflow_id=workflow.id, + actor="system", + event_type="workflow.bundle_exported", + stage=workflow.stage, + summary="Exported workflow evidence bundle.", + payload={ + "artifact_count": len(bundle.get("artifacts") or []), + "model_run_count": len(bundle.get("model_runs") or []), + "model_version_count": len(bundle.get("model_versions") or []), + "correction_set_count": len(bundle.get("correction_sets") or []), + "evaluation_result_count": len(bundle.get("evaluation_results") or []), + "missing_artifact_path_count": len( + [ + path_info + for path_info in bundle.get("artifact_paths") or [] + if not path_info.get("exists") + ] + ), + }, + commit=True, + ) + return WorkflowExportBundleResponse(**bundle) + + +@router.post("/{workflow_id}/events", response_model=WorkflowEventResponse) async def create_workflow_event( workflow_id: int, body: WorkflowEventCreateRequest, @@ -781,6 +3061,78 @@ async def reject_agent_action( return _event_response(event) +def _workflow_agent_conversation_for_query( + db: Session, + *, + user_id: int, + conversation_id: Optional[int], + query: str, +) -> auth_models.Conversation: + if conversation_id: + conversation = ( + db.query(auth_models.Conversation) + .filter( + auth_models.Conversation.id == conversation_id, + auth_models.Conversation.user_id == user_id, + ) + .first() + ) + if not conversation: + raise HTTPException(status_code=404, detail="Conversation not found") + if conversation.title == "New Chat": + conversation.title = query[:120].strip() or "Workflow assistant" + return conversation + + conversation = auth_models.Conversation( + user_id=user_id, + title=query[:120].strip() or "Workflow assistant", + ) + db.add(conversation) + db.flush() + return conversation + + +def _jsonable_agent_items(items: List[Any]) -> str: + encoded_items = [] + for item in items: + if hasattr(item, "model_dump"): + encoded_items.append(item.model_dump(mode="json")) + elif isinstance(item, dict): + encoded_items.append(item) + return json.dumps(encoded_items) + + +def _persist_workflow_agent_chat_exchange( + db: Session, + *, + conversation: auth_models.Conversation, + query: str, + response: str, + actions: List[AgentChatAction], + commands: List[AgentCommandBlock], + proposals: List[WorkflowEventResponse], +) -> None: + conversation.updated_at = datetime.now(timezone.utc) + db.add( + auth_models.ChatMessage( + conversation_id=conversation.id, + role="user", + content=query, + ) + ) + db.add( + auth_models.ChatMessage( + conversation_id=conversation.id, + role="assistant", + content=response, + source="workflow_orchestrator", + actions_json=_jsonable_agent_items(actions), + commands_json=_jsonable_agent_items(commands), + proposals_json=_jsonable_agent_items(proposals), + ) + ) + + @router.post("/{workflow_id}/agent/query", response_model=AgentQueryResponse) async def query_workflow_agent( workflow_id: int, @@ -788,18 +3140,142 @@ async def query_workflow_agent( user: auth_models.User = Depends(get_current_user), db: Session = Depends(get_db), ): - if not body.query.strip(): + raw_query = body.query.strip() + if not raw_query: raise HTTPException(status_code=400, detail="query must be non-empty") + query, command_alias = _normalize_workflow_agent_query(raw_query) workflow = get_user_workflow_or_404(db, workflow_id=workflow_id, user_id=user.id) + conversation = _workflow_agent_conversation_for_query( + db, + user_id=user.id, + conversation_id=body.conversation_id or body.conversationId, + query=raw_query, + ) event_rows = _event_rows(db, workflow.id) - events = _event_list(db, workflow.id) - recommendation = _recommendation_for_workflow(workflow, events) + agent_recommendation = _build_workflow_agent_recommendation(db, workflow) proposals: List[WorkflowEventResponse] = [] - lower_query = body.query.lower() + actions: List[AgentChatAction] = [] + commands: List[AgentCommandBlock] = [] + tasks = _workflow_agent_tasks_from_readiness(agent_recommendation.readiness) + intent = "recommendation" + lower_query = query.lower() + wants_greeting = _is_greeting_query(lower_query) + wants_incomplete_intent = _is_incomplete_work_intent(lower_query) + target_tab = _target_tab_from_query(lower_query) + wants_capabilities = any( + phrase in lower_query + for phrase in [ + "what can you do", + "what can the agent do", + "what can the assistant do", + "what can it do", + "help me through", + "run things", + "guide me", + ] + ) + wants_project_context = any( + phrase in lower_query + for phrase in [ + "what project", + "which project", + "current project", + "project am i", + "project i am", + "what data", + "what dataset", + "what volume", + ] + ) + wants_user_need = any( + phrase in lower_query + for phrase in [ + "what do you need", + "what you need", + "need from me", + "need me to", + "what should i provide", + "what should i do for you", + ] + ) + wants_status = _query_has( + lower_query, + [ + "status", + "where am i", + "what now", + "what's next", + "next step", + "what should i do", + "what is left", + "what remains", + "missing", + "blocker", + "ready", + ], + ) + wants_evaluation = _query_has( + lower_query, + [ + "evaluate", + "evaluation", + "compare", + "comparison", + "metric", + "metrics", + "dice", + "iou", + "before after", + "before/after", + ], + ) + wants_export = _query_has( + lower_query, + [ + "export bundle", + "export evidence", + "export report", + "evidence bundle", + "research bundle", + "download bundle", + ], + ) wants_retraining = any( term in lower_query for term in ["retrain", "training", "stage", "corrected"] ) + wants_training_launch = any( + term in lower_query + for term in ["start training", "run training", "launch training", "retrain now"] + ) + wants_inference_launch = any( + term in lower_query + for term in [ + "start inference", + "run inference", + "launch inference", + "run model", + "start model", + "launch model", + ] + ) + wants_segmentation_launch = any( + term in lower_query + for term in [ + "segment", + "segmented", + "segmentation", + "predict", + "prediction", + "get my volume", + "process volume", + "run this volume", + ] + ) + wants_proofreading_launch = any( + term in lower_query + for term in ["proofread", "proofreading", "review mask", "fix mask"] + ) wants_failure_analysis = any( term in lower_query for term in ["fail", "failure", "error", "hotspot", "where"] ) @@ -814,7 +3290,292 @@ async def query_workflow_agent( corrected_mask_path, ) - if wants_retraining and corrected_mask_path: + if wants_greeting: + intent = "greeting" + response = _format_greeting_response(agent_recommendation) + actions = [] + commands = [] + elif wants_project_context: + intent = "project_context" + response = _format_project_context_response(workflow, agent_recommendation) + actions = [ + _build_agent_chat_action( + "show-workflow-status", + "Show status", + "Open the workflow evidence and readiness panel.", + client_effects={ + "show_workflow_context": True, + "refresh_insights": True, + }, + ) + ] + commands = [] + elif wants_user_need: + intent = "needed_from_user" + response = _format_needed_from_user_response(agent_recommendation) + actions = agent_recommendation.actions[:2] + commands = agent_recommendation.commands[:1] + elif wants_incomplete_intent: + intent = "clarify_next_job" + response = ( + "Do this: choose one workflow job.\n" + "Options: run model, proofread, use edits for training, or compare results.\n" + f"Current suggestion: {agent_recommendation.decision}" + ) + actions = agent_recommendation.actions + commands = [] + elif target_tab: + intent = "navigate" + navigation_action = _build_navigation_action( + target_tab["tab"], + target_tab["label"], + target_tab["description"], + ) + response = ( + f"Do this: {target_tab['label']}.\n" f"Why: {target_tab['description']}" + ) + actions = [navigation_action] + commands = [ + _build_agent_command_block( + f"{navigation_action.id}-command", + target_tab["label"], + target_tab["description"], + navigation_action.client_effects, + ) + ] + elif wants_export: + intent = "export_evidence" + export_effects = _build_export_bundle_effects() + response = ( + "Do this: export the workflow evidence bundle.\n" + "Why: it captures the current artifacts, runs, corrections, metrics, and audit trail." + ) + actions = [ + _build_agent_chat_action( + "export-workflow-bundle", + "Export evidence", + "Export the workflow evidence bundle.", + variant="primary", + client_effects=export_effects, + ), + _build_agent_chat_action( + "show-status", + "Show status", + "Open the workflow evidence panel first.", + client_effects={"show_workflow_context": True}, + ), + ] + commands = [ + _build_agent_command_block( + "export-workflow-bundle-command", + "Export evidence in app", + "Run this in-app command block to export the workflow evidence bundle.", + export_effects, + ) + ] + elif wants_evaluation: + intent = "compute_evaluation" + evaluation_effects, missing_eval_inputs = _build_compute_evaluation_effects( + db, workflow + ) + if missing_eval_inputs: + response = ( + f"Do this: collect {missing_eval_inputs[0]} first.\n" + "Why: before/after metrics need a previous result, a new result, and a reference mask." + ) + actions = [ + _build_agent_chat_action( + "show-evaluation-status", + "Show status", + "Open the evidence panel and inspect missing comparison inputs.", + variant="primary", + client_effects={"show_workflow_context": True}, + ), + _build_agent_chat_action( + "open-inference", + "Open Infer", + "Run or register model outputs for comparison.", + client_effects={"navigate_to": "inference"}, + ), + ] + commands = [] + else: + response = ( + "Do this: compute before/after metrics.\n" + "Why: this checks whether the new result improved against the reference mask." + ) + actions = [ + _build_agent_chat_action( + "compute-evaluation", + "Compute metrics", + "Compare the previous and new segmentation results.", + variant="primary", + client_effects=evaluation_effects, + ), + _build_agent_chat_action( + "export-workflow-bundle", + "Export evidence", + "Export the workflow evidence bundle after metrics are recorded.", + client_effects=_build_export_bundle_effects(), + ), + ] + commands = [ + _build_agent_command_block( + "compute-evaluation-command", + "Compute metrics in app", + "Run this in-app command block to compute before/after metrics.", + evaluation_effects, + ) + ] + elif wants_status: + intent = "status" + response = _format_workflow_agent_response(agent_recommendation) + actions = [ + _build_agent_chat_action( + "show-workflow-status", + "Show status", + "Open the workflow evidence and readiness panel.", + variant="primary", + client_effects={ + "show_workflow_context": True, + "refresh_insights": True, + }, + ), + *agent_recommendation.actions[:2], + ] + commands = [] + elif wants_capabilities: + intent = "capabilities" + response = ( + "Do this: tell me the job, then approve the app action.\n" + "I can run model inference, start proofreading, stage edits for training, compare metrics, export evidence, and move screens.\n" + "Watch out: long-running or artifact-changing steps stay approval-gated." + ) + actions = agent_recommendation.actions + commands = [] + elif wants_training_launch and workflow.stage == "retraining_staged": + intent = "start_training" + training_effects = _build_start_training_effects(workflow, corrected_mask_path) + response = ( + "Do this: train on the saved edits.\n" + "Why: the edited masks are already linked to this workflow." + ) + actions = [ + _build_agent_chat_action( + "start-training", + "Train on edits", + "Start training with the saved mask edits.", + variant="primary", + client_effects=training_effects, + ), + _build_agent_chat_action( + "refresh-insights", + "Refresh", + "Update the recommendation before training.", + client_effects={"refresh_insights": True}, + ), + ] + commands = [ + _build_agent_command_block( + "start-training-command", + "Start training in app", + "Run this in-app command block to launch retraining from chat.", + training_effects, + ) + ] + elif wants_inference_launch: + intent = "start_inference" + inference_effects = _build_start_inference_effects(workflow) + response = ( + "Do this: run the model.\n" + "Why: this creates the mask result you can inspect and fix." + ) + actions = [ + _build_agent_chat_action( + "start-inference", + "Run model", + "Start the model run with the current settings.", + variant="primary", + client_effects=inference_effects, + ), + _build_agent_chat_action( + "start-proofreading", + "Proofread this data", + "Open the image and mask in the proofreading workbench.", + client_effects=_build_start_proofreading_effects(workflow), + ), + ] + commands = [ + _build_agent_command_block( + "start-inference-command", + "Start inference in app", + "Run this in-app command block to launch inference from chat.", + inference_effects, + ) + ] + elif wants_proofreading_launch: + intent = "start_proofreading" + proofreading_effects = _build_start_proofreading_effects(workflow) + response = ( + "Do this: proofread this data.\n" + "Why: I found the current image/mask paths in the workflow." + ) + actions = [ + _build_agent_chat_action( + "start-proofreading", + "Proofread this data", + "Open the image and mask in the proofreading workbench.", + variant="primary", + client_effects=proofreading_effects, + ), + _build_agent_chat_action( + "refresh-insights", + "Refresh", + "Update the recommendation using the latest edits.", + client_effects={"refresh_insights": True}, + ), + ] + commands = [ + _build_agent_command_block( + "start-proofreading-command", + "Start proofreading in app", + "Open the current image/mask pair in proofreading.", + proofreading_effects, + ) + ] + elif wants_segmentation_launch: + intent = "start_segmentation" + inference_effects = _build_start_inference_effects(workflow) + proofreading_effects = _build_start_proofreading_effects(workflow) + response = ( + "Do this: run the model to segment the volume.\n" + "Why: inference creates the mask result; proofreading is for fixing it afterward." + ) + actions = [ + _build_agent_chat_action( + "start-inference", + "Run model", + "Start segmentation from the current inference settings.", + variant="primary", + client_effects=inference_effects, + ), + _build_agent_chat_action( + "start-proofreading", + "Proofread this data", + "Open the current image/mask pair in proofreading.", + client_effects=proofreading_effects, + ), + ] + commands = [ + _build_agent_command_block( + "start-segmentation-command", + "Run model in app", + "Run segmentation from chat using the current app settings.", + inference_effects, + ) + ] + elif wants_retraining and corrected_mask_path: + intent = "stage_retraining" proposal = append_workflow_event( db, workflow_id=workflow.id, @@ -831,18 +3592,115 @@ async def query_workflow_agent( ) proposals.append(_event_response(proposal)) response = ( - "I found a corrected mask artifact and prepared a retraining-stage " - "proposal. Approve it when you want the app to link those corrections " - "to the next training configuration." + "Do this: approve using these edits for training.\n" + "Why: I created a reviewable proposal instead of changing state silently." ) + training_effects = { + "navigate_to": "training", + "set_training_label_path": corrected_mask_path, + } + actions = [ + _build_agent_chat_action( + "open-training", + "Set up training", + "Open training with the saved edits selected.", + variant="primary", + client_effects=training_effects, + ), + _build_agent_chat_action( + "refresh-insights", + "Refresh", + "Update the recommendation before approving training.", + client_effects={"refresh_insights": True}, + ), + ] + commands = [ + _build_agent_command_block( + "prime-training", + "Prime the training screen", + "Run this in-app command block to move directly into the next training setup.", + training_effects, + ) + ] elif wants_failure_analysis and hotspots: + intent = "inspect_failure" top_hotspot = hotspots[0] response = ( - f"Top hotspot: {top_hotspot.summary} " - f"Recommended action: {top_hotspot.recommended_action} " - f"Impact preview: {impact.summary}" + f"Do this: {top_hotspot.recommended_action}\n" + f"Why: {top_hotspot.summary}\n" + f"Confidence: {impact.confidence}." ) + proofreading_effects = _build_start_proofreading_effects(workflow) + visualization_effects = {"navigate_to": "visualization"} + actions = [ + _build_agent_chat_action( + "start-proofreading", + "Proofread this data", + "Open the image and mask in the proofreading workbench.", + variant="primary", + client_effects=proofreading_effects, + ), + _build_agent_chat_action( + "open-visualization", + "View data", + "Inspect the region before editing.", + client_effects=visualization_effects, + ), + ] + commands = [ + _build_agent_command_block( + "inspect-hotspot", + "Inspect hotspot in app", + "Run this in-app command block to pivot the UI into hotspot review mode.", + proofreading_effects, + ) + ] else: - response = recommendation + intent = "recommendation" + response = _format_workflow_agent_response(agent_recommendation) + actions = agent_recommendation.actions + commands = [] + + _persist_workflow_agent_chat_exchange( + db, + conversation=conversation, + query=raw_query, + response=response, + actions=actions, + commands=commands, + proposals=proposals, + ) + db.commit() + append_app_event( + component="workflow_agent", + event="query_completed", + level="INFO", + message="Workflow agent query persisted", + workflow_id=workflow.id, + conversation_id=conversation.id, + user_id=user.id, + query_preview=raw_query[:160], + normalized_query_preview=query[:160], + command_alias=command_alias, + query_len=len(query), + response_len=len(response), + response_source="workflow_orchestrator", + intent=intent, + action_count=len(actions), + command_count=len(commands), + proposal_count=len(proposals), + task_count=len(tasks), + ) - return AgentQueryResponse(response=response, proposals=proposals) + return AgentQueryResponse( + response=response, + source="workflow_orchestrator", + intent=intent, + permission_mode="approval_required_for_runtime", + conversation_id=conversation.id, + conversationId=conversation.id, + proposals=proposals, + actions=actions, + commands=commands, + tasks=tasks, + ) diff --git a/server_api/workflows/service.py b/server_api/workflows/service.py index 850a5852..f4d6bacb 100644 --- a/server_api/workflows/service.py +++ b/server_api/workflows/service.py @@ -1,10 +1,23 @@ import json -from typing import Any, Dict, Optional +import os +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional from fastapi import HTTPException from sqlalchemy.orm import Session -from .db_models import WorkflowEvent, WorkflowSession +from .db_models import ( + WorkflowAgentPlan, + WorkflowAgentStep, + WorkflowArtifact, + WorkflowCorrectionSet, + WorkflowEvaluationResult, + WorkflowEvent, + WorkflowModelRun, + WorkflowModelVersion, + WorkflowRegionHotspot, + WorkflowSession, +) ALLOWED_STAGES = { "setup", @@ -35,6 +48,25 @@ def decode_json(value: Optional[str]) -> Dict[str, Any]: return parsed if isinstance(parsed, dict) else {} +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def _basename(value: Optional[str]) -> Optional[str]: + if not value: + return None + return os.path.basename(str(value).rstrip("/")) or str(value) + + +def _path_size(value: Optional[str]) -> Optional[int]: + if not value: + return None + try: + return os.path.getsize(value) + except OSError: + return None + + def validate_stage(stage: Optional[str]) -> Optional[str]: if stage is None: return None @@ -82,6 +114,160 @@ def event_to_dict(event: WorkflowEvent) -> Dict[str, Any]: } +def artifact_to_dict(artifact: WorkflowArtifact) -> Dict[str, Any]: + return { + "id": artifact.id, + "workflow_id": artifact.workflow_id, + "artifact_type": artifact.artifact_type, + "role": artifact.role, + "name": artifact.name, + "path": artifact.path, + "uri": artifact.uri, + "checksum": artifact.checksum, + "size_bytes": artifact.size_bytes, + "source_event_id": artifact.source_event_id, + "metadata_json": artifact.metadata_json, + "metadata": decode_json(artifact.metadata_json), + "created_at": artifact.created_at, + "exists": bool(artifact.path and os.path.exists(artifact.path)), + } + + +def model_run_to_dict(run: WorkflowModelRun) -> Dict[str, Any]: + return { + "id": run.id, + "workflow_id": run.workflow_id, + "run_type": run.run_type, + "status": run.status, + "name": run.name, + "config_path": run.config_path, + "log_path": run.log_path, + "output_path": run.output_path, + "checkpoint_path": run.checkpoint_path, + "input_artifact_id": run.input_artifact_id, + "output_artifact_id": run.output_artifact_id, + "source_event_id": run.source_event_id, + "metrics_json": run.metrics_json, + "metrics": decode_json(run.metrics_json), + "metadata_json": run.metadata_json, + "metadata": decode_json(run.metadata_json), + "started_at": run.started_at, + "completed_at": run.completed_at, + "created_at": run.created_at, + "updated_at": run.updated_at, + } + + +def model_version_to_dict(version: WorkflowModelVersion) -> Dict[str, Any]: + return { + "id": version.id, + "workflow_id": version.workflow_id, + "version_label": version.version_label, + "status": version.status, + "checkpoint_path": version.checkpoint_path, + "training_run_id": version.training_run_id, + "checkpoint_artifact_id": version.checkpoint_artifact_id, + "correction_set_id": version.correction_set_id, + "metrics_json": version.metrics_json, + "metrics": decode_json(version.metrics_json), + "metadata_json": version.metadata_json, + "metadata": decode_json(version.metadata_json), + "created_at": version.created_at, + } + + +def correction_set_to_dict(correction_set: WorkflowCorrectionSet) -> Dict[str, Any]: + return { + "id": correction_set.id, + "workflow_id": correction_set.workflow_id, + "artifact_id": correction_set.artifact_id, + "corrected_mask_path": correction_set.corrected_mask_path, + "source_mask_path": correction_set.source_mask_path, + "proofreading_session_id": correction_set.proofreading_session_id, + "edit_count": correction_set.edit_count, + "region_count": correction_set.region_count, + "source_event_id": correction_set.source_event_id, + "metadata_json": correction_set.metadata_json, + "metadata": decode_json(correction_set.metadata_json), + "created_at": correction_set.created_at, + } + + +def evaluation_result_to_dict( + result: WorkflowEvaluationResult, +) -> Dict[str, Any]: + return { + "id": result.id, + "workflow_id": result.workflow_id, + "name": result.name, + "baseline_run_id": result.baseline_run_id, + "candidate_run_id": result.candidate_run_id, + "model_version_id": result.model_version_id, + "report_artifact_id": result.report_artifact_id, + "report_path": result.report_path, + "summary": result.summary, + "metrics_json": result.metrics_json, + "metrics": decode_json(result.metrics_json), + "metadata_json": result.metadata_json, + "metadata": decode_json(result.metadata_json), + "created_at": result.created_at, + } + + +def region_hotspot_to_dict(hotspot: WorkflowRegionHotspot) -> Dict[str, Any]: + return { + "id": hotspot.id, + "workflow_id": hotspot.workflow_id, + "region_key": hotspot.region_key, + "score": hotspot.score, + "severity": hotspot.severity, + "status": hotspot.status, + "source": hotspot.source, + "evidence_json": hotspot.evidence_json, + "evidence": decode_json(hotspot.evidence_json), + "created_at": hotspot.created_at, + "updated_at": hotspot.updated_at, + } + + +def agent_step_to_dict(step: WorkflowAgentStep) -> Dict[str, Any]: + return { + "id": step.id, + "plan_id": step.plan_id, + "step_index": step.step_index, + "action": step.action, + "status": step.status, + "requires_approval": step.requires_approval, + "summary": step.summary, + "params_json": step.params_json, + "params": decode_json(step.params_json), + "result_json": step.result_json, + "result": decode_json(step.result_json), + "created_at": step.created_at, + "updated_at": step.updated_at, + } + + +def agent_plan_to_dict(plan: WorkflowAgentPlan) -> Dict[str, Any]: + return { + "id": plan.id, + "workflow_id": plan.workflow_id, + "title": plan.title, + "status": plan.status, + "risk_level": plan.risk_level, + "approval_status": plan.approval_status, + "goal": plan.goal, + "graph_json": plan.graph_json, + "graph": decode_json(plan.graph_json), + "metadata_json": plan.metadata_json, + "metadata": decode_json(plan.metadata_json), + "source_event_id": plan.source_event_id, + "created_at": plan.created_at, + "updated_at": plan.updated_at, + "steps": [agent_step_to_dict(step) for step in getattr(plan, "steps", [])], + } + + def workflow_to_dict(workflow: WorkflowSession) -> Dict[str, Any]: return { "id": workflow.id, @@ -105,6 +291,426 @@ def workflow_to_dict(workflow: WorkflowSession) -> Dict[str, Any]: } +def create_workflow_artifact( + db: Session, + *, + workflow_id: int, + artifact_type: str, + role: Optional[str] = None, + path: Optional[str] = None, + uri: Optional[str] = None, + name: Optional[str] = None, + checksum: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, + source_event_id: Optional[int] = None, + commit: bool = False, +) -> WorkflowArtifact: + normalized_path = str(path) if path else None + existing = None + if normalized_path: + existing = ( + db.query(WorkflowArtifact) + .filter( + WorkflowArtifact.workflow_id == workflow_id, + WorkflowArtifact.artifact_type == artifact_type, + WorkflowArtifact.role == role, + WorkflowArtifact.path == normalized_path, + ) + .order_by(WorkflowArtifact.id.desc()) + .first() + ) + if existing: + changed = False + if metadata: + merged = {**decode_json(existing.metadata_json), **metadata} + existing.metadata_json = encode_json(merged) + changed = True + if source_event_id and not existing.source_event_id: + existing.source_event_id = source_event_id + changed = True + if changed: + db.flush() + if commit: + db.commit() + db.refresh(existing) + return existing + + artifact = WorkflowArtifact( + workflow_id=workflow_id, + artifact_type=artifact_type, + role=role, + name=name or _basename(normalized_path or uri), + path=normalized_path, + uri=uri, + checksum=checksum, + size_bytes=_path_size(normalized_path), + source_event_id=source_event_id, + metadata_json=encode_json(metadata), + ) + db.add(artifact) + db.flush() + if commit: + db.commit() + db.refresh(artifact) + return artifact + + +def _correction_region_key(payload: Dict[str, Any]) -> Optional[str]: + for key in ("region_key", "region_id", "region"): + value = payload.get(key) + if isinstance(value, (str, int, float)): + return str(value) + instance_id = payload.get("instance_id") + if isinstance(instance_id, (str, int)): + return f"instance:{instance_id}" + z_index = payload.get("z_index") + if z_index is not None: + axis = payload.get("axis") or "z" + return f"{axis}:{z_index}" + return None + + +def _correction_stats_from_events( + db: Session, + *, + workflow_id: int, + metadata: Optional[Dict[str, Any]] = None, +) -> Dict[str, int]: + def to_int(value: Any, fallback: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return fallback + + metadata = metadata or {} + explicit_edit_count = metadata.get("edit_count") + explicit_region_count = metadata.get("region_count") + if explicit_edit_count is not None and explicit_region_count is not None: + return { + "edit_count": int(explicit_edit_count or 0), + "region_count": int(explicit_region_count or 0), + } + + events = ( + db.query(WorkflowEvent) + .filter( + WorkflowEvent.workflow_id == workflow_id, + WorkflowEvent.event_type.in_( + [ + "proofreading.instance_classified", + "proofreading.mask_saved", + "proofreading.masks_exported", + ] + ), + ) + .all() + ) + edit_count = 0 + region_keys = set() + for event in events: + payload = decode_json(event.payload_json) + region_key = _correction_region_key(payload) + if region_key: + region_keys.add(region_key) + if event.event_type == "proofreading.mask_saved": + edit_count += 1 + + return { + "edit_count": to_int(explicit_edit_count, edit_count) + if explicit_edit_count is not None + else edit_count, + "region_count": to_int(explicit_region_count, len(region_keys)) + if explicit_region_count is not None + else len(region_keys), + } + + +def create_or_update_correction_set( + db: Session, + *, + workflow: WorkflowSession, + corrected_mask_path: str, + source_event_id: Optional[int] = None, + metadata: Optional[Dict[str, Any]] = None, + commit: bool = False, +) -> WorkflowCorrectionSet: + correction_stats = _correction_stats_from_events( + db, + workflow_id=workflow.id, + metadata=metadata, + ) + artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type="correction_set", + role="corrected_mask", + path=corrected_mask_path, + metadata=metadata, + source_event_id=source_event_id, + commit=False, + ) + existing = ( + db.query(WorkflowCorrectionSet) + .filter( + WorkflowCorrectionSet.workflow_id == workflow.id, + WorkflowCorrectionSet.corrected_mask_path == corrected_mask_path, + ) + .order_by(WorkflowCorrectionSet.id.desc()) + .first() + ) + if existing: + existing.artifact_id = artifact.id + existing.source_mask_path = workflow.mask_path + existing.proofreading_session_id = workflow.proofreading_session_id + existing.source_event_id = existing.source_event_id or source_event_id + existing.edit_count = correction_stats["edit_count"] + existing.region_count = correction_stats["region_count"] + existing.metadata_json = encode_json( + {**decode_json(existing.metadata_json), **(metadata or {})} + ) + db.flush() + if commit: + db.commit() + db.refresh(existing) + return existing + + correction_set = WorkflowCorrectionSet( + workflow_id=workflow.id, + artifact_id=artifact.id, + corrected_mask_path=corrected_mask_path, + source_mask_path=workflow.mask_path, + proofreading_session_id=workflow.proofreading_session_id, + edit_count=correction_stats["edit_count"], + region_count=correction_stats["region_count"], + source_event_id=source_event_id, + metadata_json=encode_json(metadata), + ) + db.add(correction_set) + db.flush() + if commit: + db.commit() + db.refresh(correction_set) + return correction_set + + +def _latest_incomplete_run( + db: Session, *, workflow_id: int, run_type: str +) -> Optional[WorkflowModelRun]: + return ( + db.query(WorkflowModelRun) + .filter( + WorkflowModelRun.workflow_id == workflow_id, + WorkflowModelRun.run_type == run_type, + WorkflowModelRun.status.in_(["pending", "running"]), + ) + .order_by(WorkflowModelRun.created_at.desc(), WorkflowModelRun.id.desc()) + .first() + ) + + +def create_or_update_model_run_from_event( + db: Session, + *, + workflow: WorkflowSession, + event: WorkflowEvent, + payload: Dict[str, Any], +) -> Optional[WorkflowModelRun]: + event_type = event.event_type + run_type = None + status = None + if event_type == "training.started": + run_type, status = "training", "running" + elif event_type == "training.completed": + run_type, status = "training", "completed" + elif event_type == "training.failed": + run_type, status = "training", "failed" + elif event_type == "inference.started": + run_type, status = "inference", "running" + elif event_type == "inference.completed": + run_type, status = "inference", "completed" + elif event_type == "inference.failed": + run_type, status = "inference", "failed" + if not run_type: + return None + + output_path = ( + payload.get("outputPath") + or payload.get("output_path") + or payload.get("training_output_path") + or payload.get("inference_output_path") + ) + checkpoint_path = ( + payload.get("checkpointPath") + or payload.get("checkpoint_path") + or payload.get("checkpoint") + ) + config_path = payload.get("configOriginPath") or payload.get("config_path") + log_path = payload.get("logPath") or payload.get("log_path") + now = _now() + + run = ( + _latest_incomplete_run(db, workflow_id=workflow.id, run_type=run_type) + if status in {"completed", "failed"} + else None + ) + if run is None: + run = WorkflowModelRun( + workflow_id=workflow.id, + run_type=run_type, + status=status, + source_event_id=event.id, + started_at=now if status == "running" else None, + ) + db.add(run) + + run.status = status + run.config_path = config_path or run.config_path + run.log_path = log_path or run.log_path + run.output_path = output_path or run.output_path + run.checkpoint_path = checkpoint_path or run.checkpoint_path + run.source_event_id = run.source_event_id or event.id + run.metadata_json = encode_json( + {**decode_json(run.metadata_json), "last_event_type": event_type} + ) + if status == "running" and not run.started_at: + run.started_at = now + if status in {"completed", "failed"}: + run.completed_at = now + + if output_path and run_type == "inference": + run.output_artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type="inference_output", + role="prediction", + path=output_path, + source_event_id=event.id, + metadata={"run_type": run_type, "status": status}, + commit=False, + ) + elif output_path and run_type == "training": + run.output_artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type="training_output", + role="run_directory", + path=output_path, + source_event_id=event.id, + metadata={"run_type": run_type, "status": status}, + commit=False, + ) + + db.flush() + + if run_type == "training" and status == "completed" and checkpoint_path: + checkpoint_artifact = create_workflow_artifact( + db, + workflow_id=workflow.id, + artifact_type="model_checkpoint", + role="candidate_checkpoint", + path=checkpoint_path, + source_event_id=event.id, + metadata={"model_run_id": run.id}, + commit=False, + ) + label = payload.get("checkpointName") or _basename(checkpoint_path) + version = WorkflowModelVersion( + workflow_id=workflow.id, + version_label=label or f"model-run-{run.id}", + status="candidate", + checkpoint_path=checkpoint_path, + training_run_id=run.id, + checkpoint_artifact_id=checkpoint_artifact.id, + metrics_json=encode_json(payload.get("metrics") or {}), + metadata_json=encode_json({"source_event_id": event.id}), + ) + db.add(version) + db.flush() + + return run + + +def _artifact_specs_for_event( + event_type: str, payload: Dict[str, Any] +) -> List[Dict[str, Any]]: + specs: List[Dict[str, Any]] = [] + + def add(path_key: str, artifact_type: str, role: str) -> None: + path = payload.get(path_key) + if path: + specs.append( + { + "artifact_type": artifact_type, + "role": role, + "path": str(path), + "metadata": {"source_payload_key": path_key}, + } + ) + + if event_type in {"dataset.loaded", "viewer.created"}: + add("dataset_path", "dataset", "source_dataset") + add("image_path", "image_volume", "image") + add("label_path", "label_volume", "label") + add("mask_path", "mask_volume", "mask") + add("ground_truth_path", "label_volume", "ground_truth") + add("source_ground_truth_path", "label_volume", "ground_truth") + if event_type in {"proofreading.masks_exported", "retraining.staged"}: + add("written_path", "correction_set", "corrected_mask") + add("output_path", "correction_set", "corrected_mask") + add("corrected_mask_path", "correction_set", "corrected_mask") + add("backup_path", "mask_volume", "backup_mask") + if event_type.startswith("training."): + add("outputPath", "training_output", "run_directory") + add("output_path", "training_output", "run_directory") + add("checkpointPath", "model_checkpoint", "candidate_checkpoint") + add("checkpoint_path", "model_checkpoint", "candidate_checkpoint") + if event_type.startswith("inference."): + add("outputPath", "inference_output", "prediction") + add("output_path", "inference_output", "prediction") + add("checkpointPath", "model_checkpoint", "input_checkpoint") + add("checkpoint_path", "model_checkpoint", "input_checkpoint") + return specs + + +def materialize_event_artifacts( + db: Session, + *, + workflow: WorkflowSession, + event: WorkflowEvent, + payload: Dict[str, Any], +) -> None: + for spec in _artifact_specs_for_event(event.event_type, payload): + create_workflow_artifact( + db, + workflow_id=workflow.id, + source_event_id=event.id, + commit=False, + **spec, + ) + + corrected_mask_path = ( + payload.get("corrected_mask_path") + or payload.get("written_path") + or payload.get("output_path") + ) + if ( + event.event_type in {"proofreading.masks_exported", "retraining.staged"} + and corrected_mask_path + ): + create_or_update_correction_set( + db, + workflow=workflow, + corrected_mask_path=str(corrected_mask_path), + source_event_id=event.id, + metadata={"source_event_type": event.event_type}, + commit=False, + ) + + create_or_update_model_run_from_event( + db, workflow=workflow, event=event, payload=payload + ) + + def get_user_workflow_or_404( db: Session, *, workflow_id: int, user_id: int ) -> WorkflowSession: @@ -170,7 +776,9 @@ def update_workflow_fields( ) -> WorkflowSession: for key, value in updates.items(): if key == "metadata": - workflow.metadata_json = encode_json(value if isinstance(value, dict) else {}) + workflow.metadata_json = encode_json( + value if isinstance(value, dict) else {} + ) continue if key == "metadata_json": workflow.metadata_json = value @@ -216,6 +824,17 @@ def append_workflow_event( approval_status=approval_status, ) db.add(event) + db.flush() + workflow = ( + db.query(WorkflowSession).filter(WorkflowSession.id == workflow_id).first() + ) + if workflow: + materialize_event_artifacts( + db, + workflow=workflow, + event=event, + payload=payload or {}, + ) if commit: db.commit() db.refresh(event) diff --git a/server_api/workflows/volume_io.py b/server_api/workflows/volume_io.py new file mode 100644 index 00000000..d68d75f2 --- /dev/null +++ b/server_api/workflows/volume_io.py @@ -0,0 +1,475 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, List, Optional, Sequence, Tuple, Union + +import numpy as np + +CropSpec = Union[str, Sequence[slice], None] + +COMMON_DATASET_NAMES = ( + "data", + "main", + "volume", + "vol", + "raw", + "image", + "images", + "label", + "labels", + "seg", + "segmentation", + "prediction", + "predictions", +) + +SUPPORTED_VOLUME_FORMATS = ( + "TIFF/OME-TIFF: .tif, .tiff, .ome.tif, .ome.tiff", + "HDF5: .h5, .hdf5, .hdf", + "NumPy: .npy, .npz", + "Zarr/N5: .zarr, .n5", + "Common 2D images: .png, .jpg, .jpeg, .bmp", + "Optional NIfTI if nibabel is installed: .nii, .nii.gz", + "Optional MRC if mrcfile is installed: .mrc, .map, .rec", +) + + +def split_dataset_ref(path: str) -> Tuple[str, Optional[str]]: + if "::" not in path: + return path, None + file_path, dataset_key = path.split("::", 1) + return file_path, dataset_key or None + + +def parse_crop(crop: CropSpec) -> Optional[Tuple[slice, ...]]: + if crop is None: + return None + if isinstance(crop, (list, tuple)) and all(isinstance(item, slice) for item in crop): + return tuple(crop) + if not isinstance(crop, str): + raise ValueError("crop must be a string like '0:16,0:256,0:256'") + + normalized = crop.strip().lower() + if normalized in {"", "none", "full", ":"}: + return None + + slices: List[slice] = [] + for part in normalized.split(","): + token = part.strip() + if not token: + raise ValueError(f"Invalid empty crop token in {crop!r}") + if ":" not in token: + index = int(token) + slices.append(slice(index, index + 1)) + continue + pieces = token.split(":") + if len(pieces) > 3: + raise ValueError(f"Invalid crop token {token!r}") + values = [int(piece) if piece else None for piece in pieces] + while len(values) < 3: + values.append(None) + slices.append(slice(values[0], values[1], values[2])) + return tuple(slices) + + +def _normalize_crop_for_shape( + crop: Optional[Tuple[slice, ...]], shape: Sequence[int] +) -> Union[slice, Tuple[slice, ...]]: + if crop is None: + return slice(None) + if len(crop) > len(shape): + raise ValueError(f"Crop has {len(crop)} dimensions but array has {len(shape)}") + return tuple(list(crop) + [slice(None)] * (len(shape) - len(crop))) + + +def _infer_channel_axis( + shape: Sequence[int], + *, + channel: Optional[int], + reference_ndim: Optional[int], + label: str, +) -> Optional[int]: + if channel is None: + return None + if channel < 0: + raise ValueError(f"{label} channel must be non-negative") + + reference_ndim = reference_ndim or max(1, len(shape) - 1) + if len(shape) == reference_ndim: + raise ValueError( + f"{label} volume is already {reference_ndim}D; " + "do not pass a channel selector for this artifact." + ) + if len(shape) != reference_ndim + 1: + raise ValueError( + f"{label} volume has shape {tuple(shape)}; expected " + f"{reference_ndim}D or {reference_ndim + 1}D for channel selection." + ) + + if shape[0] > channel and shape[0] <= 16: + return 0 + if shape[-1] > channel and shape[-1] <= 16: + return len(shape) - 1 + if shape[0] > channel: + return 0 + if shape[-1] > channel: + return len(shape) - 1 + raise ValueError( + f"{label} channel {channel} is out of bounds for volume shape {tuple(shape)}" + ) + + +def _normalize_crop_for_output( + crop: Optional[Tuple[slice, ...]], output_ndim: int +) -> List[slice]: + if crop is None: + return [slice(None)] * output_ndim + if len(crop) > output_ndim: + raise ValueError(f"Crop has {len(crop)} dimensions but array has {output_ndim}") + return list(crop) + [slice(None)] * (output_ndim - len(crop)) + + +def _array_index( + data: Any, + crop: Optional[Tuple[slice, ...]] = None, + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> Union[int, slice, Tuple[Any, ...]]: + shape = getattr(data, "shape", ()) + if channel is None: + return _normalize_crop_for_shape(crop, shape) + + channel_axis = _infer_channel_axis( + shape, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + output_ndim = len(shape) - 1 + crop_key = _normalize_crop_for_output(crop, output_ndim) + key: List[Any] = [] + crop_index = 0 + for axis_index in range(len(shape)): + if axis_index == channel_axis: + key.append(channel) + continue + key.append(crop_key[crop_index]) + crop_index += 1 + return tuple(key) + + +def _as_array( + data: Any, + crop: Optional[Tuple[slice, ...]] = None, + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + if crop is None and channel is None: + return np.asarray(data) + key = _array_index( + data, + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + return np.asarray(data[key]) + + +def _collect_h5_datasets(handle: Any) -> List[str]: + import h5py + + datasets: List[str] = [] + + def visitor(name: str, obj: Any) -> None: + if isinstance(obj, h5py.Dataset): + datasets.append(name) + + handle.visititems(visitor) + return datasets + + +def _select_h5_dataset(handle: Any, dataset_key: Optional[str]) -> Any: + import h5py + + if dataset_key: + if dataset_key not in handle: + raise ValueError(f"HDF5 dataset {dataset_key!r} not found") + target = handle[dataset_key] + if not isinstance(target, h5py.Dataset): + raise ValueError(f"HDF5 path {dataset_key!r} is not a dataset") + return target + + for name in COMMON_DATASET_NAMES: + if name in handle and isinstance(handle[name], h5py.Dataset): + return handle[name] + + datasets = _collect_h5_datasets(handle) + if not datasets: + raise ValueError("HDF5 file does not contain any datasets") + return handle[datasets[0]] + + +def _read_hdf5( + path: Path, + dataset_key: Optional[str], + crop: Optional[Tuple[slice, ...]], + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + import h5py + + with h5py.File(path, "r") as handle: + dataset = _select_h5_dataset(handle, dataset_key) + return _as_array( + dataset, + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + +def _is_zarr_array(value: Any) -> bool: + return hasattr(value, "shape") and hasattr(value, "dtype") and hasattr(value, "__getitem__") + + +def _collect_zarr_arrays(group: Any, prefix: str = "") -> List[str]: + arrays: List[str] = [] + try: + keys = list(group.keys()) + except Exception: + return arrays + for key in keys: + child = group[key] + child_path = f"{prefix}/{key}" if prefix else str(key) + if _is_zarr_array(child): + arrays.append(child_path) + else: + arrays.extend(_collect_zarr_arrays(child, child_path)) + return arrays + + +def _select_zarr_array(store: Any, dataset_key: Optional[str]) -> Any: + if _is_zarr_array(store): + if dataset_key: + raise ValueError("Dataset key was provided, but Zarr path is already an array") + return store + + if dataset_key: + return store[dataset_key] + + for name in COMMON_DATASET_NAMES: + try: + child = store[name] + except Exception: + continue + if _is_zarr_array(child): + return child + + arrays = _collect_zarr_arrays(store) + if not arrays: + raise ValueError("Zarr/N5 store does not contain any arrays") + return store[arrays[0]] + + +def _read_zarr( + path: Path, + dataset_key: Optional[str], + crop: Optional[Tuple[slice, ...]], + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + import zarr + + store = zarr.open(str(path), mode="r") + array = _select_zarr_array(store, dataset_key) + return _as_array( + array, + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + +def _read_npz( + path: Path, + dataset_key: Optional[str], + crop: Optional[Tuple[slice, ...]], + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + with np.load(path) as loaded: + key = dataset_key + if not key: + for candidate in COMMON_DATASET_NAMES: + if candidate in loaded: + key = candidate + break + if not key: + keys = list(loaded.keys()) + if not keys: + raise ValueError("NPZ file does not contain any arrays") + key = keys[0] + return _as_array( + loaded[key], + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + +def _read_nifti( + path: Path, + crop: Optional[Tuple[slice, ...]], + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + try: + import nibabel as nib + except Exception as exc: # pragma: no cover - optional dependency + raise RuntimeError("nibabel is required to read NIfTI volumes") from exc + + image = nib.load(str(path)) + data = image.dataobj + return _as_array( + data, + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + +def _read_mrc( + path: Path, + crop: Optional[Tuple[slice, ...]], + *, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + try: + import mrcfile + except Exception as exc: # pragma: no cover - optional dependency + raise RuntimeError("mrcfile is required to read MRC/MAP volumes") from exc + + with mrcfile.open(str(path), permissive=True) as handle: + return _as_array( + handle.data, + crop, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + +def load_volume( + path: str, + *, + dataset_key: Optional[str] = None, + crop: CropSpec = None, + channel: Optional[int] = None, + reference_ndim: Optional[int] = None, + label: str = "volume", +) -> np.ndarray: + file_path, inline_dataset_key = split_dataset_ref(str(path)) + dataset_key = dataset_key or inline_dataset_key + target = Path(file_path).expanduser() + if not target.exists(): + raise FileNotFoundError(f"Volume artifact does not exist: {target}") + + crop_slices = parse_crop(crop) + lower_name = target.name.lower() + lower_path = str(target).lower() + + if lower_name.endswith((".h5", ".hdf5", ".hdf")): + return _read_hdf5( + target, + dataset_key, + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if lower_name.endswith((".tif", ".tiff", ".ome.tif", ".ome.tiff")): + import tifffile + + return _as_array( + tifffile.imread(str(target)), + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if lower_name.endswith(".npy"): + return _as_array( + np.load(target, mmap_mode="r"), + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if lower_name.endswith(".npz"): + return _read_npz( + target, + dataset_key, + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if target.is_dir() or lower_name.endswith((".zarr", ".n5")): + return _read_zarr( + target, + dataset_key, + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if lower_path.endswith((".nii", ".nii.gz")): + return _read_nifti( + target, + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if lower_name.endswith((".mrc", ".map", ".rec")): + return _read_mrc( + target, + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + if lower_name.endswith((".png", ".jpg", ".jpeg", ".bmp")): + import imageio.v3 as iio + + return _as_array( + iio.imread(target), + crop_slices, + channel=channel, + reference_ndim=reference_ndim, + label=label, + ) + + raise ValueError( + f"Unsupported volume format for {target}. Supported formats: " + + "; ".join(SUPPORTED_VOLUME_FORMATS) + ) diff --git a/server_pytc/main.py b/server_pytc/main.py index dc5110a6..004432fa 100644 --- a/server_pytc/main.py +++ b/server_pytc/main.py @@ -1,11 +1,15 @@ +import logging +import time import uvicorn from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware +from app_event_logger import append_app_event, configure_process_logging from runtime_settings import get_allowed_origins from server_pytc.services.model import ( get_inference_status, get_inference_logs, get_tensorboard, + get_tensorboard_status, initialize_tensorboard, get_training_logs as get_training_process_logs, get_training_status as get_training_process_status, @@ -15,13 +19,8 @@ stop_training, ) -print("\n" + "=" * 80) -print("SERVER_PYTC STARTING UP") -print(f"Python executable: {__import__('sys').executable}") -print(f"Working directory: {__import__('os').getcwd()}") -print("=" * 80 + "\n") - app = FastAPI() +logger = logging.getLogger(__name__) app.add_middleware( CORSMiddleware, @@ -32,6 +31,58 @@ ) +@app.on_event("startup") +async def configure_app_event_logging(): + log_path = configure_process_logging("server_pytc") + logger.info("App event logging enabled at %s", log_path) + + +@app.middleware("http") +async def log_http_requests(request: Request, call_next): + start_time = time.perf_counter() + path = request.url.path + method = request.method + client_host = request.client.host if request.client else None + append_app_event( + component="server_pytc", + event="http_request_started", + level="INFO", + message=f"{method} {path}", + method=method, + path=path, + client_host=client_host, + ) + try: + response = await call_next(request) + except Exception as exc: + append_app_event( + component="server_pytc", + event="http_request_failed", + level="ERROR", + message=f"{method} {path} failed", + method=method, + path=path, + client_host=client_host, + latency_ms=round((time.perf_counter() - start_time) * 1000, 2), + error_type=exc.__class__.__name__, + error=str(exc), + ) + raise + + append_app_event( + component="server_pytc", + event="http_request_completed", + level="INFO", + message=f"{method} {path} -> {response.status_code}", + method=method, + path=path, + client_host=client_host, + status_code=response.status_code, + latency_ms=round((time.perf_counter() - start_time) * 1000, 2), + ) + return response + + @app.get("/hello") def hello(): return {"hello"} @@ -41,6 +92,19 @@ def hello(): async def start_model_training(req: Request): print("\n========== SERVER_PYTC: START_MODEL_TRAINING ENDPOINT CALLED ==========") req = await req.json() + append_app_event( + component="server_pytc", + event="training_request_received", + level="INFO", + message="Training request received by worker", + source="worker_endpoint", + payload_keys=sorted(req.keys()), + config_origin_path=req.get("configOriginPath"), + output_path=req.get("outputPath"), + log_path=req.get("logPath"), + training_config_length=len(req.get("trainingConfig") or ""), + workflow_id=req.get("workflow_id") or req.get("workflowId"), + ) print(f"[SERVER_PYTC] Received request payload keys: {list(req.keys())}") print(f"[SERVER_PYTC] Arguments: {req.get('arguments', {})}") print(f"[SERVER_PYTC] Log path: {req.get('logPath', 'NOT PROVIDED')}") @@ -83,13 +147,15 @@ async def training_logs(): @app.get("/start_tensorboard") async def start_tensorboard(logPath: str | None = None): - if not logPath: - raise HTTPException( - status_code=400, - detail="Missing required query parameter: logPath", - ) - initialize_tensorboard(logPath) - return {"status": "started", "url": get_tensorboard()} + try: + status = initialize_tensorboard(logPath) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return { + "status": status["phase"], + "url": get_tensorboard(), + "tensorboard": status, + } @app.get("/get_tensorboard_url") @@ -97,9 +163,28 @@ async def get_tensorboard_url(): return get_tensorboard() +@app.get("/get_tensorboard_status") +async def tensorboard_status(): + return get_tensorboard_status() + + @app.post("/start_model_inference") async def start_model_inference(req: Request): req = await req.json() + append_app_event( + component="server_pytc", + event="inference_request_received", + level="INFO", + message="Inference request received by worker", + source="worker_endpoint", + payload_keys=sorted(req.keys()), + config_origin_path=req.get("configOriginPath"), + output_path=req.get("outputPath"), + checkpoint_path=req.get("checkpointPath") + or (req.get("arguments") or {}).get("checkpoint"), + inference_config_length=len(req.get("inferenceConfig") or ""), + workflow_id=req.get("workflow_id") or req.get("workflowId"), + ) print("start model inference") return start_inference(req) @@ -121,6 +206,13 @@ async def inference_logs(): def run(): + log_path = configure_process_logging("server_pytc") + print("\n" + "=" * 80) + print("SERVER_PYTC STARTING UP") + print(f"Python executable: {__import__('sys').executable}") + print(f"Working directory: {__import__('os').getcwd()}") + print(f"App event log: {log_path}") + print("=" * 80 + "\n") print("\n" + "=" * 80) print("SERVER_PYTC: Starting Uvicorn server on port 4243...") print("=" * 80 + "\n") diff --git a/server_pytc/services/model.py b/server_pytc/services/model.py index 89180645..f7e5d54d 100644 --- a/server_pytc/services/model.py +++ b/server_pytc/services/model.py @@ -1,25 +1,36 @@ import atexit import pathlib +import socket import subprocess import sys import tempfile import threading +import time from collections import deque from datetime import datetime, timezone from typing import Any +from urllib import error as urllib_error, request as urllib_request import psutil +from app_event_logger import append_app_event # Track spawned processes so we can stop/poll cleanly. _training_process = None _inference_process = None +_tensorboard_process = None _temp_files = { "training": [], "inference": [], } -tensorboard_url = None _RUNTIME_LOG_LIMIT = 2000 +_TENSORBOARD_LOG_LIMIT = 200 +_TENSORBOARD_HOST = "127.0.0.1" +_TENSORBOARD_BIND_HOST = "0.0.0.0" +_TENSORBOARD_PREFERRED_PORT = 6006 +_TENSORBOARD_START_TIMEOUT_SECONDS = 15.0 +_TENSORBOARD_POLL_INTERVAL_SECONDS = 0.25 _runtime_lock = threading.Lock() +_tensorboard_lock = threading.Lock() def _new_runtime_state(): @@ -47,6 +58,84 @@ def _new_runtime_state(): } +def _new_tensorboard_state(): + return { + "phase": "idle", + "pid": None, + "port": None, + "url": None, + "command": None, + "logdirSpec": None, + "startedAt": None, + "endedAt": None, + "lastUpdatedAt": None, + "lastError": None, + "lineCount": 0, + "lines": deque(maxlen=_TENSORBOARD_LOG_LIMIT), + } + + +_tensorboard_state = _new_tensorboard_state() +_tensorboard_sources: dict[str, dict[str, str]] = {} +_DIRECT_VOLUME_SUFFIXES = ( + ".h5", + ".hdf5", + ".hdf", + ".tif", + ".tiff", + ".ome.tif", + ".ome.tiff", + ".zarr", + ".n5", + ".npy", + ".npz", + ".nii", + ".nii.gz", + ".mrc", + ".map", + ".rec", + ".png", + ".jpg", + ".jpeg", + ".bmp", +) +_PREDICTION_OUTPUT_SUFFIXES = ( + ".h5", + ".hdf5", + ".hdf", + ".tif", + ".tiff", + ".ome.tif", + ".ome.tiff", + ".zarr", + ".n5", + ".npy", + ".npz", + ".nii", + ".nii.gz", + ".mrc", + ".map", + ".rec", +) +_DIRECT_VOLUME_CONFIG_PATHS = { + ("DATASET", "IMAGE_NAME"): "image", + ("DATASET", "LABEL_NAME"): "label", + ("DATASET", "VALID_MASK_NAME"): "label", + ("INFERENCE", "IMAGE_NAME"): "image", +} +_FRACTIONAL_FLOAT_CONFIG_PATHS = { + ("DATASET", "VALID_RATIO"), + ("DATASET", "REJECT_SAMPLING", "P"), + ("DATASET", "MEAN"), + ("DATASET", "STD"), + ("SOLVER", "BASE_LR"), + ("SOLVER", "WEIGHT_DECAY"), + ("SOLVER", "MOMENTUM"), + ("SOLVER", "WARMUP_FACTOR"), +} +_VALID_INFERENCE_AUG_NUMS = (4, 8, 16) + + def _project_root() -> pathlib.Path: return pathlib.Path(__file__).resolve().parent.parent.parent @@ -55,6 +144,24 @@ def _utc_now() -> str: return datetime.now(timezone.utc).isoformat() +def _level_for_log_text(text: str | None, *, default: str = "INFO") -> str: + normalized = (text or "").upper() + if any( + needle in normalized + for needle in ( + "TRACEBACK", + "EXCEPTION", + "ERROR", + "FAILED", + "UNICODEDECODEERROR", + ) + ): + return "ERROR" + if "WARN" in normalized: + return "WARNING" + return default + + def _get_runtime_process(kind: str): if kind == "training": return _training_process @@ -95,24 +202,114 @@ def _update_runtime_state(kind: str, **updates): state["lastUpdatedAt"] = _utc_now() -def _append_runtime_log(kind: str, line: str): +def _merge_runtime_metadata(kind: str, **updates) -> dict[str, Any]: + clean_updates = { + key: value + for key, value in updates.items() + if value is not None and value != "" + } + with _runtime_lock: + state = _runtime_state[kind] + metadata = dict(state.get("metadata") or {}) + metadata.update(clean_updates) + state["metadata"] = metadata + state["lastUpdatedAt"] = _utc_now() + return dict(metadata) + + +def _runtime_event_fields(kind: str) -> dict[str, Any]: + with _runtime_lock: + state = _runtime_state[kind] + return { + "runtime_kind": kind, + "runtime_phase": state["phase"], + "runtime_pid": state["pid"], + "runtime_exit_code": state["exitCode"], + "runtime_config_path": state["configPath"], + "runtime_config_origin_path": state["configOriginPath"], + } + + +def _emit_runtime_app_event( + kind: str, + event: str, + message: str, + *, + level: str | None = None, + **fields, +): + append_app_event( + component="server_pytc", + event=event, + level=level or _level_for_log_text(message), + message=message, + **_runtime_event_fields(kind), + **fields, + ) + + +def _append_runtime_log( + kind: str, + line: str, + *, + event: str = "runtime_log_line", + level: str | None = None, + source: str = "runtime", + **fields, +): timestamp = _utc_now() text = "" if line is None else str(line).rstrip("\n") + effective_level = level or _level_for_log_text(text) entry = f"[{timestamp}] {text}" if text else f"[{timestamp}]" + line_index = None with _runtime_lock: state = _runtime_state[kind] state["lines"].append(entry) state["lineCount"] += 1 state["lastUpdatedAt"] = timestamp + if text and effective_level == "ERROR": + state["lastError"] = text + line_index = state["lineCount"] + + if text: + _emit_runtime_app_event( + kind, + event, + text, + level=effective_level, + source=source, + runtime_line_index=line_index, + runtime_line_timestamp=timestamp, + **fields, + ) -def _append_runtime_event(kind: str, message: str): - _append_runtime_log(kind, f"[MODEL.PY] {message}") +def _append_runtime_event( + kind: str, + message: str, + *, + event: str = "runtime_event", + level: str = "INFO", + **fields, +): + _append_runtime_log( + kind, + f"[MODEL.PY] {message}", + event=event, + level=level, + source="model", + **fields, + ) def _set_runtime_error(kind: str, message: str): _update_runtime_state(kind, lastError=message) - _append_runtime_event(kind, f"ERROR: {message}") + _append_runtime_event( + kind, + f"ERROR: {message}", + event="runtime_error", + level="ERROR", + ) def _get_runtime_snapshot(kind: str) -> dict[str, Any]: @@ -151,6 +348,582 @@ def _get_runtime_snapshot(kind: str) -> dict[str, Any]: "lines": lines, "text": "\n".join(lines), } + if kind == "training" and not snapshot["isRunning"]: + metadata = snapshot.get("metadata") or {} + if not metadata.get("checkpointPath"): + latest_checkpoint = _find_latest_checkpoint(metadata.get("outputPath")) + if latest_checkpoint: + metadata = _merge_runtime_metadata( + kind, + checkpointPath=latest_checkpoint, + latestCheckpointPath=latest_checkpoint, + latestCheckpointName=pathlib.Path(latest_checkpoint).name, + ) + snapshot["metadata"] = metadata + if kind == "inference" and not snapshot["isRunning"]: + metadata = snapshot.get("metadata") or {} + if not (metadata.get("predictionPath") or metadata.get("latestPredictionPath")): + latest_prediction = _find_latest_prediction_output(metadata.get("outputPath")) + if latest_prediction: + metadata = _merge_runtime_metadata( + kind, + predictionPath=latest_prediction, + latestPredictionPath=latest_prediction, + latestPredictionName=pathlib.Path(latest_prediction).name, + ) + snapshot["metadata"] = metadata + return snapshot + + +def _tensorboard_sources_payload() -> dict[str, dict[str, str | bool]]: + payload: dict[str, dict[str, str | bool]] = {} + for name, source in _tensorboard_sources.items(): + path = pathlib.Path(source["path"]) + payload[name] = { + "name": source["name"], + "path": str(path), + "exists": path.exists(), + "registeredAt": source["registeredAt"], + } + return payload + + +def _update_tensorboard_state(**updates): + with _tensorboard_lock: + for key, value in updates.items(): + if key == "lines": + _tensorboard_state["lines"] = deque( + value, + maxlen=_TENSORBOARD_LOG_LIMIT, + ) + else: + _tensorboard_state[key] = value + _tensorboard_state["lastUpdatedAt"] = _utc_now() + + +def _append_tensorboard_log(line: str): + timestamp = _utc_now() + text = "" if line is None else str(line).rstrip("\n") + entry = f"[{timestamp}] {text}" if text else f"[{timestamp}]" + line_index = None + phase = None + pid = None + port = None + url = None + with _tensorboard_lock: + _tensorboard_state["lines"].append(entry) + _tensorboard_state["lineCount"] += 1 + _tensorboard_state["lastUpdatedAt"] = timestamp + line_index = _tensorboard_state["lineCount"] + phase = _tensorboard_state["phase"] + pid = _tensorboard_state["pid"] + port = _tensorboard_state["port"] + url = _tensorboard_state["url"] + + if text: + append_app_event( + component="server_pytc", + event="tensorboard_log_line", + level=_level_for_log_text(text), + message=text, + source="tensorboard", + tensorboard_phase=phase, + tensorboard_pid=pid, + tensorboard_port=port, + tensorboard_url=url, + tensorboard_line_index=line_index, + tensorboard_line_timestamp=timestamp, + ) + + +def _set_tensorboard_error(message: str): + _update_tensorboard_state(lastError=message) + _append_tensorboard_log(f"[TENSORBOARD] ERROR: {message}") + + +def _load_yaml_config(config_text: str) -> dict[str, Any]: + yaml = _load_yaml_module() + if yaml is None: + return {} + + try: + config_obj = yaml.safe_load(config_text) or {} + except Exception: + return {} + + return config_obj if isinstance(config_obj, dict) else {} + + +def _load_yaml_module(): + try: + import yaml # type: ignore + except Exception: + return None + return yaml + + +def _load_yaml_config_from_path(path_value: pathlib.Path | None) -> dict[str, Any]: + if path_value is None or not path_value.exists(): + return {} + try: + return _load_yaml_config(path_value.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _get_nested_value(data: dict[str, Any], path: tuple[str, ...]) -> Any: + cursor: Any = data + for key in path: + if not isinstance(cursor, dict) or key not in cursor: + return None + cursor = cursor[key] + return cursor + + +def _set_nested_value(data: dict[str, Any], path: tuple[str, ...], value: Any) -> bool: + if not path: + return False + cursor: Any = data + for key in path[:-1]: + if not isinstance(cursor, dict): + return False + if key not in cursor or not isinstance(cursor[key], dict): + cursor[key] = {} + cursor = cursor[key] + if not isinstance(cursor, dict): + return False + cursor[path[-1]] = value + return True + + +def _path_label(path: tuple[str, ...]) -> str: + return ".".join(path) + + +def _config_path_exists(path_value: Any) -> bool: + if not isinstance(path_value, str) or not path_value.strip(): + return False + try: + return pathlib.Path(path_value).expanduser().exists() + except Exception: + return False + + +def _resolve_missing_direct_volume_path(path_value: Any, role: str) -> str | None: + if not isinstance(path_value, str) or not path_value.strip(): + return None + + candidate = pathlib.Path(path_value).expanduser() + if candidate.exists(): + return str(candidate.resolve()) + + suffix = candidate.suffix.lower() + if suffix not in _DIRECT_VOLUME_SUFFIXES: + return None + + parent = candidate.parent + stem = candidate.stem + if not parent.exists(): + return None + + role_suffixes = ( + ("_im", "_img", "_image") + if role == "image" + else ("_seg", "_mask", "_label", "_gt", "_consensus") + ) + + for extra in role_suffixes: + maybe = parent / f"{stem}{extra}{suffix}" + if maybe.exists(): + return str(maybe.resolve()) + + matches = sorted(parent.glob(f"{stem}_*{suffix}")) + if not matches: + return None + + preferred = [] + for extra in role_suffixes: + preferred.extend([match for match in matches if match.stem.endswith(extra)]) + unique_preferred = list(dict.fromkeys(preferred)) + if len(unique_preferred) == 1: + return str(unique_preferred[0].resolve()) + if len(matches) == 1: + return str(matches[0].resolve()) + return None + + +def _sanitize_fractional_float_paths( + config_obj: dict[str, Any], + origin_obj: dict[str, Any], +) -> list[dict[str, Any]]: + changes: list[dict[str, Any]] = [] + if not origin_obj: + return changes + + for path in _FRACTIONAL_FLOAT_CONFIG_PATHS: + current_value = _get_nested_value(config_obj, path) + origin_value = _get_nested_value(origin_obj, path) + if not isinstance(origin_value, float) or isinstance(current_value, bool): + continue + if not isinstance(current_value, int): + continue + if float(origin_value).is_integer(): + continue + if float(current_value) == float(origin_value): + continue + _set_nested_value(config_obj, path, float(origin_value)) + changes.append( + { + "path": _path_label(path), + "old_value": current_value, + "new_value": float(origin_value), + "reason": "restored_fractional_origin_value", + } + ) + + return changes + + +def _sanitize_direct_volume_paths( + config_obj: dict[str, Any], + origin_obj: dict[str, Any], +) -> list[dict[str, Any]]: + changes: list[dict[str, Any]] = [] + + for path, role in _DIRECT_VOLUME_CONFIG_PATHS.items(): + current_value = _get_nested_value(config_obj, path) + if not isinstance(current_value, str) or not current_value.strip(): + continue + if _config_path_exists(current_value): + continue + + origin_value = _get_nested_value(origin_obj, path) + replacement = None + reason = None + if isinstance(origin_value, str) and origin_value.strip() and _config_path_exists( + origin_value + ): + replacement = origin_value + reason = "restored_existing_origin_path" + else: + resolved = _resolve_missing_direct_volume_path(current_value, role) + if resolved: + replacement = resolved + reason = "resolved_missing_direct_volume_path" + + if replacement and replacement != current_value: + _set_nested_value(config_obj, path, replacement) + changes.append( + { + "path": _path_label(path), + "old_value": current_value, + "new_value": replacement, + "reason": reason, + } + ) + + return changes + + +def _sanitize_inference_aug_num( + config_obj: dict[str, Any], + origin_obj: dict[str, Any], +) -> list[dict[str, Any]]: + path = ("INFERENCE", "AUG_NUM") + current_value = _get_nested_value(config_obj, path) + if not isinstance(current_value, int) or current_value in _VALID_INFERENCE_AUG_NUMS: + return [] + + origin_value = _get_nested_value(origin_obj, path) + if isinstance(origin_value, int) and origin_value in _VALID_INFERENCE_AUG_NUMS: + replacement = origin_value + reason = "restored_valid_origin_aug_num" + else: + replacement = min( + _VALID_INFERENCE_AUG_NUMS, + key=lambda option: abs(option - current_value), + ) + reason = "coerced_to_supported_aug_num" + + _set_nested_value(config_obj, path, replacement) + return [ + { + "path": _path_label(path), + "old_value": current_value, + "new_value": replacement, + "reason": reason, + } + ] + + +def _sanitize_runtime_config_text( + config_text: str, + config_origin_path: str | None, +) -> tuple[str, list[dict[str, Any]]]: + yaml = _load_yaml_module() + if yaml is None: + return config_text, [] + + config_obj = _load_yaml_config(config_text) + if not config_obj: + return config_text, [] + + origin_obj = _load_yaml_config_from_path( + _resolve_config_origin_path(config_origin_path) + ) + changes: list[dict[str, Any]] = [] + changes.extend(_sanitize_fractional_float_paths(config_obj, origin_obj)) + changes.extend(_sanitize_direct_volume_paths(config_obj, origin_obj)) + changes.extend(_sanitize_inference_aug_num(config_obj, origin_obj)) + + if not changes: + return config_text, [] + + sanitized_text = yaml.safe_dump( + config_obj, + sort_keys=False, + allow_unicode=False, + ) + return sanitized_text, changes + + +def _looks_like_direct_volume(path_value: Any) -> bool: + text = str(path_value or "").strip().lower() + return bool(text) and text.endswith(_DIRECT_VOLUME_SUFFIXES) + + +def _detect_chunk_tile_mismatch(config_text: str) -> dict[str, Any] | None: + config_obj = _load_yaml_config(config_text) + dataset = config_obj.get("DATASET") or {} + if not isinstance(dataset, dict): + return None + + do_chunk_title = dataset.get("DO_CHUNK_TITLE") + if do_chunk_title in (None, False, 0, "0", "false", "False"): + return None + + image_name = dataset.get("IMAGE_NAME") + label_name = dataset.get("LABEL_NAME") + if not any(_looks_like_direct_volume(value) for value in (image_name, label_name)): + return None + + return { + "code": "tile_dataset_direct_volume_mismatch", + "message": ( + "DATASET.DO_CHUNK_TITLE is enabled, so PyTC will use TileDataset and expect " + "JSON tile manifests, but IMAGE_NAME/LABEL_NAME point to direct volume files. " + "That leads to json.load(...) on the H5/TIFF path and crashes with a decode error." + ), + "do_chunk_title": do_chunk_title, + "image_name": image_name, + "label_name": label_name, + } + + +def _resolve_tensorboard_log_dir(path_value: str | None) -> pathlib.Path | None: + if not path_value or not str(path_value).strip(): + return None + + candidate = pathlib.Path(str(path_value).strip()).expanduser() + if not candidate.is_absolute(): + candidate = (_project_root() / candidate).resolve() + else: + candidate = candidate.resolve() + + candidate.mkdir(parents=True, exist_ok=True) + return candidate + + +def _register_tensorboard_source(source_key: str, log_dir: str) -> str: + resolved = _resolve_tensorboard_log_dir(log_dir) + if resolved is None: + raise ValueError("TensorBoard log directory is required") + + _tensorboard_sources[source_key] = { + "name": source_key, + "path": str(resolved), + "registeredAt": _utc_now(), + } + _append_tensorboard_log( + f"[TENSORBOARD] Watching {source_key} logs at {resolved}" + ) + append_app_event( + component="server_pytc", + event="tensorboard_source_registered", + level="INFO", + message=f"Registered TensorBoard source {source_key}", + source="tensorboard", + tensorboard_source_key=source_key, + tensorboard_source_path=str(resolved), + ) + return str(resolved) + + +def _build_tensorboard_logdir_spec() -> str | None: + parts: list[str] = [] + for source_key in sorted(_tensorboard_sources): + source = _tensorboard_sources[source_key] + parts.append(f"{source['name']}:{source['path']}") + return ",".join(parts) if parts else None + + +def _is_port_available(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + sock.bind((_TENSORBOARD_HOST, port)) + except OSError: + return False + return True + + +def _pick_tensorboard_port(preferred: int | None = None) -> int: + candidate_ports = [] + if preferred: + candidate_ports.append(int(preferred)) + if _TENSORBOARD_PREFERRED_PORT not in candidate_ports: + candidate_ports.append(_TENSORBOARD_PREFERRED_PORT) + + for port in candidate_ports: + if _is_port_available(port): + return port + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind((_TENSORBOARD_HOST, 0)) + return int(sock.getsockname()[1]) + + +def _tensorboard_url_for_port(port: int | None) -> str | None: + if not port: + return None + return f"http://{_TENSORBOARD_HOST}:{int(port)}/" + + +def _wait_for_tensorboard_ready(port: int, process: subprocess.Popen) -> bool: + url = _tensorboard_url_for_port(port) + deadline = time.monotonic() + _TENSORBOARD_START_TIMEOUT_SECONDS + + while time.monotonic() < deadline: + if process.poll() is not None: + return False + + try: + with urllib_request.urlopen(url, timeout=1) as response: + if response.status < 500: + return True + except urllib_error.URLError: + pass + except Exception: + pass + + time.sleep(_TENSORBOARD_POLL_INTERVAL_SECONDS) + + return False + + +def _terminate_managed_tensorboard(): + global _tensorboard_process + + process = _tensorboard_process + if process is None: + return + + try: + if process.poll() is None: + process.terminate() + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + except Exception as exc: + _append_tensorboard_log( + f"[TENSORBOARD] Error terminating existing process: {exc}" + ) + finally: + if _tensorboard_process is process: + _tensorboard_process = None + + +def _watch_tensorboard_process(process: subprocess.Popen): + global _tensorboard_process + + try: + if process.stdout is not None: + for line in process.stdout: + _append_tensorboard_log(line.rstrip()) + process.wait() + except Exception as exc: + _append_tensorboard_log( + f"[TENSORBOARD] Error reading TensorBoard output: {exc}" + ) + return + + exit_code = process.returncode + with _tensorboard_lock: + current_pid = _tensorboard_state["pid"] + is_current = _tensorboard_process is process + if is_current: + _tensorboard_process = None + + if current_pid != process.pid and not is_current: + return + + updates = { + "phase": "finished" if exit_code == 0 else "failed", + "pid": None, + "endedAt": _utc_now(), + } + if exit_code not in (None, 0): + updates["lastError"] = f"TensorBoard exited with code {exit_code}" + _update_tensorboard_state(**updates) + append_app_event( + component="server_pytc", + event="tensorboard_process_finished", + level="INFO" if exit_code == 0 else "ERROR", + message="TensorBoard process exited", + source="tensorboard", + tensorboard_exit_code=exit_code, + tensorboard_pid=process.pid, + ) + + +def _get_tensorboard_snapshot() -> dict[str, Any]: + process = _tensorboard_process + is_running = bool(process and process.poll() is None) + with _tensorboard_lock: + phase = _tensorboard_state["phase"] + pid = _tensorboard_state["pid"] + ended_at = _tensorboard_state["endedAt"] + last_error = _tensorboard_state["lastError"] + lines = list(_tensorboard_state["lines"]) + line_count = _tensorboard_state["lineCount"] + port = _tensorboard_state["port"] + + if not is_running and phase in {"starting", "running"} and process is not None: + exit_code = process.returncode + phase = "finished" if exit_code == 0 else "failed" + pid = None + ended_at = ended_at or _utc_now() + if exit_code not in (None, 0) and not last_error: + last_error = f"TensorBoard exited with code {exit_code}" + + snapshot = { + "isRunning": is_running, + "phase": phase, + "pid": process.pid if is_running else pid, + "port": port, + "url": _tensorboard_state["url"] or _tensorboard_url_for_port(port), + "command": _tensorboard_state["command"], + "logdirSpec": _tensorboard_state["logdirSpec"], + "startedAt": _tensorboard_state["startedAt"], + "endedAt": ended_at, + "lastUpdatedAt": _tensorboard_state["lastUpdatedAt"], + "lastError": last_error, + "lineCount": line_count, + "lines": lines, + "text": "\n".join(lines), + "sources": _tensorboard_sources_payload(), + } return snapshot @@ -258,16 +1031,68 @@ def _start_logged_process(command: list[str], cwd: pathlib.Path, label: str, kin lastError=None, ) _append_runtime_event(kind, f"Spawned {label.lower()} process PID {process.pid}") + _emit_runtime_app_event( + kind, + "runtime_process_spawned", + f"{label} subprocess spawned", + level="INFO", + source="subprocess", + command=command, + cwd=str(cwd), + subprocess_pid=process.pid, + subprocess_label=label, + ) def _log_subprocess_output(): print(f"[MODEL.PY] === {label} subprocess output (PID {process.pid}) ===") try: if process.stdout is not None: for line in process.stdout: - _append_runtime_log(kind, line.rstrip()) + _append_runtime_log( + kind, + line.rstrip(), + event="runtime_output_line", + source="subprocess", + stream="stdout", + subprocess_label=label, + subprocess_pid=process.pid, + ) print(f"[{label}:{process.pid}] {line.rstrip()}") process.wait() exit_code = process.returncode + if exit_code == 0: + discovered_artifacts = _discover_runtime_artifacts(kind) + if discovered_artifacts: + merged_metadata = _merge_runtime_metadata( + kind, + **discovered_artifacts, + ) + artifact_path = ( + discovered_artifacts.get("latestCheckpointPath") + or discovered_artifacts.get("latestPredictionPath") + ) + artifact_label = ( + "checkpoint" if kind == "training" else "prediction output" + ) + _append_runtime_event( + kind, + f"Latest {artifact_label} discovered: {artifact_path}", + event="runtime_artifact_discovered", + checkpoint_path=discovered_artifacts.get("latestCheckpointPath"), + prediction_path=discovered_artifacts.get("latestPredictionPath"), + output_path=merged_metadata.get("outputPath"), + ) + elif kind == "training": + output_path = (_get_runtime_snapshot(kind).get("metadata") or {}).get( + "outputPath" + ) + _append_runtime_event( + kind, + f"No checkpoint artifact was found in {output_path or 'the training output directory'}.", + event="runtime_artifact_missing", + level="WARNING", + output_path=output_path, + ) _update_runtime_state( kind, phase="finished" if exit_code == 0 else "failed", @@ -277,6 +1102,17 @@ def _log_subprocess_output(): _append_runtime_event( kind, f"{label} subprocess finished with exit code: {exit_code}", + event="runtime_process_finished", + ) + _emit_runtime_app_event( + kind, + "runtime_process_finished", + f"{label} subprocess exited", + level="INFO" if exit_code == 0 else "ERROR", + source="subprocess", + subprocess_label=label, + subprocess_pid=process.pid, + exit_code=exit_code, ) _clear_runtime_process(kind, process) print( @@ -291,17 +1127,8 @@ def _log_subprocess_output(): def _extract_output_path_from_yaml(config_text: str, mode: str) -> str | None: - try: - import yaml - except Exception: - return None - - try: - config_obj = yaml.safe_load(config_text) or {} - except Exception: - return None - - if not isinstance(config_obj, dict): + config_obj = _load_yaml_config(config_text) + if not config_obj: return None if mode == "train": @@ -329,10 +1156,169 @@ def _extract_output_path_from_yaml(config_text: str, mode: str) -> str | None: return None +def _resolve_runtime_output_dir(path_value: str | None) -> pathlib.Path | None: + if not path_value or not str(path_value).strip(): + return None + + candidate = pathlib.Path(str(path_value).strip()).expanduser() + if not candidate.is_absolute(): + candidate = (_project_root() / candidate).resolve() + else: + candidate = candidate.resolve() + + if candidate.is_file(): + return candidate.parent + if candidate.exists(): + return candidate + return None + + +def _find_latest_checkpoint(output_path: str | None) -> str | None: + output_dir = _resolve_runtime_output_dir(output_path) + if output_dir is None or not output_dir.exists(): + return None + + candidates: list[pathlib.Path] = [] + for pattern in ("checkpoint_*.pth.tar", "checkpoint_*.pth", "*.ckpt"): + candidates.extend(path for path in output_dir.glob(pattern) if path.is_file()) + + if not candidates: + return None + + latest = max(candidates, key=lambda path: (path.stat().st_mtime, path.name)) + return str(latest.resolve()) + + +def _path_has_prediction_suffix(path: pathlib.Path) -> bool: + lower_name = path.name.lower() + lower_path = str(path).lower() + return lower_name.endswith(_PREDICTION_OUTPUT_SUFFIXES) or lower_path.endswith( + (".ome.tif", ".ome.tiff", ".nii.gz") + ) + + +def _is_prediction_output_candidate(path: pathlib.Path) -> bool: + if path.name.startswith("."): + return False + if not _path_has_prediction_suffix(path): + return False + if path.is_dir(): + return path.name.lower().endswith((".zarr", ".n5")) + return path.is_file() + + +def _prediction_output_priority(path: pathlib.Path) -> tuple[float, int, str]: + name = path.name.lower() + preferred = 0 + if name.startswith("result_xy"): + preferred = 3 + elif name.startswith("result"): + preferred = 2 + elif "prediction" in name or "pred" in name: + preferred = 1 + try: + mtime = path.stat().st_mtime + except OSError: + mtime = 0.0 + return (mtime, preferred, path.name) + + +def _resolve_prediction_search_dir(path_value: str | None) -> pathlib.Path | None: + if not path_value or not str(path_value).strip(): + return None + + candidate = pathlib.Path(str(path_value).strip()).expanduser() + if not candidate.is_absolute(): + candidate = (_project_root() / candidate).resolve() + else: + candidate = candidate.resolve() + + if _is_prediction_output_candidate(candidate): + return candidate.parent + if candidate.is_dir(): + return candidate + if candidate.parent.exists(): + return candidate.parent + return None + + +def _find_latest_prediction_output(output_path: str | None) -> str | None: + if not output_path or not str(output_path).strip(): + return None + + requested = pathlib.Path(str(output_path).strip()).expanduser() + if not requested.is_absolute(): + requested = (_project_root() / requested).resolve() + else: + requested = requested.resolve() + if _is_prediction_output_candidate(requested): + return str(requested) + + output_dir = _resolve_prediction_search_dir(output_path) + if output_dir is None or not output_dir.exists(): + return None + + candidates: list[pathlib.Path] = [] + for child in output_dir.rglob("*"): + if _is_prediction_output_candidate(child): + candidates.append(child) + + if not candidates: + return None + + latest = max(candidates, key=_prediction_output_priority) + return str(latest.resolve()) + + +def _discover_runtime_artifacts(kind: str) -> dict[str, Any]: + snapshot = _get_runtime_snapshot(kind) + metadata = snapshot.get("metadata") or {} + output_path = metadata.get("outputPath") + + if kind == "inference": + latest_prediction = _find_latest_prediction_output(output_path) + if not latest_prediction: + return {} + return { + "predictionPath": latest_prediction, + "latestPredictionPath": latest_prediction, + "latestPredictionName": pathlib.Path(latest_prediction).name, + } + + if kind != "training": + return {} + + latest_checkpoint = _find_latest_checkpoint(output_path) + if not latest_checkpoint: + return {} + + return { + "checkpointPath": latest_checkpoint, + "latestCheckpointPath": latest_checkpoint, + "latestCheckpointName": pathlib.Path(latest_checkpoint).name, + } + + def _launch_tensorboard(log_dir: str | None, config_text: str, mode: str): resolved = log_dir or _extract_output_path_from_yaml(config_text, mode) if resolved: - initialize_tensorboard(resolved) + source_key = "training" if mode == "train" else "inference" + try: + status = initialize_tensorboard(resolved, source_key=source_key) + _append_runtime_event( + source_key, + f"TensorBoard available at {status['url']}", + event="tensorboard_ready", + tensorboard_url=status["url"], + tensorboard_port=status["port"], + ) + except Exception as exc: + _append_runtime_event( + source_key, + f"TensorBoard could not be started automatically: {exc}", + event="tensorboard_error", + level="ERROR", + ) return resolved @@ -350,6 +1336,7 @@ def start_training(payload: dict): config_text = payload.get("trainingConfig", "") temp_filepath = None config_origin_path = payload.get("configOriginPath") + config_corrections: list[dict[str, Any]] = [] _reset_runtime_state( "training", phase="starting", @@ -363,6 +1350,10 @@ def start_training(payload: dict): try: current_dir = _project_root() script_path = _pytc_script_path() + config_text, config_corrections = _sanitize_runtime_config_text( + config_text, + config_origin_path, + ) temp_filepath = _write_temp_config( config_text, "training", @@ -375,6 +1366,14 @@ def start_training(payload: dict): ) _append_runtime_event("training", f"Config origin: {config_origin_path or 'none'}") _append_runtime_event("training", f"Staged config path: {temp_filepath}") + if config_corrections: + _append_runtime_event( + "training", + f"Sanitized staged training config with {len(config_corrections)} correction(s)", + event="runtime_config_sanitized", + level="WARNING", + corrections=config_corrections, + ) command = [ sys.executable, @@ -391,6 +1390,33 @@ def start_training(payload: dict): print(f"[MODEL.PY] Final training command: {' '.join(command)}") _append_runtime_event("training", f"Final training command: {' '.join(command)}") + _emit_runtime_app_event( + "training", + "runtime_config_snapshot", + "Training config staged", + level="INFO", + source="model", + config_origin_path=config_origin_path, + staged_config_path=temp_filepath, + command=command, + arguments=payload.get("arguments", {}) or {}, + output_path=payload.get("outputPath"), + log_path=payload.get("logPath"), + config_text=config_text, + config_text_length=len(config_text or ""), + config_line_count=(config_text or "").count("\n") + (1 if config_text else 0), + config_sanitized=bool(config_corrections), + config_corrections=config_corrections, + ) + config_diagnostic = _detect_chunk_tile_mismatch(config_text) + if config_diagnostic: + _append_runtime_event( + "training", + config_diagnostic["message"], + event="runtime_config_warning", + level="WARNING", + diagnostic=config_diagnostic, + ) _training_process = _start_logged_process( command, current_dir, @@ -507,39 +1533,136 @@ def stop_training(): _training_process = None stop_pytc_processes("train") - stop_tensorboard() cleanup_temp_files("training") _update_runtime_state("training", phase="stopped", endedAt=_utc_now()) _append_runtime_event("training", "Training stop requested") return {"status": "stopped"} -def initialize_tensorboard(logPath): - global tensorboard_url +def initialize_tensorboard(logPath: str | None = None, source_key: str = "manual"): + global _tensorboard_process - print(f"[MODEL.PY] initialize_tensorboard called with logPath: {logPath}") - from tensorboard import program + if logPath: + resolved = _register_tensorboard_source(source_key, logPath) + else: + resolved = None - tb = program.TensorBoard() - try: - tb.configure(argv=[None, "--logdir", logPath, "--host", "0.0.0.0"]) - tensorboard_url = tb.launch() - print(f"[MODEL.PY] ✓ TensorBoard is running at {tensorboard_url}") - except Exception as exc: - tensorboard_url = "http://localhost:6006/" - print( - f"[MODEL.PY] ⚠ TensorBoard fallback to {tensorboard_url} due to error: {exc}" + logdir_spec = _build_tensorboard_logdir_spec() + if not logdir_spec: + raise ValueError( + "No TensorBoard log directory is registered yet. Start a training or inference job first." ) + snapshot = _get_tensorboard_snapshot() + if snapshot["isRunning"] and snapshot["logdirSpec"] == logdir_spec: + return snapshot + + if snapshot["isRunning"] or snapshot["phase"] in {"starting", "failed", "finished"}: + _terminate_managed_tensorboard() + + port = _pick_tensorboard_port(snapshot.get("port")) + command = [ + sys.executable, + "-m", + "tensorboard.main", + "--logdir_spec", + logdir_spec, + "--host", + _TENSORBOARD_BIND_HOST, + "--port", + str(port), + ] + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + cwd=str(_project_root()), + ) + _tensorboard_process = process + _update_tensorboard_state( + phase="starting", + pid=process.pid, + port=port, + url=_tensorboard_url_for_port(port), + command=" ".join(command), + logdirSpec=logdir_spec, + startedAt=_utc_now(), + endedAt=None, + lastError=None, + ) + _append_tensorboard_log( + f"[TENSORBOARD] Launching with logdir spec: {logdir_spec}" + ) + append_app_event( + component="server_pytc", + event="tensorboard_launch", + level="INFO", + message="Launching TensorBoard", + source="tensorboard", + command=command, + tensorboard_port=port, + tensorboard_url=_tensorboard_url_for_port(port), + tensorboard_logdir_spec=logdir_spec, + ) + threading.Thread( + target=_watch_tensorboard_process, + args=(process,), + daemon=True, + ).start() + + if not _wait_for_tensorboard_ready(port, process): + _terminate_managed_tensorboard() + error_message = "TensorBoard did not become ready before the startup timeout." + _update_tensorboard_state( + phase="failed", + pid=None, + port=None, + endedAt=_utc_now(), + ) + _set_tensorboard_error(error_message) + raise RuntimeError(error_message) + + _update_tensorboard_state(phase="running") + if resolved: + _append_tensorboard_log( + f"[TENSORBOARD] Ready and watching {source_key} at {resolved}" + ) + append_app_event( + component="server_pytc", + event="tensorboard_ready", + level="INFO", + message="TensorBoard is ready", + source="tensorboard", + tensorboard_port=port, + tensorboard_url=_tensorboard_url_for_port(port), + tensorboard_logdir_spec=logdir_spec, + ) + return _get_tensorboard_snapshot() + def get_tensorboard(): - return tensorboard_url + snapshot = _get_tensorboard_snapshot() + return snapshot["url"] if snapshot["isRunning"] else None + +def get_tensorboard_status(): + return _get_tensorboard_snapshot() -def stop_tensorboard(): - global tensorboard_url - tensorboard_url = None - stop_process_by_name("tensorboard") + +def stop_tensorboard(clear_sources: bool = False): + _terminate_managed_tensorboard() + if clear_sources: + _tensorboard_sources.clear() + _update_tensorboard_state( + phase="stopped", + pid=None, + port=None, + url=None, + endedAt=_utc_now(), + lastError=None, + ) def start_inference(payload: dict): @@ -553,6 +1676,7 @@ def start_inference(payload: dict): config_text = payload.get("inferenceConfig", "") temp_filepath = None config_origin_path = payload.get("configOriginPath") + config_corrections: list[dict[str, Any]] = [] _reset_runtime_state( "inference", phase="starting", @@ -561,12 +1685,18 @@ def start_inference(payload: dict): "outputPath": payload.get("outputPath"), "checkpointPath": payload.get("checkpointPath") or (payload.get("arguments") or {}).get("checkpoint"), + "configOriginPath": config_origin_path, + "workflowId": payload.get("workflow_id") or payload.get("workflowId"), }, ) try: current_dir = _project_root() script_path = _pytc_script_path() + config_text, config_corrections = _sanitize_runtime_config_text( + config_text, + config_origin_path, + ) temp_filepath = _write_temp_config( config_text, "inference", @@ -581,6 +1711,14 @@ def start_inference(payload: dict): "inference", f"Config origin: {config_origin_path or 'none'}" ) _append_runtime_event("inference", f"Staged config path: {temp_filepath}") + if config_corrections: + _append_runtime_event( + "inference", + f"Sanitized staged inference config with {len(config_corrections)} correction(s)", + event="runtime_config_sanitized", + level="WARNING", + corrections=config_corrections, + ) arguments = dict(payload.get("arguments") or {}) if payload.get("checkpointPath") and not arguments.get("checkpoint"): @@ -602,12 +1740,43 @@ def start_inference(payload: dict): print(f"[MODEL.PY] Final inference command: {' '.join(command)}") _append_runtime_event("inference", f"Final inference command: {' '.join(command)}") + _emit_runtime_app_event( + "inference", + "runtime_config_snapshot", + "Inference config staged", + level="INFO", + source="model", + config_origin_path=config_origin_path, + staged_config_path=temp_filepath, + command=command, + arguments=arguments, + output_path=payload.get("outputPath"), + checkpoint_path=payload.get("checkpointPath"), + config_text=config_text, + config_text_length=len(config_text or ""), + config_line_count=(config_text or "").count("\n") + (1 if config_text else 0), + config_sanitized=bool(config_corrections), + config_corrections=config_corrections, + ) + config_diagnostic = _detect_chunk_tile_mismatch(config_text) + if config_diagnostic: + _append_runtime_event( + "inference", + config_diagnostic["message"], + event="runtime_config_warning", + level="WARNING", + diagnostic=config_diagnostic, + ) _inference_process = _start_logged_process( command, current_dir, "INFERENCE", "inference", ) + log_dir = _launch_tensorboard(payload.get("outputPath"), config_text, "test") + if log_dir: + _append_runtime_event("inference", f"TensorBoard log dir: {log_dir}") + print(f"[MODEL.PY] TensorBoard monitoring directory: {log_dir}") result = {"status": "started", "pid": _inference_process.pid} print(f"[MODEL.PY] Returning: {result}") print("========== MODEL.PY: END OF START_INFERENCE ==========\n") @@ -680,4 +1849,5 @@ def stop_inference(): return {"status": "stopped"} +atexit.register(stop_tensorboard) atexit.register(cleanup_temp_files) diff --git a/tests/test_app_event_logger.py b/tests/test_app_event_logger.py new file mode 100644 index 00000000..9127e0d1 --- /dev/null +++ b/tests/test_app_event_logger.py @@ -0,0 +1,48 @@ +import json +import io + +from app_event_logger import _AppEventStream, append_app_event, get_app_event_log_path + + +def test_append_app_event_writes_jsonl(tmp_path, monkeypatch): + log_path = tmp_path / "app-events.jsonl" + monkeypatch.setenv("PYTC_APP_EVENT_LOG_PATH", str(log_path)) + + record = append_app_event( + component="test", + event="unit_test", + level="warning", + message="hello world", + data={"answer": 42}, + ) + + assert get_app_event_log_path() == log_path + assert log_path.is_file() + assert record["component"] == "test" + assert record["event"] == "unit_test" + assert record["level"] == "WARNING" + + written = json.loads(log_path.read_text(encoding="utf-8").strip()) + assert written["component"] == "test" + assert written["event"] == "unit_test" + assert written["message"] == "hello world" + assert written["data"] == {"answer": 42} + + +def test_app_event_stream_detects_info_lines_on_stderr(tmp_path, monkeypatch): + log_path = tmp_path / "app-events.jsonl" + monkeypatch.setenv("PYTC_APP_EVENT_LOG_PATH", str(log_path)) + + stream = _AppEventStream( + component="test", + stream_name="stderr", + level="ERROR", + original_stream=io.StringIO(), + ) + stream.write("INFO: Shutting down\n") + stream.flush() + + written = json.loads(log_path.read_text(encoding="utf-8").strip()) + assert written["event"] == "stderr_line" + assert written["level"] == "INFO" + assert written["message"] == "INFO: Shutting down" diff --git a/tests/test_closed_loop_smoke_script.py b/tests/test_closed_loop_smoke_script.py new file mode 100644 index 00000000..7346f7cc --- /dev/null +++ b/tests/test_closed_loop_smoke_script.py @@ -0,0 +1,166 @@ +import pathlib + +import pytest + +pytest.importorskip("fastapi") +pytest.importorskip("sqlalchemy") +pytest.importorskip("tifffile") +h5py = pytest.importorskip("h5py") + +import numpy as np + +from scripts.run_closed_loop_smoke import run_closed_loop_smoke + + +def test_closed_loop_smoke_script_writes_researcher_evidence_bundle(tmp_path): + report = run_closed_loop_smoke(pathlib.Path(tmp_path)) + + assert report["ready_for_case_study"] is True + assert report["metric_summary"]["candidate_improved_dice"] is True + assert report["bundle_counts"]["model_runs"] >= 3 + assert report["bundle_counts"]["model_versions"] == 1 + assert report["bundle_counts"]["correction_sets"] >= 1 + assert report["bundle_counts"]["evaluation_results"] == 1 + assert report["bundle_counts"]["agent_plans"] == 1 + assert "actual PyTC training subprocess" in report["simulated_or_not_exercised"] + assert "browser-level proofreading editor interaction" in report[ + "simulated_or_not_exercised" + ] + + assert (tmp_path / "smoke-report.json").exists() + assert (tmp_path / "workflow-bundle.json").exists() + assert (tmp_path / "readiness.json").exists() + assert (tmp_path / "evaluation-report.json").exists() + + +def test_closed_loop_smoke_script_accepts_real_hdf5_pair(tmp_path): + image_path = tmp_path / "image.h5" + mask_path = tmp_path / "mask.h5" + image = np.arange(8 * 16 * 16, dtype=np.uint8).reshape(8, 16, 16) + mask = np.zeros((8, 16, 16), dtype=np.uint16) + mask[:, 4:12, 5:13] = 7 + with h5py.File(image_path, "w") as handle: + handle.create_dataset("main", data=image) + with h5py.File(mask_path, "w") as handle: + handle.create_dataset("data", data=mask) + + report = run_closed_loop_smoke( + pathlib.Path(tmp_path / "smoke"), + image_path=str(image_path), + mask_path=str(mask_path), + image_dataset="main", + mask_dataset="data", + crop="0:4,0:12,0:12", + ) + + assert report["artifact_mode"] == "real_pair_derived_predictions" + assert report["ready_for_case_study"] is True + assert report["source_data"]["image_dataset"] == "main" + assert report["source_data"]["mask_dataset"] == "data" + assert "real biomedical/connectomics sample data" not in report[ + "simulated_or_not_exercised" + ] + assert any( + item.startswith("baseline/candidate predictions derived") + for item in report["simulated_or_not_exercised"] + ) + + +def test_closed_loop_smoke_script_accepts_real_hdf5_predictions(tmp_path): + image_path = tmp_path / "image.h5" + mask_path = tmp_path / "mask.h5" + baseline_path = tmp_path / "baseline-prediction.h5" + candidate_path = tmp_path / "candidate-prediction.h5" + image = np.arange(8 * 16 * 16, dtype=np.uint8).reshape(8, 16, 16) + mask = np.zeros((8, 16, 16), dtype=np.uint16) + mask[:, 4:12, 5:13] = 7 + baseline = mask.copy() + baseline[:, :, :7] = 0 + candidate = mask.copy() + for path, dataset, array in [ + (image_path, "main", image), + (mask_path, "data", mask), + (baseline_path, "prediction", baseline), + (candidate_path, "prediction", candidate), + ]: + with h5py.File(path, "w") as handle: + handle.create_dataset(dataset, data=array) + + report = run_closed_loop_smoke( + pathlib.Path(tmp_path / "smoke"), + image_path=str(image_path), + mask_path=str(mask_path), + image_dataset="main", + mask_dataset="data", + baseline_prediction_path=str(baseline_path), + candidate_prediction_path=str(candidate_path), + baseline_dataset="prediction", + candidate_dataset="prediction", + crop="0:4,0:12,0:12", + ) + + assert report["artifact_mode"] == "real_pair_real_predictions" + assert report["ready_for_case_study"] is True + assert report["metric_summary"]["candidate_improved_dice"] is True + assert report["metric_summary"]["dice_delta"] > 0 + assert report["source_data"]["baseline_dataset"] == "prediction" + assert report["source_data"]["candidate_dataset"] == "prediction" + assert "real prediction artifacts supplied by caller" in report[ + "real_checks_exercised" + ] + assert not any( + item.startswith("baseline/candidate predictions derived") + for item in report["simulated_or_not_exercised"] + ) + + +def test_closed_loop_smoke_accepts_channel_first_pytc_predictions(tmp_path): + image_path = tmp_path / "image.h5" + mask_path = tmp_path / "mask.h5" + baseline_path = tmp_path / "baseline-result_xy.h5" + candidate_path = tmp_path / "candidate-result_xy.h5" + image = np.arange(8 * 16 * 16, dtype=np.uint8).reshape(8, 16, 16) + mask = np.zeros((8, 16, 16), dtype=np.uint16) + mask[:, 4:12, 5:13] = 1 + baseline = np.stack([np.zeros_like(mask), mask.copy()]) + baseline[1, :, :, :7] = 0 + candidate = np.stack([np.zeros_like(mask), mask.copy()]) + for path, dataset, array in [ + (image_path, "main", image), + (mask_path, "data", mask), + (baseline_path, "vol0", baseline), + (candidate_path, "vol0", candidate), + ]: + with h5py.File(path, "w") as handle: + handle.create_dataset(dataset, data=array) + + report = run_closed_loop_smoke( + pathlib.Path(tmp_path / "smoke"), + image_path=str(image_path), + mask_path=str(mask_path), + image_dataset="main", + mask_dataset="data", + baseline_prediction_path=str(baseline_path), + candidate_prediction_path=str(candidate_path), + baseline_dataset="vol0", + candidate_dataset="vol0", + baseline_channel=1, + candidate_channel=1, + crop="0:4,0:12,0:12", + ) + + assert report["artifact_mode"] == "real_pair_real_predictions" + assert report["ready_for_case_study"] is True + assert report["metric_summary"]["candidate_improved_dice"] is True + assert report["source_data"]["baseline_channel"] == 1 + assert report["source_data"]["candidate_channel"] == 1 + + +def test_closed_loop_smoke_rejects_partial_real_prediction_inputs(tmp_path): + with pytest.raises(ValueError, match="requires both baseline_prediction_path"): + run_closed_loop_smoke( + pathlib.Path(tmp_path / "smoke"), + image_path=str(tmp_path / "missing-image.h5"), + mask_path=str(tmp_path / "missing-mask.h5"), + baseline_prediction_path=str(tmp_path / "baseline.h5"), + ) diff --git a/tests/test_pytc_runtime_routes.py b/tests/test_pytc_runtime_routes.py index 17692de5..7f23c5b3 100644 --- a/tests/test_pytc_runtime_routes.py +++ b/tests/test_pytc_runtime_routes.py @@ -1,11 +1,17 @@ import json +import os import pathlib +import tempfile import unittest from unittest.mock import patch import requests from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from server_api.auth import database as auth_database +from server_api.auth import models from server_api.main import app as server_api_app from server_pytc.main import app as server_pytc_app from server_pytc.services import model as model_service @@ -27,6 +33,9 @@ class ServerPytcRouteTests(unittest.TestCase): def setUp(self): self.client = TestClient(server_pytc_app) + def tearDown(self): + model_service.stop_tensorboard(clear_sources=True) + def test_inference_status_route_returns_worker_payload(self): payload = {"isRunning": True, "pid": 1234, "exitCode": None} with patch("server_pytc.main.get_inference_status", return_value=payload): @@ -35,14 +44,32 @@ def test_inference_status_route_returns_worker_payload(self): self.assertEqual(response.status_code, 200) self.assertEqual(response.json(), payload) - def test_start_tensorboard_without_log_path_returns_400(self): - response = self.client.get("/start_tensorboard") + def test_start_tensorboard_without_registered_sources_returns_400(self): + with patch( + "server_pytc.main.initialize_tensorboard", + side_effect=ValueError("No TensorBoard log directory is registered yet."), + ): + response = self.client.get("/start_tensorboard") self.assertEqual(response.status_code, 400) self.assertEqual( - response.json()["detail"], "Missing required query parameter: logPath" + response.json()["detail"], "No TensorBoard log directory is registered yet." ) + def test_tensorboard_status_route_returns_worker_payload(self): + payload = { + "isRunning": True, + "phase": "running", + "pid": 4321, + "url": "http://127.0.0.1:6006/", + "sources": {"training": {"name": "training", "path": "/tmp/out"}}, + } + with patch("server_pytc.main.get_tensorboard_status", return_value=payload): + response = self.client.get("/get_tensorboard_status") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), payload) + def test_training_logs_route_returns_worker_payload(self): payload = {"phase": "running", "text": "hello", "lines": ["hello"]} with patch( @@ -84,7 +111,9 @@ def test_inference_status_proxy_returns_worker_payload(self): request_mock.assert_called_once() def test_start_tensorboard_proxy_propagates_worker_client_error(self): - worker_payload = {"detail": "Missing required query parameter: logPath"} + worker_payload = { + "detail": "No TensorBoard log directory is registered yet." + } with patch( "server_api.main.requests.request", return_value=FakeResponse(400, payload=worker_payload), @@ -96,6 +125,29 @@ def test_start_tensorboard_proxy_propagates_worker_client_error(self): self.assertEqual(detail["upstream_status"], 400) self.assertEqual(detail["upstream_body"], worker_payload) + def test_tensorboard_status_proxy_returns_worker_payload(self): + payload = { + "isRunning": False, + "phase": "stopped", + "pid": None, + "url": None, + "sources": { + "training": { + "name": "training", + "path": "/tmp/train", + "exists": True, + } + }, + } + with patch( + "server_api.main.requests.request", + return_value=FakeResponse(200, payload=payload), + ): + response = self.client.get("/get_tensorboard_status") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), payload) + def test_start_model_training_proxy_returns_504_on_timeout(self): with patch( "server_api.main.requests.request", @@ -121,9 +173,104 @@ def test_training_logs_proxy_returns_worker_payload(self): self.assertEqual(response.json(), payload) +class WorkflowInferenceRuntimeSyncTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.db_path = pathlib.Path(self.temp_dir.name) / "workflow-runtime-sync.db" + self.engine = create_engine( + f"sqlite:///{self.db_path}", connect_args={"check_same_thread": False} + ) + self.SessionLocal = sessionmaker( + autocommit=False, autoflush=False, bind=self.engine + ) + models.Base.metadata.create_all(bind=self.engine) + + def override_get_db(): + db = self.SessionLocal() + try: + yield db + finally: + db.close() + + server_api_app.dependency_overrides[auth_database.get_db] = override_get_db + self.client = TestClient(server_api_app) + + def tearDown(self): + server_api_app.dependency_overrides.clear() + self.engine.dispose() + self.temp_dir.cleanup() + + def _workflow_id(self): + response = self.client.get("/api/workflows/current") + self.assertEqual(response.status_code, 200) + return response.json()["workflow"]["id"] + + def test_sync_completed_inference_runtime_materializes_prediction_run(self): + workflow_id = self._workflow_id() + output_dir = pathlib.Path(self.temp_dir.name) / "inference-output" + output_dir.mkdir() + prediction = output_dir / "result_xy.h5" + checkpoint = output_dir / "checkpoint_00001.pth.tar" + prediction.write_text("prediction", encoding="utf-8") + checkpoint.write_text("checkpoint", encoding="utf-8") + + runtime_payload = { + "phase": "finished", + "pid": None, + "exitCode": 0, + "configPath": "/tmp/staged-inference.yaml", + "configOriginPath": "configs/MitoEM/Mito25-Local-Smoke-BC.yaml", + "startedAt": "2026-04-25T10:00:00+00:00", + "endedAt": "2026-04-25T10:01:00+00:00", + "lineCount": 12, + "metadata": { + "outputPath": str(output_dir), + "checkpointPath": str(checkpoint), + }, + } + + with patch("server_api.main._proxy_to_worker", return_value=runtime_payload): + response = self.client.post( + f"/api/workflows/{workflow_id}/sync-inference-runtime", + json={}, + ) + duplicate_response = self.client.post( + f"/api/workflows/{workflow_id}/sync-inference-runtime", + json={}, + ) + + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertTrue(payload["synced"]) + self.assertEqual(payload["outputPath"], str(prediction.resolve())) + self.assertFalse(payload["deduplicated"]) + self.assertEqual(duplicate_response.status_code, 200) + self.assertTrue(duplicate_response.json()["deduplicated"]) + + runs_response = self.client.get( + f"/api/workflows/{workflow_id}/model-runs?run_type=inference" + ) + self.assertEqual(runs_response.status_code, 200) + runs = runs_response.json() + self.assertEqual(len(runs), 1) + self.assertEqual(runs[0]["status"], "completed") + self.assertEqual(runs[0]["output_path"], str(prediction.resolve())) + + artifacts_response = self.client.get( + f"/api/workflows/{workflow_id}/artifacts?artifact_type=inference_output" + ) + self.assertEqual(artifacts_response.status_code, 200) + artifacts = artifacts_response.json() + self.assertEqual(len(artifacts), 1) + self.assertEqual(artifacts[0]["path"], str(prediction.resolve())) + + class ModelServiceTests(unittest.TestCase): def tearDown(self): + model_service.stop_tensorboard(clear_sources=True) model_service.cleanup_temp_files() + model_service._reset_runtime_state("training") + model_service._reset_runtime_state("inference") def test_write_temp_config_uses_origin_parent_for_relative_bases(self): config_path = model_service._write_temp_config( @@ -139,6 +286,226 @@ def test_write_temp_config_uses_origin_parent_for_relative_bases(self): self.assertTrue(written_path.exists()) self.assertEqual(written_path.parent, expected_parent) + def test_register_tensorboard_source_builds_named_logdir_spec(self): + with tempfile.TemporaryDirectory() as tmpdir: + training_dir = pathlib.Path(tmpdir) / "train" + inference_dir = pathlib.Path(tmpdir) / "infer" + + first = model_service._register_tensorboard_source( + "training", + str(training_dir), + ) + second = model_service._register_tensorboard_source( + "inference", + str(inference_dir), + ) + + self.assertTrue(pathlib.Path(first).exists()) + self.assertTrue(pathlib.Path(second).exists()) + self.assertEqual( + model_service._build_tensorboard_logdir_spec(), + f"inference:{pathlib.Path(second).resolve()},training:{pathlib.Path(first).resolve()}", + ) + + def test_runtime_log_lines_are_exported_to_app_event_log(self): + with tempfile.TemporaryDirectory() as tmpdir: + log_path = pathlib.Path(tmpdir) / "app-events.jsonl" + previous = os.environ.get("PYTC_APP_EVENT_LOG_PATH") + os.environ["PYTC_APP_EVENT_LOG_PATH"] = str(log_path) + try: + model_service._reset_runtime_state("training", phase="starting") + model_service._update_runtime_state("training", pid=12345) + model_service._append_runtime_event("training", "Config origin: foo.yaml") + model_service._append_runtime_log( + "training", + "UnicodeDecodeError: invalid start byte", + event="runtime_output_line", + source="subprocess", + stream="stdout", + ) + finally: + if previous is None: + os.environ.pop("PYTC_APP_EVENT_LOG_PATH", None) + else: + os.environ["PYTC_APP_EVENT_LOG_PATH"] = previous + + records = [ + json.loads(line) + for line in log_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + self.assertEqual(records[0]["event"], "runtime_event") + self.assertEqual(records[0]["runtime_kind"], "training") + self.assertEqual(records[0]["runtime_phase"], "starting") + self.assertEqual(records[0]["runtime_pid"], 12345) + self.assertEqual(records[1]["event"], "runtime_output_line") + self.assertEqual(records[1]["level"], "ERROR") + self.assertEqual(records[1]["source"], "subprocess") + self.assertEqual(records[1]["stream"], "stdout") + + def test_detect_chunk_tile_mismatch_for_direct_h5_volume(self): + diagnostic = model_service._detect_chunk_tile_mismatch( + """ +DATASET: + DO_CHUNK_TITLE: 1 + IMAGE_NAME: /tmp/train-volume.h5 + LABEL_NAME: /tmp/train-label.h5 +""" + ) + + self.assertIsNotNone(diagnostic) + self.assertEqual( + diagnostic["code"], "tile_dataset_direct_volume_mismatch" + ) + self.assertIn("TileDataset", diagnostic["message"]) + self.assertEqual(diagnostic["image_name"], "/tmp/train-volume.h5") + + def test_sanitize_runtime_config_restores_fractional_values_and_paths(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = pathlib.Path(tmpdir) + image_path = tmp_root / "pair_im.h5" + label_path = tmp_root / "pair_seg.h5" + infer_path = tmp_root / "pair2_im.h5" + for path in (image_path, label_path, infer_path): + path.write_text("placeholder", encoding="utf-8") + + origin_config = tmp_root / "origin.yaml" + origin_config.write_text( + "\n".join( + [ + "DATASET:", + f" IMAGE_NAME: {image_path}", + f" LABEL_NAME: {label_path}", + " VALID_RATIO: 0.5", + " REJECT_SAMPLING:", + " P: 0.95", + "INFERENCE:", + f" IMAGE_NAME: {infer_path}", + " AUG_NUM: 4", + "", + ] + ), + encoding="utf-8", + ) + + corrupted_config = "\n".join( + [ + "DATASET:", + f" IMAGE_NAME: {image_path.with_name('pair.h5')}", + f" LABEL_NAME: {label_path.with_name('pair.h5')}", + " VALID_RATIO: 0", + " REJECT_SAMPLING:", + " P: 0", + "INFERENCE:", + f" IMAGE_NAME: {infer_path.with_name('pair2.h5')}", + " AUG_NUM: 1", + "", + ] + ) + + sanitized_text, changes = model_service._sanitize_runtime_config_text( + corrupted_config, + str(origin_config), + ) + sanitized = model_service._load_yaml_config(sanitized_text) + + self.assertEqual(sanitized["DATASET"]["IMAGE_NAME"], str(image_path)) + self.assertEqual(sanitized["DATASET"]["LABEL_NAME"], str(label_path)) + self.assertEqual(sanitized["INFERENCE"]["IMAGE_NAME"], str(infer_path)) + self.assertEqual(sanitized["DATASET"]["VALID_RATIO"], 0.5) + self.assertEqual(sanitized["DATASET"]["REJECT_SAMPLING"]["P"], 0.95) + self.assertEqual(sanitized["INFERENCE"]["AUG_NUM"], 4) + self.assertGreaterEqual(len(changes), 5) + + def test_runtime_error_lines_update_last_error(self): + model_service._reset_runtime_state("training", phase="running") + model_service._append_runtime_log( + "training", + "ValueError: boom", + event="runtime_output_line", + source="subprocess", + stream="stdout", + ) + + snapshot = model_service.get_training_status() + self.assertEqual(snapshot["lastError"], "ValueError: boom") + + def test_find_latest_checkpoint_returns_newest_training_artifact(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = pathlib.Path(tmpdir) + older = output_dir / "checkpoint_00050.pth.tar" + newer = output_dir / "checkpoint_00200.pth.tar" + older.write_text("older", encoding="utf-8") + newer.write_text("newer", encoding="utf-8") + + os.utime(older, (1, 1)) + os.utime(newer, (2, 2)) + + latest = model_service._find_latest_checkpoint(str(output_dir)) + + self.assertEqual(latest, str(newer.resolve())) + + def test_training_snapshot_backfills_checkpoint_metadata_after_exit(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = pathlib.Path(tmpdir) + checkpoint_path = output_dir / "checkpoint_00010.pth.tar" + checkpoint_path.write_text("checkpoint", encoding="utf-8") + + model_service._reset_runtime_state( + "training", + phase="finished", + metadata={"outputPath": str(output_dir)}, + ) + + snapshot = model_service.get_training_logs() + + self.assertEqual( + snapshot["metadata"]["checkpointPath"], + str(checkpoint_path.resolve()), + ) + self.assertEqual( + snapshot["metadata"]["latestCheckpointName"], + checkpoint_path.name, + ) + + def test_find_latest_prediction_output_returns_newest_inference_artifact(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = pathlib.Path(tmpdir) + older = output_dir / "prediction.h5" + newer = output_dir / "result_xy.h5" + older.write_text("older", encoding="utf-8") + newer.write_text("newer", encoding="utf-8") + + os.utime(older, (1, 1)) + os.utime(newer, (2, 2)) + + latest = model_service._find_latest_prediction_output(str(output_dir)) + + self.assertEqual(latest, str(newer.resolve())) + + def test_inference_snapshot_backfills_prediction_metadata_after_exit(self): + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = pathlib.Path(tmpdir) + prediction_path = output_dir / "result_xy.h5" + prediction_path.write_text("prediction", encoding="utf-8") + + model_service._reset_runtime_state( + "inference", + phase="finished", + metadata={"outputPath": str(output_dir)}, + ) + + snapshot = model_service.get_inference_logs() + + self.assertEqual( + snapshot["metadata"]["predictionPath"], + str(prediction_path.resolve()), + ) + self.assertEqual( + snapshot["metadata"]["latestPredictionName"], + prediction_path.name, + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_workflow_artifact_records.py b/tests/test_workflow_artifact_records.py new file mode 100644 index 00000000..e45a55c7 --- /dev/null +++ b/tests/test_workflow_artifact_records.py @@ -0,0 +1,347 @@ +import pathlib +import tempfile +import unittest + +import pytest +import numpy as np + +pytest.importorskip("sqlalchemy") +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +pytest.importorskip("fastapi") +tifffile = pytest.importorskip("tifffile") +h5py = pytest.importorskip("h5py") +from fastapi.testclient import TestClient + +from server_api.auth import database as auth_database +from server_api.auth import models +from server_api.main import app as server_api_app + + +class WorkflowArtifactRecordTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temp_dir.name) + self.db_path = self.root / "workflow-artifacts-test.db" + self.engine = create_engine( + f"sqlite:///{self.db_path}", connect_args={"check_same_thread": False} + ) + self.SessionLocal = sessionmaker( + autocommit=False, autoflush=False, bind=self.engine + ) + models.Base.metadata.create_all(bind=self.engine) + + def override_get_db(): + db = self.SessionLocal() + try: + yield db + finally: + db.close() + + server_api_app.dependency_overrides[auth_database.get_db] = override_get_db + self.client = TestClient(server_api_app) + + def tearDown(self): + server_api_app.dependency_overrides.clear() + self.engine.dispose() + self.temp_dir.cleanup() + + def _workflow_id(self): + response = self.client.get("/api/workflows/current") + self.assertEqual(response.status_code, 200) + return response.json()["workflow"]["id"] + + def _write_file(self, name: str) -> str: + path = self.root / name + path.write_text(name, encoding="utf-8") + return str(path) + + def test_events_materialize_artifacts_runs_versions_and_corrections(self): + workflow_id = self._workflow_id() + image_path = self._write_file("image.tif") + mask_path = self._write_file("mask.tif") + ground_truth_path = self._write_file("ground-truth.tif") + corrected_path = self._write_file("corrected.tif") + training_output = self._write_file("training-output") + checkpoint_path = self._write_file("checkpoint.pth") + + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "dataset.loaded", + "stage": "proofreading", + "summary": "Loaded dataset.", + "payload": { + "dataset_path": image_path, + "image_path": image_path, + "mask_path": mask_path, + "source_ground_truth_path": ground_truth_path, + }, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "proofreading.mask_saved", + "stage": "proofreading", + "summary": "Saved first correction.", + "payload": {"instance_id": 12, "z_index": 1}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "proofreading.mask_saved", + "stage": "proofreading", + "summary": "Saved second correction.", + "payload": {"instance_id": 13, "z_index": 2}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "proofreading.masks_exported", + "stage": "proofreading", + "summary": "Exported corrections.", + "payload": {"written_path": corrected_path}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "training.started", + "stage": "retraining_staged", + "summary": "Started training.", + "payload": {"outputPath": training_output}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "system", + "event_type": "training.completed", + "stage": "evaluation", + "summary": "Training completed.", + "payload": { + "outputPath": training_output, + "checkpointPath": checkpoint_path, + "checkpointName": "checkpoint.pth", + }, + }, + ) + + artifacts = self.client.get(f"/api/workflows/{workflow_id}/artifacts").json() + artifact_types = {artifact["artifact_type"] for artifact in artifacts} + self.assertIn("image_volume", artifact_types) + self.assertIn("mask_volume", artifact_types) + self.assertIn("correction_set", artifact_types) + self.assertIn("model_checkpoint", artifact_types) + ground_truth_artifacts = [ + artifact + for artifact in artifacts + if artifact["role"] == "ground_truth" + ] + self.assertEqual(ground_truth_artifacts[0]["path"], ground_truth_path) + + correction_sets = self.client.get( + f"/api/workflows/{workflow_id}/correction-sets" + ).json() + self.assertEqual(len(correction_sets), 1) + self.assertEqual(correction_sets[0]["corrected_mask_path"], corrected_path) + self.assertEqual(correction_sets[0]["edit_count"], 2) + self.assertEqual(correction_sets[0]["region_count"], 2) + + model_runs = self.client.get(f"/api/workflows/{workflow_id}/model-runs").json() + self.assertEqual(len(model_runs), 1) + self.assertEqual(model_runs[0]["run_type"], "training") + self.assertEqual(model_runs[0]["status"], "completed") + self.assertEqual(model_runs[0]["checkpoint_path"], checkpoint_path) + + model_versions = self.client.get( + f"/api/workflows/{workflow_id}/model-versions" + ).json() + self.assertEqual(len(model_versions), 1) + self.assertEqual(model_versions[0]["version_label"], "checkpoint.pth") + + bundle = self.client.post(f"/api/workflows/{workflow_id}/export-bundle").json() + self.assertGreaterEqual(len(bundle["artifacts"]), 4) + self.assertEqual(len(bundle["model_runs"]), 1) + self.assertEqual(len(bundle["model_versions"]), 1) + self.assertEqual(len(bundle["correction_sets"]), 1) + + events = self.client.get(f"/api/workflows/{workflow_id}/events").json() + self.assertIn( + "workflow.bundle_exported", + {event["event_type"] for event in events}, + ) + + def test_manual_evaluation_records_and_readiness_endpoint(self): + workflow_id = self._workflow_id() + report_path = self._write_file("eval-report.json") + + artifact_response = self.client.post( + f"/api/workflows/{workflow_id}/artifacts", + json={ + "artifact_type": "evaluation_report", + "role": "case_study_evidence", + "path": report_path, + }, + ) + self.assertEqual(artifact_response.status_code, 200) + self.assertTrue(artifact_response.json()["exists"]) + + run_response = self.client.post( + f"/api/workflows/{workflow_id}/model-runs", + json={ + "run_type": "inference", + "status": "completed", + "output_path": self._write_file("prediction.tif"), + }, + ) + self.assertEqual(run_response.status_code, 200) + + evaluation_response = self.client.post( + f"/api/workflows/{workflow_id}/evaluation-results", + json={ + "name": "before-after-smoke", + "candidate_run_id": run_response.json()["id"], + "report_path": report_path, + "summary": "Smoke comparison completed.", + "metrics": {"dice": 0.75}, + }, + ) + self.assertEqual(evaluation_response.status_code, 200) + self.assertEqual(evaluation_response.json()["metrics"]["dice"], 0.75) + + readiness_response = self.client.get( + f"/api/workflows/{workflow_id}/case-study-readiness" + ) + self.assertEqual(readiness_response.status_code, 200) + readiness = readiness_response.json() + self.assertEqual(readiness["workflow_id"], workflow_id) + self.assertEqual(readiness["total_count"], len(readiness["gates"])) + self.assertIn("next_required_items", readiness) + + def test_compute_evaluation_result_records_before_after_metrics(self): + workflow_id = self._workflow_id() + ground_truth_path = self.root / "ground-truth.tif" + baseline_path = self.root / "baseline-prediction.tif" + candidate_path = self.root / "candidate-prediction.tif" + report_path = self.root / "computed-eval.json" + + ground_truth = np.zeros((1, 4, 4), dtype=np.uint8) + ground_truth[:, 1:3, 1:3] = 1 + baseline = np.zeros_like(ground_truth) + baseline[:, 1:2, 1:2] = 1 + candidate = ground_truth.copy() + tifffile.imwrite(str(ground_truth_path), ground_truth) + tifffile.imwrite(str(baseline_path), baseline) + tifffile.imwrite(str(candidate_path), candidate) + + response = self.client.post( + f"/api/workflows/{workflow_id}/evaluation-results/compute", + json={ + "name": "computed-before-after", + "ground_truth_path": str(ground_truth_path), + "baseline_prediction_path": str(baseline_path), + "candidate_prediction_path": str(candidate_path), + "report_path": str(report_path), + }, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertEqual(payload["name"], "computed-before-after") + self.assertTrue(payload["metrics"]["summary"]["candidate_improved_dice"]) + self.assertGreater(payload["metrics"]["delta"]["dice"], 0) + self.assertTrue(report_path.exists()) + + def test_compute_evaluation_result_supports_hdf5_dataset_keys_and_crop(self): + workflow_id = self._workflow_id() + ground_truth_path = self.root / "ground-truth.h5" + baseline_path = self.root / "baseline-prediction.h5" + candidate_path = self.root / "candidate-prediction.h5" + + ground_truth = np.zeros((4, 8, 8), dtype=np.uint16) + ground_truth[:, 2:6, 2:6] = 1 + baseline = ground_truth.copy() + baseline[:, :, :3] = 0 + candidate = ground_truth.copy() + for path, dataset, array in [ + (ground_truth_path, "data", ground_truth), + (baseline_path, "prediction", baseline), + (candidate_path, "prediction", candidate), + ]: + with h5py.File(path, "w") as handle: + handle.create_dataset(dataset, data=array) + + response = self.client.post( + f"/api/workflows/{workflow_id}/evaluation-results/compute", + json={ + "name": "computed-hdf5-before-after", + "ground_truth_path": str(ground_truth_path), + "ground_truth_dataset": "data", + "baseline_prediction_path": str(baseline_path), + "baseline_dataset": "prediction", + "candidate_prediction_path": str(candidate_path), + "candidate_dataset": "prediction", + "crop": "0:2,0:8,0:8", + }, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertTrue(payload["metrics"]["summary"]["candidate_improved_dice"]) + self.assertEqual(payload["metadata"]["ground_truth_dataset"], "data") + self.assertEqual(payload["metadata"]["crop"], "0:2,0:8,0:8") + + def test_compute_evaluation_result_supports_prediction_channel_selection(self): + workflow_id = self._workflow_id() + ground_truth_path = self.root / "ground-truth.h5" + baseline_path = self.root / "baseline-result_xy.h5" + candidate_path = self.root / "candidate-result_xy.h5" + + ground_truth = np.zeros((4, 8, 8), dtype=np.uint16) + ground_truth[:, 2:6, 2:6] = 1 + baseline_mask = ground_truth.copy() + baseline_mask[:, :, :4] = 0 + candidate_mask = ground_truth.copy() + baseline = np.stack([np.zeros_like(ground_truth), baseline_mask]) + candidate = np.stack([np.zeros_like(ground_truth), candidate_mask]) + for path, dataset, array in [ + (ground_truth_path, "data", ground_truth), + (baseline_path, "vol0", baseline), + (candidate_path, "vol0", candidate), + ]: + with h5py.File(path, "w") as handle: + handle.create_dataset(dataset, data=array) + + response = self.client.post( + f"/api/workflows/{workflow_id}/evaluation-results/compute", + json={ + "name": "computed-channel-before-after", + "ground_truth_path": str(ground_truth_path), + "ground_truth_dataset": "data", + "baseline_prediction_path": str(baseline_path), + "baseline_dataset": "vol0", + "baseline_channel": 1, + "candidate_prediction_path": str(candidate_path), + "candidate_dataset": "vol0", + "candidate_channel": 1, + "crop": "0:2,0:8,0:8", + }, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertTrue(payload["metrics"]["summary"]["candidate_improved_dice"]) + self.assertGreater(payload["metrics"]["delta"]["dice"], 0) + self.assertEqual(payload["metadata"]["baseline_channel"], 1) + self.assertEqual(payload["metadata"]["candidate_channel"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflow_case_study_acceptance.py b/tests/test_workflow_case_study_acceptance.py new file mode 100644 index 00000000..2cb680d7 --- /dev/null +++ b/tests/test_workflow_case_study_acceptance.py @@ -0,0 +1,214 @@ +import pathlib +import tempfile +import unittest + +import numpy as np +import pytest + +pytest.importorskip("sqlalchemy") +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +pytest.importorskip("fastapi") +tifffile = pytest.importorskip("tifffile") +from fastapi.testclient import TestClient + +from server_api.auth import database as auth_database +from server_api.auth import models +from server_api.main import app as server_api_app + + +class WorkflowCaseStudyAcceptanceTests(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.root = pathlib.Path(self.temp_dir.name) + self.db_path = self.root / "workflow-case-study.db" + self.engine = create_engine( + f"sqlite:///{self.db_path}", connect_args={"check_same_thread": False} + ) + self.SessionLocal = sessionmaker( + autocommit=False, autoflush=False, bind=self.engine + ) + models.Base.metadata.create_all(bind=self.engine) + + def override_get_db(): + db = self.SessionLocal() + try: + yield db + finally: + db.close() + + server_api_app.dependency_overrides[auth_database.get_db] = override_get_db + self.client = TestClient(server_api_app) + + def tearDown(self): + server_api_app.dependency_overrides.clear() + self.engine.dispose() + self.temp_dir.cleanup() + + def _workflow_id(self): + response = self.client.get("/api/workflows/current") + self.assertEqual(response.status_code, 200) + return response.json()["workflow"]["id"] + + def _write_file(self, name: str, content: str = "artifact") -> str: + path = self.root / name + path.write_text(content, encoding="utf-8") + return str(path) + + def test_synthetic_closed_loop_reaches_case_study_readiness(self): + workflow_id = self._workflow_id() + image_path = self._write_file("image.tif") + mask_path = self._write_file("mask.tif") + corrected_path = self._write_file("corrected-mask.tif") + training_output = self._write_file("training-output") + checkpoint_path = self._write_file("checkpoint.pth") + report_path = self.root / "before-after-report.json" + + ground_truth_path = self.root / "ground-truth.tif" + baseline_path = self.root / "baseline-prediction.tif" + candidate_path = self.root / "candidate-prediction.tif" + ground_truth = np.zeros((1, 4, 4), dtype=np.uint8) + ground_truth[:, 1:3, 1:3] = 1 + baseline = np.zeros_like(ground_truth) + baseline[:, 1:2, 1:2] = 1 + candidate = ground_truth.copy() + tifffile.imwrite(str(ground_truth_path), ground_truth) + tifffile.imwrite(str(baseline_path), baseline) + tifffile.imwrite(str(candidate_path), candidate) + + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "dataset.loaded", + "stage": "visualization", + "summary": "Loaded sample image and mask.", + "payload": {"image_path": image_path, "mask_path": mask_path}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "system", + "event_type": "inference.completed", + "stage": "inference", + "summary": "Baseline inference completed.", + "payload": {"outputPath": str(baseline_path)}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "proofreading.instance_classified", + "stage": "proofreading", + "summary": "Marked a top hotspot as incorrect.", + "payload": {"region_id": "z:1", "classification": "incorrect"}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "user", + "event_type": "proofreading.mask_saved", + "stage": "proofreading", + "summary": "Saved corrected hotspot mask.", + "payload": {"region_id": "z:1", "instance_id": 42}, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "system", + "event_type": "proofreading.masks_exported", + "stage": "proofreading", + "summary": "Exported corrected masks.", + "payload": {"written_path": corrected_path, "region_id": "z:1"}, + }, + ) + self.client.get(f"/api/workflows/{workflow_id}/hotspots") + + proposal = self.client.post( + f"/api/workflows/{workflow_id}/agent-actions", + json={ + "action": "stage_retraining_from_corrections", + "summary": "Stage corrections for retraining.", + "payload": {"corrected_mask_path": corrected_path}, + }, + ).json() + approve_proposal = self.client.post( + f"/api/workflows/{workflow_id}/agent-actions/{proposal['id']}/approve" + ) + self.assertEqual(approve_proposal.status_code, 200) + + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "system", + "event_type": "training.completed", + "stage": "evaluation", + "summary": "Retraining completed.", + "payload": { + "outputPath": training_output, + "checkpointPath": checkpoint_path, + "checkpointName": "checkpoint.pth", + }, + }, + ) + self.client.post( + f"/api/workflows/{workflow_id}/events", + json={ + "actor": "system", + "event_type": "inference.completed", + "stage": "evaluation", + "summary": "Candidate inference completed.", + "payload": { + "outputPath": str(candidate_path), + "checkpointPath": checkpoint_path, + }, + }, + ) + + plan = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans", + json={"title": "Synthetic case-study loop"}, + ).json() + approve_plan = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans/{plan['id']}/approve" + ) + self.assertEqual(approve_plan.status_code, 200) + + evaluation = self.client.post( + f"/api/workflows/{workflow_id}/evaluation-results/compute", + json={ + "name": "case-study-before-after", + "ground_truth_path": str(ground_truth_path), + "baseline_prediction_path": str(baseline_path), + "candidate_prediction_path": str(candidate_path), + "report_path": str(report_path), + }, + ) + self.assertEqual(evaluation.status_code, 200) + self.assertTrue( + evaluation.json()["metrics"]["summary"]["candidate_improved_dice"] + ) + + readiness = self.client.get( + f"/api/workflows/{workflow_id}/case-study-readiness" + ) + self.assertEqual(readiness.status_code, 200) + readiness_payload = readiness.json() + self.assertTrue(readiness_payload["ready_for_case_study"]) + self.assertEqual(readiness_payload["next_required_items"], []) + + bundle = self.client.post(f"/api/workflows/{workflow_id}/export-bundle").json() + self.assertGreaterEqual(len(bundle["events"]), 10) + self.assertEqual(len(bundle["agent_plans"]), 1) + self.assertGreaterEqual(len(bundle["model_runs"]), 3) + self.assertEqual(len(bundle["model_versions"]), 1) + self.assertEqual(len(bundle["evaluation_results"]), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflow_routes.py b/tests/test_workflow_routes.py index da48d11f..5d69310c 100644 --- a/tests/test_workflow_routes.py +++ b/tests/test_workflow_routes.py @@ -4,6 +4,7 @@ import numpy as np import pytest + pytest.importorskip("sqlalchemy") from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -108,7 +109,9 @@ def test_agent_action_approve_and_reject_flow(self): f"{reject_proposal.json()['id']}/reject" ) self.assertEqual(reject_response.status_code, 200) - self.assertEqual(reject_response.json()["event_type"], "agent.proposal_rejected") + self.assertEqual( + reject_response.json()["event_type"], "agent.proposal_rejected" + ) approve_proposal = self.client.post( f"/api/workflows/{workflow_id}/agent-actions", @@ -141,6 +144,76 @@ def test_agent_action_approve_and_reject_flow(self): self.assertIn("agent.proposal_rejected", event_types) self.assertIn("retraining.staged", event_types) + def test_agent_plan_preview_control_and_bundle_export(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + + plan_response = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans", + json={ + "title": "Case-study acceptance loop", + "goal": "Drive a bounded closed-loop segmentation study.", + }, + ) + self.assertEqual(plan_response.status_code, 200) + plan = plan_response.json() + self.assertEqual(plan["title"], "Case-study acceptance loop") + self.assertEqual(plan["approval_status"], "pending") + self.assertGreaterEqual(len(plan["steps"]), 8) + self.assertEqual( + plan["graph"]["execution_model"]["mode"], + "bounded_human_approved_plan_preview", + ) + self.assertIn( + plan["graph"]["execution_model"]["langgraph"]["status"], + {"available_not_executing", "available_compile_failed", "unavailable"}, + ) + + approval_step = next( + step for step in plan["steps"] if step["requires_approval"] + ) + step_approval = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans/{plan['id']}/steps/" + f"{approval_step['id']}/approve" + ) + self.assertEqual(step_approval.status_code, 200) + self.assertEqual(step_approval.json()["status"], "approved") + + approve_response = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans/{plan['id']}/approve" + ) + self.assertEqual(approve_response.status_code, 200) + self.assertEqual(approve_response.json()["status"], "approved") + self.assertEqual(approve_response.json()["approval_status"], "approved") + + interrupt_response = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans/{plan['id']}/interrupt" + ) + self.assertEqual(interrupt_response.status_code, 200) + self.assertEqual(interrupt_response.json()["status"], "interrupted") + + resume_response = self.client.post( + f"/api/workflows/{workflow_id}/agent-plans/{plan['id']}/resume" + ) + self.assertEqual(resume_response.status_code, 200) + self.assertEqual(resume_response.json()["status"], "approved") + + readiness_response = self.client.get( + f"/api/workflows/{workflow_id}/case-study-readiness" + ) + self.assertEqual(readiness_response.status_code, 200) + readiness_gates = { + gate["id"]: gate for gate in readiness_response.json()["gates"] + } + self.assertTrue(readiness_gates["agent_plan_preview"]["complete"]) + self.assertTrue(readiness_gates["agent_audit"]["complete"]) + + bundle_response = self.client.post( + f"/api/workflows/{workflow_id}/export-bundle" + ) + self.assertEqual(bundle_response.status_code, 200) + self.assertEqual(len(bundle_response.json()["agent_plans"]), 1) + def test_hotspots_and_impact_preview(self): workflow, _ = self._current_workflow() workflow_id = workflow["id"] @@ -218,6 +291,264 @@ def test_hotspots_and_impact_preview(self): self.assertIn(impact_payload["confidence"], {"low", "medium", "high"}) self.assertIn("proofreading_mask_saved", impact_payload["signals"]) + recommendation_response = self.client.get( + f"/api/workflows/{workflow_id}/agent/recommendation" + ) + self.assertEqual(recommendation_response.status_code, 200) + recommendation = recommendation_response.json() + self.assertEqual(recommendation["stage"], "proofreading") + self.assertIn("training", recommendation["decision"].lower()) + self.assertEqual( + recommendation["impact_preview"]["corrected_mask_path"], + "/tmp/corrected-z12.tif", + ) + action_ids = {action["id"] for action in recommendation["actions"]} + self.assertIn("propose-retraining-handoff", action_ids) + readiness = {item["id"]: item for item in recommendation["readiness"]} + self.assertTrue(readiness["corrections"]["complete"]) + self.assertGreaterEqual(len(recommendation["commands"]), 1) + + def test_agent_can_start_proofreading_from_current_image_mask_pair(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + image_path = "/tmp/mito-image.h5" + mask_path = "/tmp/mito-seg.h5" + patch_response = self.client.patch( + f"/api/workflows/{workflow_id}", + json={ + "title": "Mito proofread", + "stage": "visualization", + "image_path": image_path, + "mask_path": mask_path, + }, + ) + self.assertEqual(patch_response.status_code, 200) + + recommendation_response = self.client.get( + f"/api/workflows/{workflow_id}/agent/recommendation" + ) + self.assertEqual(recommendation_response.status_code, 200) + recommendation = recommendation_response.json() + primary = next( + action + for action in recommendation["actions"] + if action["variant"] == "primary" + ) + self.assertEqual(primary["id"], "start-proofreading") + effects = primary["client_effects"] + self.assertEqual(effects["navigate_to"], "mask-proofreading") + self.assertEqual(effects["runtime_action"]["kind"], "start_proofreading") + self.assertEqual(effects["set_proofreading_dataset_path"], image_path) + self.assertEqual(effects["set_proofreading_mask_path"], mask_path) + self.assertEqual(effects["set_proofreading_project_name"], "Mito proofread") + + query_response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "proofread this data"}, + ) + self.assertEqual(query_response.status_code, 200) + query_payload = query_response.json() + self.assertIn("proofread this data", query_payload["response"].lower()) + self.assertEqual( + query_payload["commands"][0]["client_effects"]["runtime_action"]["kind"], + "start_proofreading", + ) + + def test_agent_routes_segmentation_and_capability_requests_to_app_actions(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + self.client.patch( + f"/api/workflows/{workflow_id}", + json={ + "stage": "inference", + "image_path": "/tmp/image.h5", + "mask_path": "/tmp/mask.h5", + "checkpoint_path": "/tmp/checkpoint.pth.tar", + }, + ) + + segment_response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "I want to get my volume segmented"}, + ) + self.assertEqual(segment_response.status_code, 200) + segment_payload = segment_response.json() + self.assertIn("run the model", segment_payload["response"].lower()) + conversation_id = segment_payload["conversationId"] + self.assertIsInstance(conversation_id, int) + self.assertEqual(segment_payload["actions"][0]["label"], "Run model") + self.assertEqual( + segment_payload["commands"][0]["client_effects"]["runtime_action"]["kind"], + "start_inference", + ) + conversation_response = self.client.get( + f"/chat/conversations/{conversation_id}" + ) + self.assertEqual(conversation_response.status_code, 200) + messages = conversation_response.json()["messages"] + self.assertEqual( + [message["role"] for message in messages], ["user", "assistant"] + ) + self.assertEqual(messages[0]["content"], "I want to get my volume segmented") + self.assertIn("run the model", messages[1]["content"].lower()) + self.assertEqual(messages[1]["source"], "workflow_orchestrator") + self.assertEqual(messages[1]["actions"][0]["label"], "Run model") + self.assertEqual( + messages[1]["commands"][0]["client_effects"]["runtime_action"]["kind"], + "start_inference", + ) + + capabilities_response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={ + "query": "what can the agent do then? can it run things?", + "conversation_id": conversation_id, + }, + ) + self.assertEqual(capabilities_response.status_code, 200) + capabilities_payload = capabilities_response.json() + self.assertEqual(capabilities_payload["conversationId"], conversation_id) + self.assertIn("run model inference", capabilities_payload["response"]) + self.assertGreaterEqual(len(capabilities_payload["actions"]), 1) + updated_conversation_response = self.client.get( + f"/chat/conversations/{conversation_id}" + ) + self.assertEqual(updated_conversation_response.status_code, 200) + self.assertEqual(len(updated_conversation_response.json()["messages"]), 4) + + def test_agent_answers_current_project_context_instead_of_repeating_next_step(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + self.client.patch( + f"/api/workflows/{workflow_id}", + json={ + "title": "mito25-paper-loop-smoke", + "stage": "proofreading", + "image_path": "/projects/mito25/data/image/mito25_im.h5", + "mask_path": "/projects/mito25/data/seg/mito25_seg.h5", + }, + ) + + response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "what exactly is the project I am on right now?"}, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertEqual(payload["intent"], "project_context") + self.assertIn("mito25-paper-loop-smoke", payload["response"]) + self.assertIn("proofreading", payload["response"]) + self.assertIn("mito25_im.h5", payload["response"]) + self.assertNotIn("0 inference failures", payload["response"]) + + def test_agent_handles_greetings_without_prompt_leakage(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + self.client.patch( + f"/api/workflows/{workflow_id}", + json={ + "stage": "proofreading", + "image_path": "/tmp/image.h5", + "mask_path": "/tmp/mask.h5", + }, + ) + + response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "hi!"}, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertIn("Hi.", payload["response"]) + self.assertIn("Next:", payload["response"]) + self.assertNotIn("Supervisor Agent", payload["response"]) + self.assertNotIn("RESPONSE STYLE", payload["response"]) + self.assertEqual(payload["actions"], []) + self.assertEqual(payload["commands"], []) + self.assertEqual(payload["intent"], "greeting") + + def test_agent_can_offer_evaluation_and_export_actions(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + self.client.patch( + f"/api/workflows/{workflow_id}", + json={ + "stage": "evaluation", + "label_path": "/tmp/ground-truth.tif", + "inference_output_path": "/tmp/candidate.tif", + }, + ) + baseline_run = self.client.post( + f"/api/workflows/{workflow_id}/model-runs", + json={ + "run_type": "inference", + "status": "completed", + "output_path": "/tmp/baseline.tif", + }, + ) + self.assertEqual(baseline_run.status_code, 200) + candidate_run = self.client.post( + f"/api/workflows/{workflow_id}/model-runs", + json={ + "run_type": "inference", + "status": "completed", + "output_path": "/tmp/candidate.tif", + }, + ) + self.assertEqual(candidate_run.status_code, 200) + + evaluation_response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "compare results and compute metrics"}, + ) + self.assertEqual(evaluation_response.status_code, 200) + evaluation_payload = evaluation_response.json() + self.assertIn("compute before/after metrics", evaluation_payload["response"]) + compute_action = evaluation_payload["actions"][0] + self.assertEqual(compute_action["id"], "compute-evaluation") + self.assertEqual(compute_action["risk_level"], "writes_workflow_record") + self.assertTrue(compute_action["requires_approval"]) + self.assertEqual( + compute_action["client_effects"]["workflow_action"]["kind"], + "compute_evaluation", + ) + self.assertEqual( + compute_action["client_effects"]["workflow_action"][ + "baseline_prediction_path" + ], + "/tmp/baseline.tif", + ) + + export_response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "export evidence bundle"}, + ) + self.assertEqual(export_response.status_code, 200) + export_payload = export_response.json() + self.assertEqual(export_payload["actions"][0]["id"], "export-workflow-bundle") + self.assertEqual(export_payload["actions"][0]["risk_level"], "exports_evidence") + self.assertTrue(export_payload["actions"][0]["requires_approval"]) + self.assertEqual( + export_payload["actions"][0]["client_effects"]["workflow_action"]["kind"], + "export_bundle", + ) + + def test_agent_slash_command_aliases_route_to_actions(self): + workflow, _ = self._current_workflow() + workflow_id = workflow["id"] + + response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "/infer"}, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertEqual(payload["intent"], "start_inference") + self.assertEqual(payload["actions"][0]["id"], "start-inference") + self.assertEqual(payload["actions"][0]["risk_level"], "runs_job") + self.assertTrue(payload["actions"][0]["requires_approval"]) + self.assertTrue(payload["tasks"]) + def test_ehtool_load_classify_save_and_export_append_workflow_events(self): workflow, _ = self._current_workflow() workflow_id = workflow["id"] diff --git a/tests/test_workflow_spine_smoke.py b/tests/test_workflow_spine_smoke.py index d46ace5c..f0565b62 100644 --- a/tests/test_workflow_spine_smoke.py +++ b/tests/test_workflow_spine_smoke.py @@ -12,6 +12,7 @@ from server_api.auth import database as auth_database from server_api.auth import models +import server_api.main as server_api_main from server_api.main import app as server_api_app @@ -42,6 +43,9 @@ def tearDown(self): self.engine.dispose() self.temp_dir.cleanup() + def test_chatbot_backend_import_contract_is_available(self): + self.assertIsNotNone(server_api_main.build_chain) + def test_spine_loop_load_proofread_stage_and_export_evidence(self): workflow_response = self.client.get("/api/workflows/current") self.assertEqual(workflow_response.status_code, 200) @@ -71,8 +75,15 @@ def test_spine_loop_load_proofread_stage_and_export_evidence(self): json={"query": "stage corrected masks for retraining"}, ) self.assertEqual(query_response.status_code, 200) - proposals = query_response.json()["proposals"] + query_payload = query_response.json() + proposals = query_payload["proposals"] self.assertEqual(len(proposals), 1) + self.assertGreaterEqual(len(query_payload["actions"]), 1) + self.assertGreaterEqual(len(query_payload["commands"]), 1) + self.assertEqual( + query_payload["actions"][0]["client_effects"]["navigate_to"], + "training", + ) proposal_id = proposals[0]["id"] approve_response = self.client.post( @@ -83,6 +94,23 @@ def test_spine_loop_load_proofread_stage_and_export_evidence(self): approve_response.json()["workflow"]["stage"], "retraining_staged" ) + launch_query_response = self.client.post( + f"/api/workflows/{workflow_id}/agent/query", + json={"query": "start training"}, + ) + self.assertEqual(launch_query_response.status_code, 200) + launch_query_payload = launch_query_response.json() + self.assertEqual( + launch_query_payload["commands"][0]["client_effects"]["runtime_action"][ + "kind" + ], + "start_training", + ) + self.assertEqual( + launch_query_payload["commands"][0]["client_effects"]["navigate_to"], + "training", + ) + hotspots_response = self.client.get(f"/api/workflows/{workflow_id}/hotspots") impact_response = self.client.get(f"/api/workflows/{workflow_id}/impact-preview") metrics_response = self.client.get(f"/api/workflows/{workflow_id}/metrics") diff --git a/uv.lock b/uv.lock index 44b6b7d0..94326a49 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.10, <3.12" resolution-markers = [ "python_full_version >= '3.11' and sys_platform == 'darwin'", @@ -2829,6 +2829,7 @@ dependencies = [ { name = "langchain-classic" }, { name = "langchain-community" }, { name = "langchain-ollama" }, + { name = "langgraph" }, { name = "neuroglancer" }, { name = "numpy" }, { name = "opencv-python" }, @@ -2869,6 +2870,7 @@ requires-dist = [ { name = "langchain-classic", specifier = "==1.0.0" }, { name = "langchain-community", specifier = "==0.4.1" }, { name = "langchain-ollama", specifier = "==1.0.0" }, + { name = "langgraph", specifier = ">=1.0.0" }, { name = "neuroglancer", specifier = "==2.38" }, { name = "numpy", specifier = ">=1.24,<2.0" }, { name = "opencv-python", specifier = ">=4.11.0.86" }, From 9b9940a8b5e57c0abfaf1583e849c203b5d17d31 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Sun, 26 Apr 2026 09:12:25 -0400 Subject: [PATCH 03/38] feat: improve project mounting and volume intake --- client/src/components/FilePickerModal.js | 20 +- client/src/components/UnifiedFileInput.js | 47 +-- client/src/views/FilesManager.js | 231 ++++++++++++- client/src/views/FilesManager.test.js | 106 +++++- server_api/auth/router.py | 318 +++++++++++++++++- server_api/ehtool/utils.py | 24 +- tests/test_file_workspace_routes.py | 125 +++++++ .../test_neuroglancer_volume_normalization.py | 89 +++++ tests/test_pytc_architectures_route.py | 24 ++ tests/test_volume_io.py | 72 ++++ 10 files changed, 1011 insertions(+), 45 deletions(-) create mode 100644 tests/test_neuroglancer_volume_normalization.py create mode 100644 tests/test_pytc_architectures_route.py create mode 100644 tests/test_volume_io.py diff --git a/client/src/components/FilePickerModal.js b/client/src/components/FilePickerModal.js index c7739888..ab7fb3ea 100644 --- a/client/src/components/FilePickerModal.js +++ b/client/src/components/FilePickerModal.js @@ -11,6 +11,7 @@ import { apiClient } from "../api"; const HIDDEN_SYSTEM_FILES = new Set([ "workflow_preference.json", ".pytc_proofreading.json", + ".pytc_instance_labels.tif", ".ds_store", "thumbs.db", ]); @@ -23,6 +24,16 @@ const IMAGE_EXTENSIONS = new Set([ ".tif", ".tiff", ".webp", + ".h5", + ".hdf5", + ".npy", + ".npz", + ".zarr", + ".n5", + ".nii", + ".nii.gz", + ".mrc", + ".mrcs", ]); const FilePickerModal = ({ @@ -181,10 +192,9 @@ const FilePickerModal = ({ const isImageFile = (item) => { if (!item || item.is_folder) return false; if (item.type && item.type.startsWith("image/")) return true; - const ext = `.${String(item.name || "") - .split(".") - .pop()}`.toLowerCase(); - return IMAGE_EXTENSIONS.has(ext); + const name = String(item.name || "").toLowerCase(); + const ext = `.${name.split(".").pop()}`; + return IMAGE_EXTENSIONS.has(ext) || name.endsWith(".nii.gz"); }; const getPreviewUrl = (item) => `${previewBaseUrl}/files/preview/${item.id}`; @@ -256,7 +266,7 @@ const FilePickerModal = ({ onClick={() => setOnlyImages((prev) => !prev)} style={{ marginLeft: 8 }} > - Images Only + Volume files diff --git a/client/src/components/UnifiedFileInput.js b/client/src/components/UnifiedFileInput.js index 7fe8dec0..05b88e36 100644 --- a/client/src/components/UnifiedFileInput.js +++ b/client/src/components/UnifiedFileInput.js @@ -1,5 +1,5 @@ import React, { useState } from "react"; -import { Input, message } from "antd"; +import { Button, Input, message, Space } from "antd"; import { FolderOpenOutlined } from "@ant-design/icons"; import FilePickerModal from "./FilePickerModal"; @@ -108,27 +108,30 @@ const UnifiedFileInput = ({ ...style, }} > - - } - /> + + + } + /> + + {isDragOver && (
{ + if (!rootPath || !relativePath) return ""; + return `${String(rootPath).replace(/\/+$/, "")}/${String(relativePath).replace(/^\/+/, "")}`; +}; + +const buildWorkflowPatchFromProjectSuggestion = (suggestion) => { + const profile = suggestion?.profile || {}; + const examples = profile.examples || {}; + const paired = profile.paired_examples?.[0] || {}; + const rootPath = suggestion?.directory_path; + const imageRelative = paired.image || examples.image?.[0]; + const labelRelative = paired.label || examples.label?.[0]; + const predictionRelative = examples.prediction?.[0]; + const checkpointRelative = examples.checkpoint?.[0]; + const patch = { + dataset_path: rootPath || null, + image_path: joinProjectPath(rootPath, imageRelative) || null, + label_path: joinProjectPath(rootPath, labelRelative) || null, + mask_path: joinProjectPath(rootPath, labelRelative) || null, + inference_output_path: joinProjectPath(rootPath, predictionRelative) || null, + checkpoint_path: joinProjectPath(rootPath, checkpointRelative) || null, + }; + return Object.fromEntries( + Object.entries(patch).filter(([, value]) => Boolean(value)), + ); +}; const collectDescendantFolderIds = (folderList, rootIds) => { const removed = new Set(); @@ -118,6 +165,7 @@ const transformFiles = (fileList) => { function FilesManager() { const context = useContext(AppContext); + const workflowContext = useWorkflow(); const [folders, setFolders] = useState([]); const [files, setFiles] = useState({}); const foldersRef = useRef([]); @@ -142,6 +190,7 @@ function FilesManager() { const [serverUnavailable, setServerUnavailable] = useState(false); const [hasShownServerWarning, setHasShownServerWarning] = useState(false); const [previewStatus, setPreviewStatus] = useState({}); + const [projectSuggestions, setProjectSuggestions] = useState([]); const containerRef = useRef(null); const itemRefs = useRef({}); const isDragSelecting = useRef(false); @@ -435,13 +484,29 @@ function FilesManager() { setNewItemType(null); }; + const loadProjectSuggestions = React.useCallback(async () => { + try { + const response = await apiClient.get("/files/project-suggestions", { + withCredentials: true, + }); + setProjectSuggestions(response.data || []); + } catch (error) { + // Suggestions are convenience only; normal mounting must keep working. + console.warn("Could not load project suggestions", error); + setProjectSuggestions([]); + } + }, []); + + useEffect(() => { + loadProjectSuggestions(); + }, [loadProjectSuggestions]); + const isImageFile = (file) => { if (!file || file.is_folder) return false; if (file.type && file.type.startsWith("image/")) return true; - const ext = `.${String(file.name || "") - .split(".") - .pop()}`.toLowerCase(); - return IMAGE_EXTENSIONS.has(ext); + const name = String(file.name || "").toLowerCase(); + const ext = `.${name.split(".").pop()}`; + return IMAGE_EXTENSIONS.has(ext) || name.endsWith(".nii.gz"); }; const getPreviewUrl = (fileKey) => @@ -1297,6 +1362,83 @@ function FilesManager() { const currentFolders = folders.filter((f) => f.parent === currentFolder); const currentFiles = files[currentFolder] || []; + const suggestedProject = + projectSuggestions.find((item) => item.recommended) || projectSuggestions[0]; + + const renderProjectSuggestion = () => { + if (!suggestedProject || currentFolder !== "root") { + return null; + } + const profile = suggestedProject.profile || {}; + const counts = profile.counts || {}; + const shownRoles = Object.entries(PROJECT_ROLE_LABELS).filter( + ([role]) => counts[role] > 0, + ); + const missingRoles = profile.missing_roles || []; + const pairedExample = profile.paired_examples?.[0]; + + return ( +
+
+
+ Project setup: {suggestedProject.name} +
+
+ {profile.ready_for_smoke + ? "Ready for image/label, checkpoint, prediction, and metric checks." + : missingRoles.length + ? `Missing detected ${missingRoles.join(", ")} role(s).` + : suggestedProject.description} +
+ {pairedExample && ( +
+ Pair: {pairedExample.image} -> {pairedExample.label} +
+ )} +
+
+ {shownRoles.map(([role, label]) => ( + + {counts[role]} {label} + + ))} +
+ +
+ ); + }; const getContextMenuItems = () => { if (contextMenu?.type === "container") { @@ -1435,6 +1577,7 @@ function FilesManager() { ); await fetchFolderContents("root", { force: true }); + await loadProjectSuggestions(); if (res?.data?.mounted_root_id) { const mountedRootId = String(res.data.mounted_root_id); await fetchFolderContents(mountedRootId, { force: true }); @@ -1447,6 +1590,72 @@ function FilesManager() { } }; + const handleSuggestedProject = async () => { + const suggestion = + projectSuggestions.find((item) => item.recommended) || projectSuggestions[0]; + if (!suggestion) { + message.info("No suggested local project is available."); + return; + } + if (suggestion.already_mounted && suggestion.mounted_root_id) { + const mountedRootId = String(suggestion.mounted_root_id); + await fetchFolderContents(mountedRootId, { force: true }); + handleNavigate(mountedRootId); + await registerProjectWithWorkflow(suggestion); + message.success(`${suggestion.name} is already mounted.`); + return; + } + try { + const res = await apiClient.post( + "/files/mount", + { + directory_path: suggestion.directory_path, + destination_path: "root", + mount_name: suggestion.name, + }, + { withCredentials: true }, + ); + await fetchFolderContents("root", { force: true }); + await loadProjectSuggestions(); + if (res?.data?.mounted_root_id) { + const mountedRootId = String(res.data.mounted_root_id); + await fetchFolderContents(mountedRootId, { force: true }); + handleNavigate(mountedRootId); + } + await registerProjectWithWorkflow(suggestion); + message.success(res?.data?.message || `${suggestion.name} mounted.`); + } catch (error) { + console.error("Suggested project mount error", error); + message.error(`Failed to mount ${suggestion.name}`); + } + }; + + const registerProjectWithWorkflow = async (suggestion) => { + if (!workflowContext?.updateWorkflow) { + return; + } + const patch = buildWorkflowPatchFromProjectSuggestion(suggestion); + if (!Object.keys(patch).length) { + return; + } + try { + await workflowContext.updateWorkflow(patch); + await workflowContext.appendEvent?.({ + actor: "user", + event_type: "dataset.loaded", + stage: workflowContext.workflow?.stage || "setup", + summary: `Registered project roles from ${suggestion.name}.`, + payload: { + source: "file_management_project_suggestion", + suggestion_id: suggestion.id, + ...patch, + }, + }); + } catch (error) { + console.warn("Failed to register mounted project with workflow", error); + } + }; + const handleResetWorkspace = () => { Modal.confirm({ title: "Reset workspace?", @@ -1766,6 +1975,19 @@ function FilesManager() { > Mount Project + } title="More file actions" />
+ {renderProjectSuggestion()} {/* Content Area */}
({ jest.mock("../components/FileTreeSidebar", () => () => null); +const mockWorkflowContext = { + workflow: { id: 1, stage: "setup" }, + updateWorkflow: jest.fn(), + appendEvent: jest.fn(), +}; + +jest.mock("../contexts/WorkflowContext", () => ({ + useWorkflow: () => mockWorkflowContext, +})); + jest.mock("@ant-design/icons", () => { const Icon = () => ; return { @@ -90,6 +100,12 @@ jest.mock("antd", () => { }); describe("FilesManager", () => { + beforeEach(() => { + jest.clearAllMocks(); + mockWorkflowContext.updateWorkflow.mockResolvedValue({}); + mockWorkflowContext.appendEvent.mockResolvedValue({}); + }); + it("loads root-level files with a parent-scoped request on initial render", async () => { apiClient.get.mockResolvedValue({ data: [] }); @@ -107,4 +123,92 @@ describe("FilesManager", () => { expect(apiClient.get).not.toHaveBeenCalledWith("/files"); }); + + it("mounts a suggested smoke project without manually browsing for a path", async () => { + apiClient.get.mockImplementation((url) => { + if (url === "/files/project-suggestions") { + return Promise.resolve({ + data: [ + { + id: "mito25-paper-loop-smoke", + name: "mito25-paper-loop-smoke", + directory_path: "/tmp/mito25_paper_loop_smoke", + description: "Curated smoke project", + recommended: true, + already_mounted: false, + profile: { + ready_for_smoke: true, + counts: { + image: 1, + label: 1, + prediction: 2, + config: 1, + checkpoint: 1, + }, + paired_examples: [ + { + image: "data/image/mito25_smoke_im.h5", + label: "data/seg/mito25_smoke_seg.h5", + }, + ], + examples: { + image: ["data/image/mito25_smoke_im.h5"], + label: ["data/seg/mito25_smoke_seg.h5"], + prediction: ["predictions/baseline.tif"], + checkpoint: ["checkpoints/checkpoint_00200.pth.tar"], + }, + missing_roles: [], + }, + }, + ], + }); + } + return Promise.resolve({ data: [] }); + }); + apiClient.post.mockResolvedValue({ + data: { mounted_root_id: 7, message: "Mounted smoke project" }, + }); + + render( + + + , + ); + + expect( + await screen.findByText("Project setup: mito25-paper-loop-smoke"), + ).toBeTruthy(); + expect(screen.getByText("2 prediction")).toBeTruthy(); + fireEvent.click(await screen.findByText("Mount Test Project")); + + await waitFor(() => { + expect(apiClient.post).toHaveBeenCalledWith( + "/files/mount", + { + directory_path: "/tmp/mito25_paper_loop_smoke", + destination_path: "root", + mount_name: "mito25-paper-loop-smoke", + }, + { withCredentials: true }, + ); + }); + expect(apiClient.get).toHaveBeenCalledWith("/files", { + params: { parent: "7" }, + }); + expect(mockWorkflowContext.updateWorkflow).toHaveBeenCalledWith({ + dataset_path: "/tmp/mito25_paper_loop_smoke", + image_path: "/tmp/mito25_paper_loop_smoke/data/image/mito25_smoke_im.h5", + label_path: "/tmp/mito25_paper_loop_smoke/data/seg/mito25_smoke_seg.h5", + mask_path: "/tmp/mito25_paper_loop_smoke/data/seg/mito25_smoke_seg.h5", + inference_output_path: "/tmp/mito25_paper_loop_smoke/predictions/baseline.tif", + checkpoint_path: + "/tmp/mito25_paper_loop_smoke/checkpoints/checkpoint_00200.pth.tar", + }); + expect(mockWorkflowContext.appendEvent).toHaveBeenCalledWith( + expect.objectContaining({ + event_type: "dataset.loaded", + summary: "Registered project roles from mito25-paper-loop-smoke.", + }), + ); + }); }); diff --git a/server_api/auth/router.py b/server_api/auth/router.py index f50f7c7b..d95126a4 100644 --- a/server_api/auth/router.py +++ b/server_api/auth/router.py @@ -9,6 +9,7 @@ import os import uuid import mimetypes +from server_api.utils.utils import resolve_existing_path try: # Optional preview dependencies import cv2 @@ -22,7 +23,32 @@ router = APIRouter() oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token", auto_error=False) -IGNORED_SYSTEM_FILENAMES = {".ds_store", "thumbs.db"} +IGNORED_SYSTEM_FILENAMES = { + ".ds_store", + "thumbs.db", + ".pytc_proofreading.json", + ".pytc_instance_labels.tif", +} +PROJECT_PROFILE_MAX_FILES = 2500 + +VOLUME_EXTENSIONS = { + ".h5", + ".hdf5", + ".tif", + ".tiff", + ".ome.tif", + ".ome.tiff", + ".npy", + ".npz", + ".zarr", + ".n5", + ".nii", + ".nii.gz", + ".mrc", + ".mrcs", +} +CONFIG_EXTENSIONS = {".yaml", ".yml", ".json", ".toml"} +CHECKPOINT_EXTENSIONS = {".pt", ".pth", ".pth.tar", ".ckpt", ".onnx"} def _format_size(size_bytes: int) -> str: @@ -39,6 +65,186 @@ def _is_ignored_system_file(name: Optional[str]) -> bool: return str(name or "").strip().lower() in IGNORED_SYSTEM_FILENAMES +def _project_suggestion_candidates() -> List[dict]: + repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) + workspace_root = os.path.abspath(os.path.join(repo_root, "..")) + return [ + { + "id": "mito25-paper-loop-smoke", + "name": "mito25-paper-loop-smoke", + "directory_path": os.path.join( + repo_root, "testing_projects", "mito25_paper_loop_smoke" + ), + "description": "Curated mito25 smoke project with image/seg, configs, checkpoint, and prediction artifacts.", + "recommended": True, + }, + { + "id": "mito25-raw-data", + "name": "mito25", + "directory_path": os.path.join(workspace_root, "testing_data", "mito25"), + "description": "Raw mito25 image/seg source data.", + "recommended": False, + }, + ] + + +def _lower_path_parts(path: str) -> List[str]: + normalized = os.path.normpath(path).lower() + return [part for part in normalized.split(os.sep) if part] + + +def _project_extension(name: str) -> str: + lower = name.lower() + for compound in (".ome.tiff", ".ome.tif", ".nii.gz", ".pth.tar"): + if lower.endswith(compound): + return compound + return os.path.splitext(lower)[1] + + +def _role_for_project_file(relative_path: str) -> Optional[str]: + parts = _lower_path_parts(relative_path) + name = parts[-1] if parts else os.path.basename(relative_path).lower() + stem = name + extension = _project_extension(name) + if extension: + stem = name[: -len(extension)] + haystack = " ".join(parts + [stem]) + + if extension in CHECKPOINT_EXTENSIONS or "checkpoint" in haystack: + return "checkpoint" + if extension in CONFIG_EXTENSIONS and ( + "config" in haystack or "mito" in haystack or "pytc" in haystack + ): + return "config" + if extension not in VOLUME_EXTENSIONS: + if extension in {".md", ".txt"} or "notes" in haystack: + return "notes" + return None + + if any( + token in haystack + for token in ( + "prediction", + "predictions", + "result", + "inference", + "candidate", + "baseline", + ) + ): + return "prediction" + if any( + token in haystack + for token in ( + "_seg", + " seg", + "segmentation", + "label", + "labels", + "mask", + "masks", + "ground", + "truth", + "consensus", + "gt", + ) + ): + return "label" + if any(token in haystack for token in ("image", "images", "_im", "raw")): + return "image" + return "volume" + + +def _looks_like_image_label_pair(image_path: str, label_path: str) -> bool: + image_name = os.path.basename(image_path).lower() + label_name = os.path.basename(label_path).lower() + normalized_image = ( + image_name.replace("_image", "") + .replace("_images", "") + .replace("_im", "") + .replace("-image", "") + ) + normalized_label = ( + label_name.replace("_seg", "") + .replace("_label", "") + .replace("_labels", "") + .replace("_mask", "") + .replace("_consensus", "") + .replace("-seg", "") + ) + return normalized_image.split(".")[0] == normalized_label.split(".")[0] + + +def _scan_project_profile(directory_path: str) -> dict: + roles = { + "image": [], + "label": [], + "prediction": [], + "config": [], + "checkpoint": [], + "volume": [], + "notes": [], + } + counts = {role: 0 for role in roles} + scanned_files = 0 + truncated = False + + for current_dir, dirnames, filenames in os.walk(directory_path): + dirnames[:] = [ + dirname + for dirname in sorted(dirnames) + if not _is_ignored_system_file(dirname) + ] + for filename in sorted(filenames): + if _is_ignored_system_file(filename): + continue + scanned_files += 1 + if scanned_files > PROJECT_PROFILE_MAX_FILES: + truncated = True + break + absolute_path = os.path.join(current_dir, filename) + relative_path = os.path.relpath(absolute_path, directory_path) + role = _role_for_project_file(relative_path) + if role and role in roles: + counts[role] += 1 + if len(roles[role]) < 8: + roles[role].append(relative_path) + if truncated: + break + + paired_examples = [] + for image_path in roles["image"]: + for label_path in roles["label"]: + if _looks_like_image_label_pair(image_path, label_path): + paired_examples.append( + {"image": image_path, "label": label_path} + ) + break + if len(paired_examples) >= 4: + break + + required_roles = { + "image": counts["image"] > 0, + "label": counts["label"] > 0, + "config": counts["config"] > 0, + "checkpoint": counts["checkpoint"] > 0, + "prediction": counts["prediction"] >= 2, + } + missing_roles = [ + role for role, present in required_roles.items() if not present + ] + + return { + "scanned_files": min(scanned_files, PROJECT_PROFILE_MAX_FILES), + "truncated": truncated, + "counts": counts, + "examples": roles, + "paired_examples": paired_examples, + "ready_for_smoke": not missing_roles, + "missing_roles": missing_roles, + } + + def _ensure_unique_name( db: Session, user_id: int, parent_path: str, base_name: str ) -> str: @@ -69,6 +275,74 @@ def _is_managed_upload_path(user_id: int, physical_path: Optional[str]) -> bool: return False +def _repair_stale_mounted_entries(db: Session, user_id: int) -> None: + candidates = ( + db.query(models.File) + .filter( + models.File.user_id == user_id, + models.File.physical_path.isnot(None), + models.File.path != "root", + ) + .all() + ) + + changed = False + for entry in candidates: + if not entry.physical_path: + continue + if _is_managed_upload_path(user_id, entry.physical_path): + continue + if os.path.exists(entry.physical_path): + continue + + repaired = resolve_existing_path(entry.physical_path) + if repaired is None or not repaired.exists(): + continue + if entry.is_folder and not repaired.is_dir(): + continue + if not entry.is_folder and not repaired.is_file(): + continue + + entry.physical_path = str(repaired) + entry.name = repaired.name + if not entry.is_folder: + entry.type = mimetypes.guess_type(str(repaired))[0] or entry.type + try: + entry.size = _format_size(repaired.stat().st_size) + except OSError: + pass + changed = True + + if changed: + db.commit() + + +def _prune_missing_managed_upload_entries(db: Session, user_id: int) -> None: + candidates = ( + db.query(models.File) + .filter( + models.File.user_id == user_id, + models.File.is_folder.is_(False), + models.File.physical_path.isnot(None), + ) + .all() + ) + + removed = False + for entry in candidates: + if not entry.physical_path: + continue + if not _is_managed_upload_path(user_id, entry.physical_path): + continue + if os.path.exists(entry.physical_path): + continue + db.delete(entry) + removed = True + + if removed: + db.commit() + + def _delete_file_tree( db: Session, user_id: int, @@ -189,6 +463,8 @@ def get_files( current_user: models.User = Depends(get_current_user), db: Session = Depends(database.get_db), ): + _repair_stale_mounted_entries(db, current_user.id) + _prune_missing_managed_upload_entries(db, current_user.id) query = db.query(models.File).filter(models.File.user_id == current_user.id) if parent is not None: query = query.filter(models.File.path == parent) @@ -513,6 +789,46 @@ def mount_directory( } +@router.get("/files/project-suggestions") +def list_project_suggestions( + current_user: models.User = Depends(get_current_user), + db: Session = Depends(database.get_db), +): + suggestions = [] + mounted_roots = ( + db.query(models.File) + .filter( + models.File.user_id == current_user.id, + models.File.path == "root", + models.File.is_folder.is_(True), + models.File.physical_path.isnot(None), + ) + .all() + ) + mounted_by_path = { + os.path.abspath(os.path.expanduser(root.physical_path)): root + for root in mounted_roots + if root.physical_path + } + for candidate in _project_suggestion_candidates(): + directory_path = os.path.abspath( + os.path.expanduser(candidate["directory_path"]) + ) + if not os.path.isdir(directory_path): + continue + mounted_root = mounted_by_path.get(directory_path) + suggestions.append( + { + **candidate, + "directory_path": directory_path, + "already_mounted": mounted_root is not None, + "mounted_root_id": mounted_root.id if mounted_root else None, + "profile": _scan_project_profile(directory_path), + } + ) + return suggestions + + @router.put("/files/{file_id}", response_model=models.FileResponse) def update_file( file_id: int, diff --git a/server_api/ehtool/utils.py b/server_api/ehtool/utils.py index d74d4563..3b4b5644 100644 --- a/server_api/ehtool/utils.py +++ b/server_api/ehtool/utils.py @@ -40,21 +40,21 @@ def _build_glasbey_palette(size: int = 256) -> None: def labels_to_rgba(label_slice: np.ndarray) -> np.ndarray: """Convert a label slice into an RGBA overlay using a Glasbey-style palette.""" _build_glasbey_palette() - labels = np.unique(label_slice) - labels = labels[labels != 0] - - h, w = label_slice.shape + label_array = np.asarray(label_slice) + h, w = label_array.shape rgba = np.zeros((h, w, 4), dtype=np.uint8) - if labels.size == 0: + + foreground = label_array != 0 + if not np.any(foreground): return rgba - for label in labels: - color = GLASBEY_COLORS[int(label) % len(GLASBEY_COLORS)] - mask = label_slice == label - rgba[mask, 0] = color[0] - rgba[mask, 1] = color[1] - rgba[mask, 2] = color[2] - rgba[mask, 3] = 255 + palette = np.asarray(GLASBEY_COLORS, dtype=np.uint8) + label_indices = np.mod( + label_array[foreground].astype(np.int64, copy=False), + len(palette), + ) + rgba[foreground, :3] = palette[label_indices] + rgba[foreground, 3] = 255 return rgba diff --git a/tests/test_file_workspace_routes.py b/tests/test_file_workspace_routes.py index 7d64c74a..fc52df3d 100644 --- a/tests/test_file_workspace_routes.py +++ b/tests/test_file_workspace_routes.py @@ -9,6 +9,7 @@ from server_api.auth import database as auth_database from server_api.auth import models +from server_api.auth.router import _scan_project_profile from server_api.main import app as server_api_app @@ -157,6 +158,130 @@ def test_reset_workspace_preserves_mounted_sources_and_clears_uploads(self): self.assertEqual(files_response.status_code, 200) self.assertEqual(files_response.json(), []) + def test_get_files_repairs_renamed_mounted_file_entries(self): + mount_root = pathlib.Path(self.temp_dir.name) / "project" + mount_root.mkdir() + original_file = mount_root / "volume.h5" + original_file.write_bytes(b"data") + + mount_response = self.client.post( + "/files/mount", + json={ + "directory_path": str(mount_root), + "destination_path": "root", + }, + ) + self.assertEqual(mount_response.status_code, 200) + mounted_root_id = mount_response.json()["mounted_root_id"] + + renamed_file = mount_root / "volume_im.h5" + original_file.rename(renamed_file) + + child_response = self.client.get( + "/files", + params={"parent": str(mounted_root_id)}, + ) + self.assertEqual(child_response.status_code, 200) + + child_entries = child_response.json() + self.assertEqual([item["name"] for item in child_entries], ["volume_im.h5"]) + self.assertEqual(child_entries[0]["physical_path"], str(renamed_file)) + + def test_get_files_prunes_missing_managed_upload_entries(self): + missing_upload = self.uploads_root / "ghost.h5" + self._create_file( + name="ghost.h5", + physical_path=str(missing_upload), + size="10B", + file_type="application/x-hdf5", + ) + + response = self.client.get("/files") + self.assertEqual(response.status_code, 200) + returned_names = [item["name"] for item in response.json()] + self.assertNotIn("ghost.h5", returned_names) + + with self.SessionLocal() as db: + ghost = db.query(models.File).filter(models.File.name == "ghost.h5").first() + self.assertIsNone(ghost) + + def test_project_suggestions_marks_existing_smoke_project_mount(self): + repo_root = pathlib.Path(__file__).resolve().parents[1] + smoke_project = repo_root / "testing_projects" / "mito25_paper_loop_smoke" + smoke_project.mkdir(parents=True, exist_ok=True) + + response = self.client.get("/files/project-suggestions") + self.assertEqual(response.status_code, 200) + suggestions = response.json() + smoke_suggestion = next( + item for item in suggestions if item["id"] == "mito25-paper-loop-smoke" + ) + self.assertFalse(smoke_suggestion["already_mounted"]) + + mount_response = self.client.post( + "/files/mount", + json={ + "directory_path": str(smoke_project), + "destination_path": "root", + "mount_name": "mito25-paper-loop-smoke", + }, + ) + self.assertEqual(mount_response.status_code, 200) + + mounted_response = self.client.get("/files/project-suggestions") + self.assertEqual(mounted_response.status_code, 200) + mounted_smoke = next( + item + for item in mounted_response.json() + if item["id"] == "mito25-paper-loop-smoke" + ) + self.assertTrue(mounted_smoke["already_mounted"]) + self.assertEqual( + mounted_smoke["mounted_root_id"], + mount_response.json()["mounted_root_id"], + ) + self.assertIn("profile", mounted_smoke) + + def test_project_profile_detects_smoke_project_roles(self): + project_root = pathlib.Path(self.temp_dir.name) / "project-profile" + (project_root / "data" / "image").mkdir(parents=True) + (project_root / "data" / "seg").mkdir(parents=True) + (project_root / "configs").mkdir() + (project_root / "checkpoints").mkdir() + (project_root / "predictions").mkdir() + (project_root / "data" / "image" / "mito25_smoke_im.h5").write_bytes(b"im") + (project_root / "data" / "seg" / "mito25_smoke_seg.h5").write_bytes(b"seg") + (project_root / "configs" / "Mito25-Local-Smoke.yaml").write_text( + "SYSTEM: {}\n", encoding="utf-8" + ) + (project_root / "checkpoints" / "checkpoint_00001.pth.tar").write_bytes( + b"ckpt" + ) + (project_root / "predictions" / "baseline_result_xy.h5").write_bytes( + b"baseline" + ) + (project_root / "predictions" / "candidate_result_xy.h5").write_bytes( + b"candidate" + ) + + profile = _scan_project_profile(str(project_root)) + + self.assertTrue(profile["ready_for_smoke"]) + self.assertEqual(profile["counts"]["image"], 1) + self.assertEqual(profile["counts"]["label"], 1) + self.assertEqual(profile["counts"]["prediction"], 2) + self.assertEqual(profile["counts"]["config"], 1) + self.assertEqual(profile["counts"]["checkpoint"], 1) + self.assertEqual( + profile["paired_examples"], + [ + { + "image": "data/image/mito25_smoke_im.h5", + "label": "data/seg/mito25_smoke_seg.h5", + } + ], + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_neuroglancer_volume_normalization.py b/tests/test_neuroglancer_volume_normalization.py new file mode 100644 index 00000000..07cc0143 --- /dev/null +++ b/tests/test_neuroglancer_volume_normalization.py @@ -0,0 +1,89 @@ +import numpy as np + +from server_api.main import ( + _normalize_segmentation_volume_for_neuroglancer, +) + + +def test_normalize_segmentation_converts_non_negative_int64_to_unsigned(): + volume = np.array([[0, 1], [2, 255]], dtype=np.int64) + + normalized = _normalize_segmentation_volume_for_neuroglancer(volume) + + assert normalized.dtype == np.uint8 + assert np.array_equal(normalized, volume) + + +def test_normalize_segmentation_uses_uint64_for_large_labels(): + volume = np.array([[0, 2**40]], dtype=np.int64) + + normalized = _normalize_segmentation_volume_for_neuroglancer(volume) + + assert normalized.dtype == np.uint64 + assert np.array_equal(normalized, volume.astype(np.uint64)) + + +def test_normalize_segmentation_rejects_negative_labels(): + volume = np.array([[0, -1]], dtype=np.int64) + + try: + _normalize_segmentation_volume_for_neuroglancer(volume) + except ValueError as exc: + assert "non-negative" in str(exc) + else: + raise AssertionError("expected negative labels to be rejected") + + +def test_normalize_segmentation_accepts_integral_float_labels(): + volume = np.array([[0.0, 1.0], [2.0, 3.0]], dtype=np.float32) + + normalized = _normalize_segmentation_volume_for_neuroglancer(volume) + + assert normalized.dtype == np.uint8 + assert np.array_equal(normalized, volume.astype(np.uint8)) + + +def test_normalize_segmentation_rejects_fractional_float_labels(): + volume = np.array([[0.5, 1.0]], dtype=np.float32) + + try: + _normalize_segmentation_volume_for_neuroglancer(volume) + except ValueError as exc: + assert "integer-valued" in str(exc) + else: + raise AssertionError("expected fractional labels to be rejected") + + +def test_normalize_segmentation_squeezes_singleton_channel_volumes(): + volume = np.array([[[[0, 1], [2, 3]]]], dtype=np.uint8) + + normalized = _normalize_segmentation_volume_for_neuroglancer(volume) + + assert normalized.shape == (1, 2, 2) + assert normalized.dtype == np.uint8 + assert np.array_equal(normalized, volume[0]) + + +def test_normalize_segmentation_builds_preview_from_two_channel_prediction(): + semantic = np.zeros((8, 8, 8), dtype=np.uint8) + boundary = np.zeros_like(semantic) + semantic[1:7, 1:7, 1:7] = 255 + volume = np.stack([semantic, boundary], axis=0) + + normalized = _normalize_segmentation_volume_for_neuroglancer(volume) + + assert normalized.ndim == 3 + assert normalized.shape == semantic.shape + assert normalized.dtype == np.uint8 + assert int(normalized.max()) > 0 + + +def test_normalize_segmentation_uses_argmax_for_multiclass_prediction(): + volume = np.zeros((3, 2, 2, 2), dtype=np.uint8) + volume[2, 1, 1, 1] = 255 + + normalized = _normalize_segmentation_volume_for_neuroglancer(volume) + + assert normalized.shape == (2, 2, 2) + assert normalized.dtype == np.uint8 + assert int(normalized[1, 1, 1]) == 2 diff --git a/tests/test_pytc_architectures_route.py b/tests/test_pytc_architectures_route.py new file mode 100644 index 00000000..6c4d3404 --- /dev/null +++ b/tests/test_pytc_architectures_route.py @@ -0,0 +1,24 @@ +import unittest + +from fastapi.testclient import TestClient + +from server_api.main import app as server_api_app + + +class PytcArchitectureRouteTests(unittest.TestCase): + def setUp(self): + self.client = TestClient(server_api_app) + + def test_pytc_architectures_returns_known_models(self): + response = self.client.get("/pytc/architectures") + + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertIn("architectures", payload) + self.assertIn("unet_3d", payload["architectures"]) + self.assertIn("unet_plus_3d", payload["architectures"]) + self.assertIn("swinunetr", payload["architectures"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_volume_io.py b/tests/test_volume_io.py new file mode 100644 index 00000000..9e6e36d4 --- /dev/null +++ b/tests/test_volume_io.py @@ -0,0 +1,72 @@ +import numpy as np +import pytest + +h5py = pytest.importorskip("h5py") +pytest.importorskip("tifffile") + +from server_api.workflows.volume_io import load_volume, parse_crop + + +def test_parse_crop_accepts_voxel_slice_strings(): + assert parse_crop("0:4,10:20,30:40") == ( + slice(0, 4, None), + slice(10, 20, None), + slice(30, 40, None), + ) + assert parse_crop("full") is None + + +def test_load_volume_reads_hdf5_dataset_with_crop(tmp_path): + path = tmp_path / "volume.h5" + volume = np.arange(4 * 5 * 6, dtype=np.uint16).reshape(4, 5, 6) + with h5py.File(path, "w") as handle: + handle.create_dataset("data", data=volume) + + loaded = load_volume(str(path), dataset_key="data", crop="1:3,2:5,1:4") + + np.testing.assert_array_equal(loaded, volume[1:3, 2:5, 1:4]) + + +def test_load_volume_reads_inline_hdf5_dataset_reference(tmp_path): + path = tmp_path / "volume.h5" + volume = np.arange(2 * 3 * 4, dtype=np.uint8).reshape(2, 3, 4) + with h5py.File(path, "w") as handle: + handle.create_dataset("main", data=volume) + + loaded = load_volume(f"{path}::main") + + np.testing.assert_array_equal(loaded, volume) + + +def test_load_volume_selects_channel_before_crop_for_hdf5(tmp_path): + path = tmp_path / "prediction.h5" + volume = np.arange(2 * 4 * 5 * 6, dtype=np.float32).reshape(2, 4, 5, 6) + with h5py.File(path, "w") as handle: + handle.create_dataset("vol0", data=volume) + + loaded = load_volume( + str(path), + dataset_key="vol0", + crop="1:3,2:5,1:4", + channel=1, + reference_ndim=3, + label="prediction", + ) + + np.testing.assert_array_equal(loaded, volume[1, 1:3, 2:5, 1:4]) + + +def test_load_volume_rejects_channel_for_already_3d_volume(tmp_path): + path = tmp_path / "mask.h5" + volume = np.arange(4 * 5 * 6, dtype=np.uint16).reshape(4, 5, 6) + with h5py.File(path, "w") as handle: + handle.create_dataset("data", data=volume) + + with pytest.raises(ValueError, match="already 3D"): + load_volume( + str(path), + dataset_key="data", + channel=0, + reference_ndim=3, + label="mask", + ) From f51d31e595b24f952d9d263caf95a3146662bcf2 Mon Sep 17 00:00:00 2001 From: Adam Gohain Date: Sun, 26 Apr 2026 09:12:46 -0400 Subject: [PATCH 04/38] feat: make assistant actions persistent and runnable --- .../src/__tests__/assistantActionCard.test.js | 40 + .../__tests__/assistantCommandCard.test.js | 44 + client/src/api.js | 256 ++++- client/src/components/Chatbot.js | 499 ++++++++- client/src/components/Chatbot.test.js | 228 +++++ client/src/components/WorkflowTimeline.js | 158 ++- .../src/components/WorkflowTimeline.test.js | 1 + .../src/components/YamlConfigHelpChat.test.js | 188 ++++ .../src/components/chat/AgentProposalCard.js | 20 +- .../components/chat/AssistantActionCard.js | 88 ++ .../components/chat/AssistantCommandCard.js | 118 +++ .../workflow/WorkflowEvidencePanel.js | 968 ++++++++++++++++++ .../workflow/WorkflowEvidencePanel.test.js | 315 ++++++ client/src/contexts/WorkflowContext.js | 313 +++++- client/src/contexts/WorkflowContext.test.js | 242 +++++ client/src/design/workflowDesignSystem.js | 103 ++ client/src/logging/appEventLog.js | 160 +++ client/src/logging/configLogSummary.js | 161 +++ server_api/auth/models.py | 34 +- server_api/chatbot/chatbot.py | 131 ++- .../chatbot/file_summaries/AgentRole.md | 20 + server_api/chatbot/rag_eval.py | 4 +- server_api/chatbot/update_faiss.py | 51 +- tests/test_chatbot_faiss_generation.py | 71 ++ tests/test_inline_helper_safe_mode.py | 30 + 25 files changed, 4052 insertions(+), 191 deletions(-) create mode 100644 client/src/__tests__/assistantActionCard.test.js create mode 100644 client/src/__tests__/assistantCommandCard.test.js create mode 100644 client/src/components/Chatbot.test.js create mode 100644 client/src/components/YamlConfigHelpChat.test.js create mode 100644 client/src/components/chat/AssistantActionCard.js create mode 100644 client/src/components/chat/AssistantCommandCard.js create mode 100644 client/src/components/workflow/WorkflowEvidencePanel.js create mode 100644 client/src/components/workflow/WorkflowEvidencePanel.test.js create mode 100644 client/src/design/workflowDesignSystem.js create mode 100644 client/src/logging/appEventLog.js create mode 100644 client/src/logging/configLogSummary.js create mode 100644 server_api/chatbot/file_summaries/AgentRole.md create mode 100644 tests/test_inline_helper_safe_mode.py diff --git a/client/src/__tests__/assistantActionCard.test.js b/client/src/__tests__/assistantActionCard.test.js new file mode 100644 index 00000000..5d4bada4 --- /dev/null +++ b/client/src/__tests__/assistantActionCard.test.js @@ -0,0 +1,40 @@ +import React from "react"; +import { fireEvent, render, screen } from "@testing-library/react"; + +import AssistantActionCard from "../components/chat/AssistantActionCard"; + +jest.mock("antd", () => ({ + Button: ({ children, ...props }) => ( + + ), + Space: ({ children }) =>
{children}
, + Tag: ({ children }) => {children}, + Typography: { + Text: ({ children }) => {children}, + }, +})); + +describe("AssistantActionCard", () => { + it("renders the action metadata and triggers execution", () => { + const onRun = jest.fn(); + const action = { + id: "open-training", + label: "Open Training", + description: "Jump to training with staged labels.", + variant: "primary", + client_effects: { + navigate_to: "training", + }, + }; + + render(); + + expect(screen.getByText("Open Training")).toBeTruthy(); + expect(screen.getByText("Jump to training with staged labels.")).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Run in app" })); + expect(onRun).toHaveBeenCalledWith(action); + }); +}); diff --git a/client/src/__tests__/assistantCommandCard.test.js b/client/src/__tests__/assistantCommandCard.test.js new file mode 100644 index 00000000..d52747f9 --- /dev/null +++ b/client/src/__tests__/assistantCommandCard.test.js @@ -0,0 +1,44 @@ +import React from "react"; +import { fireEvent, render, screen } from "@testing-library/react"; + +import AssistantCommandCard from "../components/chat/AssistantCommandCard"; + +jest.mock("antd", () => ({ + Button: ({ children, ...props }) => ( + + ), + Space: ({ children }) =>
{children}
, + Tag: ({ children }) => {children}, + Typography: { + Text: ({ children }) => {children}, + }, +})); + +describe("AssistantCommandCard", () => { + it("renders a terminal-style command block and runs it", () => { + const onRun = jest.fn(); + const command = { + id: "prime-training", + title: "Prime the training screen", + description: "Move the UI into training setup mode.", + command: 'app open training\napp training labels set "/tmp/corrected.tif"', + run_label: "Execute", + client_effects: { + navigate_to: "training", + set_training_label_path: "/tmp/corrected.tif", + }, + }; + + render(); + + expect(screen.getByText("Prime the training screen")).toBeTruthy(); + expect(screen.queryByText(/app open training/)).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: "Route" })); + expect(screen.getByText(/app open training/)).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Execute" })); + expect(onRun).toHaveBeenCalledWith(command); + }); +}); diff --git a/client/src/api.js b/client/src/api.js index c90e416a..f92c6410 100644 --- a/client/src/api.js +++ b/client/src/api.js @@ -1,6 +1,10 @@ import axios from "axios"; -import { message } from "antd"; import yaml from "js-yaml"; +import { logClientEvent } from "./logging/appEventLog"; +import { + detectConfigDiagnostics, + summarizeConfigText, +} from "./logging/configLogSummary"; import { setInferenceExecutionDefaults, setInferenceOutputPath, @@ -15,6 +19,102 @@ export const apiClient = axios.create({ withCredentials: true, }); +const summarizePayload = (payload) => { + if (!payload) return { hasBody: false }; + if (typeof payload === "string") { + return { hasBody: true, bodyLength: payload.length }; + } + if (payload instanceof FormData) { + return { hasBody: true, bodyType: "FormData" }; + } + if (typeof payload === "object") { + return { + hasBody: true, + bodyType: Array.isArray(payload) ? "array" : "object", + keys: Object.keys(payload).slice(0, 20), + }; + } + return { hasBody: true, bodyType: typeof payload }; +}; + +const attachApiLogging = (instance, source) => { + instance.interceptors.request.use( + (config) => { + config.metadata = { + startedAt: + typeof performance !== "undefined" ? performance.now() : Date.now(), + }; + logClientEvent("api_request", { + level: "INFO", + message: `${String(config.method || "get").toUpperCase()} ${config.url}`, + source, + data: { + method: String(config.method || "get").toUpperCase(), + url: config.url, + baseURL: config.baseURL, + ...summarizePayload(config.data), + }, + }); + return config; + }, + (error) => { + logClientEvent("api_request_setup_failed", { + level: "ERROR", + message: error.message || "Axios request setup failed", + source, + data: { error: error.message }, + }); + return Promise.reject(error); + }, + ); + + instance.interceptors.response.use( + (response) => { + const startedAt = response.config?.metadata?.startedAt; + const endedAt = + typeof performance !== "undefined" ? performance.now() : Date.now(); + logClientEvent("api_response", { + level: "INFO", + message: `${String(response.config?.method || "get").toUpperCase()} ${response.config?.url} -> ${response.status}`, + source, + data: { + method: String(response.config?.method || "get").toUpperCase(), + url: response.config?.url, + status: response.status, + latencyMs: + startedAt !== undefined ? Number((endedAt - startedAt).toFixed(2)) : null, + }, + }); + return response; + }, + (error) => { + const config = error.config || {}; + const startedAt = config.metadata?.startedAt; + const endedAt = + typeof performance !== "undefined" ? performance.now() : Date.now(); + logClientEvent("api_response_error", { + level: "ERROR", + message: + error.message || + `${String(config.method || "get").toUpperCase()} ${config.url} failed`, + source, + data: { + method: String(config.method || "get").toUpperCase(), + url: config.url, + status: error.response?.status, + latencyMs: + startedAt !== undefined ? Number((endedAt - startedAt).toFixed(2)) : null, + detail: error.response?.data?.detail || null, + }, + }); + return Promise.reject(error); + }, + ); +}; + +attachApiLogging(apiClient, "apiClient"); +attachApiLogging(axios, "axios"); + const buildFilePath = (file) => { if (!file) return ""; if (typeof file === "string") return file; @@ -35,6 +135,9 @@ const getErrorDetailMessage = (detail) => { return detail.map(getErrorDetailMessage).filter(Boolean).join("; "); } if (typeof detail === "object") { + if (detail.user_message) { + return getErrorDetailMessage(detail.user_message); + } const nestedUpstream = detail.upstream_body !== undefined ? getErrorDetailMessage(detail.upstream_body) @@ -86,9 +189,7 @@ export async function getNeuroglancerViewer(image, label, scales, workflowId = n const res = await axios.post(url, data); return res.data; } catch (error) { - message.error( - 'Invalid Data Path(s). Be sure to include all "/" and that data path is correct.', - ); + handleError(error); } } @@ -171,6 +272,18 @@ export async function startModelTraining( "[API] Failed to parse/modify YAML, using original config:", e, ); + logClientEvent("training_config_transform_failed", { + level: "WARNING", + message: "Failed to transform training config before request", + source: "api", + data: { + error: e.message || "unknown error", + configOriginPath, + outputPath, + logPath, + workflowId, + }, + }); } } else { console.warn( @@ -190,6 +303,23 @@ export async function startModelTraining( console.log("[API] Note: TensorBoard will monitor outputPath, not logPath"); console.log("[API] ========================================="); + const configSummary = summarizeConfigText(configToSend, "training"); + const diagnostics = detectConfigDiagnostics(configSummary); + logClientEvent("training_api_payload_prepared", { + level: diagnostics.length ? "WARNING" : "INFO", + message: "Training API payload prepared", + source: "api", + data: { + workflowId, + configOriginPath, + outputPath, + logPath, + requestBytes: data.length, + configSummary, + diagnostics, + }, + }); + return makeApiRequest("start_model_training", "post", data); } catch (error) { handleError(error); @@ -226,6 +356,10 @@ export async function getTensorboardURL() { return makeApiRequest("get_tensorboard_url", "get"); } +export async function getTensorboardStatus() { + return makeApiRequest("get_tensorboard_status", "get"); +} + export async function startTensorboard(logPath) { const query = logPath ? `?${new URLSearchParams({ logPath }).toString()}` @@ -284,6 +418,18 @@ export async function startModelInference( console.error("[API] Error type:", e.constructor.name); console.error("[API] Error message:", e.message); console.warn("[API] Falling back to original config"); + logClientEvent("inference_config_transform_failed", { + level: "WARNING", + message: "Failed to transform inference config before request", + source: "api", + data: { + error: e.message || "unknown error", + configOriginPath, + outputPath, + checkpointPath, + workflowId, + }, + }); configToSend = inferenceConfig; } } else { @@ -321,6 +467,23 @@ export async function startModelInference( data.substring(0, 300), ); + const configSummary = summarizeConfigText(configToSend, "inference"); + const diagnostics = detectConfigDiagnostics(configSummary); + logClientEvent("inference_api_payload_prepared", { + level: diagnostics.length ? "WARNING" : "INFO", + message: "Inference API payload prepared", + source: "api", + data: { + workflowId, + configOriginPath, + outputPath, + checkpointPath, + requestBytes: data.length, + configSummary, + diagnostics, + }, + }); + console.log("[API] Calling makeApiRequest..."); console.log("[API] Target endpoint: start_model_inference"); console.log("[API] Method: POST"); @@ -361,6 +524,18 @@ export async function getInferenceLogs() { } } +export async function syncWorkflowInferenceRuntime(workflowId, body = {}) { + try { + const res = await apiClient.post( + `/api/workflows/${workflowId}/sync-inference-runtime`, + body, + ); + return res.data; + } catch (error) { + handleError(error); + } +} + export async function stopModelInference() { try { await axios.post(`${BASE_URL}/stop_model_inference`); @@ -527,6 +702,76 @@ export async function getWorkflowMetrics(workflowId) { } } +export async function getWorkflowAgentRecommendation(workflowId) { + try { + const res = await apiClient.get( + `/api/workflows/${workflowId}/agent/recommendation`, + ); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function listWorkflowArtifacts(workflowId) { + try { + const res = await apiClient.get(`/api/workflows/${workflowId}/artifacts`); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function listWorkflowModelRuns(workflowId) { + try { + const res = await apiClient.get(`/api/workflows/${workflowId}/model-runs`); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function listWorkflowModelVersions(workflowId) { + try { + const res = await apiClient.get(`/api/workflows/${workflowId}/model-versions`); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function listWorkflowCorrectionSets(workflowId) { + try { + const res = await apiClient.get(`/api/workflows/${workflowId}/correction-sets`); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function listWorkflowEvaluationResults(workflowId) { + try { + const res = await apiClient.get( + `/api/workflows/${workflowId}/evaluation-results`, + ); + return res.data; + } catch (error) { + handleError(error); + } +} + +export async function computeWorkflowEvaluationResult(workflowId, body) { + try { + const res = await apiClient.post( + `/api/workflows/${workflowId}/evaluation-results/compute`, + body, + ); + return res.data; + } catch (error) { + handleError(error); + } +} + export async function exportWorkflowBundle(workflowId) { try { const res = await apiClient.post(`/api/workflows/${workflowId}/export-bundle`); @@ -579,10 +824,11 @@ export async function rejectAgentAction(workflowId, eventId) { } } -export async function queryWorkflowAgent(workflowId, query) { +export async function queryWorkflowAgent(workflowId, query, conversationId = null) { try { const res = await apiClient.post(`/api/workflows/${workflowId}/agent/query`, { query, + conversation_id: conversationId, }); return res.data; } catch (error) { diff --git a/client/src/components/Chatbot.js b/client/src/components/Chatbot.js index 38de04e0..4b87259a 100644 --- a/client/src/components/Chatbot.js +++ b/client/src/components/Chatbot.js @@ -13,6 +13,7 @@ import { SendOutlined, CloseOutlined, DeleteOutlined, + EditOutlined, PlusOutlined, MessageOutlined, MenuFoldOutlined, @@ -24,60 +25,121 @@ import { listConversations, getConversation, deleteConversation, + updateConversationTitle, } from "../api"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; import WorkflowTimeline from "./WorkflowTimeline"; import { useWorkflow } from "../contexts/WorkflowContext"; +import AgentProposalCard from "./chat/AgentProposalCard"; +import AssistantActionCard from "./chat/AssistantActionCard"; +import AssistantCommandCard from "./chat/AssistantCommandCard"; +import WorkflowEvidencePanel from "./workflow/WorkflowEvidencePanel"; +import { logClientEvent } from "../logging/appEventLog"; const { TextArea } = Input; const { Text } = Typography; +const MONO_FONT = + "'SFMono-Regular', Menlo, Monaco, Consolas, 'Liberation Mono', monospace"; const GREETING = { role: "assistant", content: - "Hello! I'm your AI assistant, built to help you navigate PyTC Client. How can I help you today?", + "Tell me the workflow job. I can run the model, start proofreading, use edits for training, compare results, or move screens.", +}; + +const WORKFLOW_SLASH_COMMANDS = { + "/status": "status", + "/next": "next step", + "/help": "what can the agent do", + "/infer": "run model", + "/inference": "run model", + "/segment": "run model to segment this volume", + "/proofread": "proofread this data", + "/train": "start training", + "/compare": "compare results and compute metrics", + "/metrics": "compare results and compute metrics", + "/export": "export evidence bundle", +}; + +const normalizeWorkflowAgentQuery = (query) => { + const trimmed = query.trim(); + if (!trimmed.startsWith("/")) { + return { agentQuery: query, commandAlias: null }; + } + const [command, ...rest] = trimmed.split(/\s+/); + const alias = WORKFLOW_SLASH_COMMANDS[command.toLowerCase()]; + if (!alias) { + return { agentQuery: query, commandAlias: null }; + } + const suffix = rest.join(" ").trim(); + return { + agentQuery: suffix ? `${alias}: ${suffix}` : alias, + commandAlias: command.toLowerCase(), + }; }; /* ─── helper: truncate a string to `n` chars ─────────────────────────────── */ const truncate = (str, n = 50) => str.length > n ? str.slice(0, n).trimEnd() + "…" : str; -const WORKFLOW_QUERY_TERMS = [ - "workflow", - "next", - "stage", - "retrain", - "training", - "corrected", - "proofread", - "mask", - "inference", - "visualize", - "evaluate", +const PROMPT_LEAK_MARKERS = [ + "you are the supervisor agent", + "response style for biologists", + "routing — decide", + "routing - decide", + "critical rules:", + "sub-agents:", + "search_documentation", + "delegate_to_training_agent", + "delegate_to_inference_agent", ]; +const sanitizeLoadedMessage = (message) => { + if (message.role !== "assistant") return message; + const content = String(message.content || ""); + const lower = content.toLowerCase(); + const leaked = PROMPT_LEAK_MARKERS.some((marker) => lower.includes(marker)); + if (!leaked) return message; + return { + ...message, + content: + "Hi. I can help run this segmentation loop. Ask me to run the model, proofread masks, use saved edits for training, or compare results.", + }; +}; + /* ═══════════════════════════════════════════════════════════════════════════ */ -function Chatbot({ onClose }) { +function Chatbot({ + onClose, + forceShowWorkflowInspector = false, + onWorkflowInspectorConsumed, +}) { /* ── state ─────────────────────────────────────────────────────────────── */ const [conversations, setConversations] = useState([]); const [activeConvoId, setActiveConvoId] = useState(null); const [messages, setMessages] = useState([GREETING]); const [inputValue, setInputValue] = useState(""); const [isSending, setIsSending] = useState(false); - const [sidebarOpen, setSidebarOpen] = useState(true); + const [sidebarOpen, setSidebarOpen] = useState(false); const [isLoadingConvo, setIsLoadingConvo] = useState(false); + const [showWorkflowInspector, setShowWorkflowInspector] = useState(false); + const [showWorkflowTimeline, setShowWorkflowTimeline] = useState(false); + const [editingTitleId, setEditingTitleId] = useState(null); + const [draftTitle, setDraftTitle] = useState(""); + const [isRenamingTitle, setIsRenamingTitle] = useState(false); const lastMessageRef = useRef(null); const workflowContext = useWorkflow(); + const runClientEffects = workflowContext?.runClientEffects; + const executeAssistantItem = workflowContext?.executeAssistantItem; + const workflow = workflowContext?.workflow; - const shouldUseWorkflowAgent = (query) => { + const shouldUseWorkflowAgent = () => { if (!workflowContext?.workflow?.id || !workflowContext?.queryAgent) { return false; } - const lower = query.toLowerCase(); - return WORKFLOW_QUERY_TERMS.some((term) => lower.includes(term)); + return true; }; /* ── scroll ────────────────────────────────────────────────────────────── */ @@ -94,6 +156,12 @@ function Chatbot({ onClose }) { scrollToBottom(); }, [messages, isSending, scrollToBottom]); + useEffect(() => { + if (!forceShowWorkflowInspector) return; + setShowWorkflowInspector(true); + onWorkflowInspectorConsumed?.(); + }, [forceShowWorkflowInspector, onWorkflowInspectorConsumed]); + /* ── load conversation list on mount ────────────────────────────────────── */ useEffect(() => { let cancelled = false; @@ -120,7 +188,16 @@ function Chatbot({ onClose }) { await clearChat(); // reset LangChain in-memory state setActiveConvoId(convo.id); const dbMessages = - convo.messages?.map((m) => ({ role: m.role, content: m.content })) ?? + convo.messages + ?.map((m) => ({ + role: m.role, + content: m.content, + source: m.source, + actions: m.actions || [], + commands: m.commands || [], + proposals: m.proposals || [], + })) + .map(sanitizeLoadedMessage) ?? []; setMessages([GREETING, ...dbMessages]); } finally { @@ -140,19 +217,62 @@ function Chatbot({ onClose }) { const handleSendMessage = async () => { if (!inputValue.trim() || isSending) return; const query = inputValue; + const isGreeting = + /^(hi|hello|hey|yo|sup|hiya)[\s!.?,]*$/i.test(query.trim()); setInputValue(""); setMessages((prev) => [...prev, { role: "user", content: query }]); setIsSending(true); try { if (shouldUseWorkflowAgent(query)) { - const data = await workflowContext.queryAgent(query); + const { agentQuery, commandAlias } = normalizeWorkflowAgentQuery(query); + logClientEvent("workflow_agent_chat_sent", { + source: "chatbot", + message: "Workflow-agent chat query sent", + data: { + workflowId: workflowContext.workflow.id, + conversationId: activeConvoId, + queryPreview: query.slice(0, 160), + agentQueryPreview: agentQuery.slice(0, 160), + commandAlias, + queryLength: query.length, + }, + }); + const data = await workflowContext.queryAgent(agentQuery, activeConvoId); const response = data?.response || "I could not inspect the workflow state for that request."; + const returnedConvoId = + data?.conversationId ?? data?.conversation_id ?? activeConvoId; setMessages((prev) => [ ...prev, - { role: "assistant", content: response }, + { + role: "assistant", + content: response, + source: data?.source || "workflow_orchestrator", + actions: isGreeting ? [] : data?.actions || [], + commands: isGreeting ? [] : data?.commands || [], + proposals: isGreeting ? [] : data?.proposals || [], + }, ]); + if (!activeConvoId && returnedConvoId) { + setActiveConvoId(returnedConvoId); + } + const convos = await listConversations(); + if (convos) setConversations(convos); + logClientEvent("workflow_agent_chat_completed", { + source: "chatbot", + message: "Workflow-agent chat query completed", + data: { + workflowId: workflowContext.workflow.id, + conversationId: returnedConvoId, + responseSource: data?.source || "workflow_orchestrator", + intent: data?.intent || null, + commandAlias, + actionCount: data?.actions?.length || 0, + commandCount: data?.commands?.length || 0, + proposalCount: data?.proposals?.length || 0, + }, + }); return; } @@ -175,6 +295,16 @@ function Chatbot({ onClose }) { const convos = await listConversations(); if (convos) setConversations(convos); } catch (e) { + logClientEvent("chat_send_failed", { + level: "ERROR", + source: "chatbot", + message: e.message || "Error contacting chatbot.", + data: { + workflowId: workflowContext?.workflow?.id || null, + activeConvoId, + queryPreview: query.slice(0, 160), + }, + }); setMessages((prev) => [ ...prev, { @@ -204,6 +334,97 @@ function Chatbot({ onClose }) { } }; + const startRenameConvo = (conversation, e) => { + e?.stopPropagation(); + setEditingTitleId(conversation.id); + setDraftTitle(conversation.title || "New Chat"); + }; + + const cancelRenameConvo = (e) => { + e?.stopPropagation(); + setEditingTitleId(null); + setDraftTitle(""); + }; + + const submitRenameConvo = async (conversation, e) => { + e?.stopPropagation(); + const nextTitle = draftTitle.trim(); + if (!nextTitle || nextTitle === conversation.title || isRenamingTitle) { + cancelRenameConvo(e); + return; + } + setIsRenamingTitle(true); + try { + const updated = await updateConversationTitle(conversation.id, nextTitle); + setConversations((prev) => + prev.map((item) => + item.id === conversation.id ? { ...item, ...updated } : item, + ), + ); + logClientEvent("chat_conversation_renamed", { + source: "chatbot", + message: "Chat conversation renamed", + data: { + conversationId: conversation.id, + titleLength: nextTitle.length, + }, + }); + } catch (error) { + logClientEvent("chat_conversation_rename_failed", { + level: "ERROR", + source: "chatbot", + message: error.message || "Chat conversation rename failed", + data: { conversationId: conversation.id }, + }); + } finally { + setIsRenamingTitle(false); + setEditingTitleId(null); + setDraftTitle(""); + } + }; + + const handleRunAssistantItem = async (item) => { + const itemId = item?.id || item?.title || item?.label || "assistant-item"; + logClientEvent("assistant_item_run_started", { + source: "chatbot", + message: "Assistant in-app item run started", + data: { + workflowId: workflow?.id || null, + activeConvoId, + itemId, + itemLabel: item?.title || item?.label || null, + itemType: item?.command ? "command" : "action", + runtimeActionKind: item?.client_effects?.runtime_action?.kind || null, + }, + }); + try { + if (executeAssistantItem) { + await executeAssistantItem(item); + logClientEvent("assistant_item_run_completed", { + source: "chatbot", + message: "Assistant in-app item run completed", + data: { workflowId: workflow?.id || null, activeConvoId, itemId }, + }); + return; + } + if (!item?.client_effects || !runClientEffects) return; + await runClientEffects(item.client_effects); + logClientEvent("assistant_item_run_completed", { + source: "chatbot", + message: "Assistant in-app item run completed", + data: { workflowId: workflow?.id || null, activeConvoId, itemId }, + }); + } catch (error) { + logClientEvent("assistant_item_run_failed", { + level: "ERROR", + source: "chatbot", + message: error.message || "Assistant in-app item run failed", + data: { workflowId: workflow?.id || null, activeConvoId, itemId }, + }); + throw error; + } + }; + /* ── markdown renderers ────────────────────────────────────────────────── */ const mdComponents = { ul: ({ children }) => ( @@ -279,17 +500,23 @@ function Chatbot({ onClose }) { /* RENDER */ /* ═══════════════════════════════════════════════════════════════════════ */ return ( -
+
{/* ── Sidebar ─────────────────────────────────────────────────────── */} {sidebarOpen && (
{/* header */} @@ -311,6 +538,7 @@ function Chatbot({ onClose }) { size="small" icon={} onClick={handleNewChat} + aria-label="New chat" /> @@ -319,6 +547,7 @@ function Chatbot({ onClose }) { size="small" icon={} onClick={() => setSidebarOpen(false)} + aria-label="Collapse conversations" /> @@ -346,22 +575,21 @@ function Chatbot({ onClose }) { style={{ padding: "8px", margin: "2px 0", - borderRadius: 6, + borderRadius: 8, cursor: "pointer", display: "flex", alignItems: "center", gap: 8, - background: - c.id === activeConvoId ? "#e6f4ff" : "transparent", + background: c.id === activeConvoId ? "#f3f4f6" : "transparent", border: c.id === activeConvoId - ? "1px solid #91caff" + ? "1px solid #d1d5db" : "1px solid transparent", - transition: "background 0.15s", + transition: "background 0.15s ease, border-color 0.15s ease", }} onMouseEnter={(e) => { if (c.id !== activeConvoId) - e.currentTarget.style.background = "#f0f0f0"; + e.currentTarget.style.background = "#f5f5f5"; }} onMouseLeave={(e) => { if (c.id !== activeConvoId) @@ -371,12 +599,54 @@ function Chatbot({ onClose }) { - - {truncate(c.title)} - + {editingTitleId === c.id ? ( + e.stopPropagation()} + onChange={(e) => setDraftTitle(e.target.value)} + onPressEnter={(e) => submitRenameConvo(c, e)} + onKeyDown={(e) => { + if (e.key === "Escape") cancelRenameConvo(e); + }} + onBlur={(e) => submitRenameConvo(c, e)} + aria-label={`Rename chat ${c.title}`} + style={{ flex: 1, minWidth: 0 }} + /> + ) : ( + startRenameConvo(c, e)} + style={{ flex: 1, fontSize: 13, lineHeight: "18px" }} + title="Double-click to rename" + > + {truncate(c.title)} + + )} + {editingTitleId !== c.id && ( + + + )}
- + {showWorkflowInspector && ( +
+ +
+ +
+ {showWorkflowTimeline && ( + + )} +
+ )} {/* messages */} -
+
{isLoadingConvo ? (
@@ -474,17 +801,20 @@ function Chatbot({ onClose }) { ref={isLast ? lastMessageRef : null} style={{ border: "none", - padding: "8px 0", + padding: "10px 0", justifyContent: isUser ? "flex-end" : "flex-start", }} >
{isUser ? ( @@ -492,12 +822,67 @@ function Chatbot({ onClose }) { {message.content} ) : ( - - {message.content} - + + {message.content} + + {(message.actions?.length > 0 || + message.commands?.length > 0 || + message.proposals?.length > 0) && ( +
+ {message.actions?.map((action) => ( + + ))} + {message.commands?.map((command) => ( + + ))} + {message.proposals?.map((proposal) => ( + + workflowContext?.approveAgentAction?.( + proposal.id, + ) + } + onReject={() => + workflowContext?.rejectAgentAction?.( + proposal.id, + ) + } + /> + ))} +
+ )} + )}
@@ -509,13 +894,19 @@ function Chatbot({ onClose }) {
{/* input */} -
+
+ ), + Button: ({ children, ...props }) => ( + + ), + Input: { + TextArea: ({ autoSize, children, ...props }) => ( + + ), + }, + Space: ({ children }) =>
{children}
, + Tag: ({ children, ...props }) => {children}, + Typography: { + Text: ({ children, strong, type, ...props }) => {children}, + }, +})); + +describe("AgentProposalCard", () => { + it("lets users click fields or tap Edit details before approval", () => { + const onApprove = jest.fn(); + const proposal = { + id: 12, + approval_status: "pending", + type: "start_training_run", + config_preset: "/project/configs/base.yaml", + image_path: "/project/subsets/image", + label_path: "/project/subsets/seg", + output_path: "/project/outputs/original", + autopick_parameters: true, + }; + + render(); + + fireEvent.click(screen.getByRole("button", { name: "Edit details" })); + fireEvent.change(screen.getByLabelText("Edit Output"), { + target: { value: "/project/outputs/edited" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Approve with edits" })); + + expect(onApprove).toHaveBeenCalledWith(proposal, { + output_path: "/project/outputs/edited", + }); + + cleanup(); + render(); + fireEvent.click(screen.getByText("/project/outputs/original")); + fireEvent.change(screen.getByLabelText("Edit Output"), { + target: { value: "/project/outputs/edited-direct" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Approve with edits" })); + + expect(onApprove).toHaveBeenLastCalledWith(proposal, { + output_path: "/project/outputs/edited-direct", + }); + }); + + it("exposes client-effect paths as editable approval fields", () => { + const onApprove = jest.fn(); + const proposal = { + id: 13, + approval_status: "pending", + type: "run_client_effects", + item_label: "Run inference", + client_effects: { + set_inference_output_path: "/project/outputs/inference", + }, + }; + + render(); + + fireEvent.click(screen.getByRole("button", { name: "Edit details" })); + fireEvent.change(screen.getByLabelText("Edit Output"), { + target: { value: "/project/outputs/inference-edited" }, + }); + fireEvent.click(screen.getByRole("button", { name: "Approve with edits" })); + + expect(onApprove).toHaveBeenCalledWith(proposal, { + inference_output_path: "/project/outputs/inference-edited", + }); + }); + + it("disables actions after decision is recorded", () => { + render( + , + ); + + expect(screen.getByRole("button", { name: "Approve" }).disabled).toBe(true); + expect(screen.getByRole("button", { name: "Reject" }).disabled).toBe(true); + expect(screen.queryByRole("button", { name: "Edit details" })).toBeNull(); + }); + + it("keeps long labels and specialist agent badges visible", () => { + render( + , + ); + + expect( + screen.getByText("specialist_inference_validation_and_retraining_review_pipeline").className, + ).toContain("workflow-proposal-card__type-tag"); + expect(screen.getByText("Inference Specialist")).toBeTruthy(); + expect(screen.getByText("bar")).toBeTruthy(); + }); +}); diff --git a/client/src/components/chat/AgentVisuals.js b/client/src/components/chat/AgentVisuals.js new file mode 100644 index 00000000..173bd175 --- /dev/null +++ b/client/src/components/chat/AgentVisuals.js @@ -0,0 +1,174 @@ +import React from "react"; +import { Tag } from "antd"; +import { + BarChartOutlined, + BugOutlined, + ExperimentOutlined, + EyeOutlined, + FileDoneOutlined, + FolderOpenOutlined, + ProjectOutlined, + ThunderboltOutlined, +} from "@ant-design/icons"; + +const ICONS = { + bar_chart: BarChartOutlined, + bug: BugOutlined, + experiment: ExperimentOutlined, + eye: EyeOutlined, + file_done: FileDoneOutlined, + folder: FolderOpenOutlined, + project: ProjectOutlined, + thunderbolt: ThunderboltOutlined, +}; + +const ALLOWED_BORDER_STYLES = new Set([ + "solid", + "dotted", + "double", + "dashed", + "thick", + "dashdot", + "top", + "rail", +]); + +const normalizeAgentText = (value) => { + if (typeof value !== "string") return ""; + return value.trim(); +}; + +const normalizeAgentColor = (value) => { + const text = normalizeAgentText(value); + if (!text) return DEFAULT_AGENT_VISUAL.color; + return text; +}; + +const normalizeAgentIconKey = (value) => { + const text = normalizeAgentText(value); + return ICONS[text] ? text : DEFAULT_AGENT_VISUAL.icon_key; +}; + +const normalizeBorderStyle = (value) => { + const text = normalizeAgentText(value).toLowerCase(); + return ALLOWED_BORDER_STYLES.has(text) + ? text + : DEFAULT_AGENT_VISUAL.border_style; +}; + +export const DEFAULT_AGENT_VISUAL = { + label: "Project Manager", + shortLabel: "PM", + color: "#111827", + icon_key: "project", + border_style: "solid", +}; + +export function getAgentVisual(agent = {}) { + const explicitLabel = + normalizeAgentText(agent.agent_label) || normalizeAgentText(agent.label); + const shortLabel = + normalizeAgentText(agent.agent_short_label) || + normalizeAgentText(agent.short_label) || + normalizeAgentText(agent.shortLabel); + + return { + label: explicitLabel || DEFAULT_AGENT_VISUAL.label, + shortLabel: shortLabel || explicitLabel || DEFAULT_AGENT_VISUAL.shortLabel, + color: normalizeAgentColor(agent.agent_color || agent.color), + iconKey: + normalizeAgentIconKey(agent.agent_icon_key || agent.icon_key), + borderStyle: + normalizeBorderStyle(agent.agent_border_style || agent.border_style), + }; +} + +export function getAgentBorderStyles(borderStyle, color, width = 4) { + const accent = color || DEFAULT_AGENT_VISUAL.color; + const base = { + borderLeft: `${width}px solid ${accent}`, + }; + + switch (borderStyle) { + case "dotted": + return { borderLeft: `${width}px dotted ${accent}` }; + case "double": + return { borderLeft: `${Math.max(width + 1, 5)}px double ${accent}` }; + case "dashed": + return { borderLeft: `${width}px dashed ${accent}` }; + case "thick": + return { borderLeft: `${Math.max(width + 2, 6)}px solid ${accent}` }; + case "dashdot": + return { + borderLeft: `${width}px solid ${accent}`, + borderTop: `2px dashed ${accent}`, + }; + case "top": + return { + borderLeft: `${width}px solid ${accent}`, + borderTop: `3px solid ${accent}`, + }; + case "rail": + return { + borderLeft: `${width}px solid ${accent}`, + boxShadow: `inset ${width + 2}px 0 0 rgba(0, 166, 166, 0.14)`, + }; + default: + return base; + } +} + +function AgentIcon({ iconKey }) { + const Icon = ICONS[iconKey] || ICONS.project; + return