From c0994d07ae907d0903376ae33589a0e7d87920e6 Mon Sep 17 00:00:00 2001 From: James Kane Date: Sun, 23 Aug 2026 14:55:44 -0500 Subject: [PATCH 1/4] perf(app): project_report reads only the five artifact kinds it uses `AlignmentArtifacts::load` called `artifact::list_for_alignments`, which selects the payload of *every* artifact of every listed alignment. A `tree-genotype` row runs to megabytes, so opening one project tab pulled gigabytes of JSON to read five small kinds. `list_for_alignments_of_kind` already existed for exactly this; the caller was never converted. `documents/BACKLOG.md` has carried it under "Cross-cutting" since 2026-07-26. `load` now takes the `(kind, algorithm version)` pairs to read, and runs one narrow query for each. Five small queries cost less than one wide one, because the wide one selects every payload. The five pairs are `PROJECT_REPORT_KINDS`, next to the method that reads them. All six accessor call sites in `project_report` resolve to those five pairs, and two of the six sit on a continuation line, so `grep artifacts\.` alone does not find them. A kind that is absent from the const reads as `None` at all times, which the const documents. `list_for_alignments` now has no caller in the workspace. It stays, because a caller that does not know the kinds in advance still needs it, but its doc says which form to reach for and what this one costs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SequryftjcFkLnjQRMKCx3 --- crates/navigator-app/src/queries.rs | 36 ++++++++++++++++++++------ crates/navigator-store/src/artifact.rs | 6 +++++ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/navigator-app/src/queries.rs b/crates/navigator-app/src/queries.rs index 5a98a310..a8f4fbbb 100644 --- a/crates/navigator-app/src/queries.rs +++ b/crates/navigator-app/src/queries.rs @@ -11,7 +11,12 @@ use super::*; /// /// A project report reads five kinds of artifact for each alignment of each member. So the earlier /// form, one read for each cell, sent thousands of queries to open one tab. This type reads each -/// artifact in one `IN` query and stats each BAM file one time. +/// wanted kind in one `IN` query and stats each BAM file one time. +/// +/// The load asks for the kinds that the caller names, and for no other kind. A read of every kind +/// is not correct here. Some payloads are very large. A `tree-genotype` row runs to megabytes, and +/// a cohort of them is gigabytes of JSON that no report builder reads. One narrow query for each +/// kind costs less than one wide query, because the wide query selects every payload. /// /// The rule for an old result is the rule of [`App::load_analysis`]. A cached payload is absent when /// the `mtime:size` value of the source file changed after the calculation. @@ -25,13 +30,16 @@ struct AlignmentArtifacts { } impl AlignmentArtifacts { - async fn load(store: &Store, alignments: &[&Alignment]) -> Result { + /// `kinds` holds the `(kind, algorithm version)` pairs to read. A pair that the caller does not + /// name is absent from the result, and [`Self::raw`] gives `None` for it. + async fn load(store: &Store, alignments: &[&Alignment], kinds: &[(&str, &str)]) -> Result { let ids: Vec = alignments.iter().map(|a| a.id).collect(); - let by_key = artifact::list_for_alignments(store.pool(), &ids) - .await? - .into_iter() - .map(|a| ((a.alignment_id, a.kind.clone(), a.algorithm_version.clone()), a)) - .collect(); + let mut by_key = HashMap::new(); + for (kind, version) in kinds { + for a in artifact::list_for_alignments_of_kind(store.pool(), &ids, kind, version).await? { + by_key.insert((a.alignment_id, a.kind.clone(), a.algorithm_version.clone()), a); + } + } // One stat call for each alignment, from the row that the code already holds. The code // does not make one stat call for each artifact. let sigs = alignments @@ -72,6 +80,18 @@ impl AlignmentArtifacts { } } +/// Each `(kind, algorithm version)` pair that [`App::project_report`] reads. +/// +/// Keep this list and the body of that method together. A cell that reads a kind which is absent +/// here gets `None` at all times. The column then stays empty for each member of the project. +const PROJECT_REPORT_KINDS: &[(&str, &str)] = &[ + ("coverage", coverage::COVERAGE_VERSION), + ("sex", "1"), + ("read_metrics", "1"), + ("sv", "1"), + (ERROR_KIND, ERROR_VERSION), +]; + impl App { // ---- queries ----------------------------------------------------------- @@ -570,7 +590,7 @@ impl App { by_subject.entry(guid).or_default().push(aln); } let all_alignments: Vec<&Alignment> = by_subject.values().flatten().collect(); - let artifacts = AlignmentArtifacts::load(&self.store, &all_alignments).await?; + let artifacts = AlignmentArtifacts::load(&self.store, &all_alignments, PROJECT_REPORT_KINDS).await?; // The order is the same as the order in `haplogroup_consensus`. It is the vote of each // run, then the placed label, then a value that the user set. let terminals = self.haplogroup_terminals().await?; diff --git a/crates/navigator-store/src/artifact.rs b/crates/navigator-store/src/artifact.rs index 790955ff..b150bf0c 100644 --- a/crates/navigator-store/src/artifact.rs +++ b/crates/navigator-store/src/artifact.rs @@ -154,6 +154,12 @@ pub async fn list_kinds(pool: &SqlitePool, alignment_id: i64) -> Result Date: Sun, 23 Aug 2026 14:56:06 -0500 Subject: [PATCH 2/4] =?UTF-8?q?docs(analysis):=20TrioPhaser=20does=20not?= =?UTF-8?q?=20exist=20=E2=80=94=20fix=20two=20broken=20intra-doc=20links?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `phasing.rs` linked `[`TrioPhaser`]` twice, and no such type has ever been written. The module header listed it as one of two phasers that "can then go in", which is accurate, but the `Phaser` trait doc read as though trio phasing were available today: "a Mendelian trio phaser when a parent is available (see [`TrioPhaser`])". Both now say plainly that `ReferencePhaser` is the only implementation, and that the beam search stands in for the full PBWT. The trio phaser and the PBWT phaser are still the two things the trait seam exists for; the text no longer implies either one is written. `TrioPhaser` is gone from rustdoc's unresolved-link list. Nine unresolved links remain in this crate and are untouched: `resolve_chip` (x4), `0,1` (x2), `haplo::INDEL_DERIVED`, `crate::revert::collate`, and `AnalysisError::io`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SequryftjcFkLnjQRMKCx3 --- crates/navigator-analysis/src/phasing.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/navigator-analysis/src/phasing.rs b/crates/navigator-analysis/src/phasing.rs index 3aa90e1a..1bfbe4ec 100644 --- a/crates/navigator-analysis/src/phasing.rs +++ b/crates/navigator-analysis/src/phasing.rs @@ -17,8 +17,9 @@ //! beam width `B`, and the candidate count `M`. //! //! The [`Phaser`] trait holds the seam steady. Two other phasers can then go in, and no caller -//! changes. One is a Mendelian [`TrioPhaser`], for a workspace that holds a parent sample. The -//! other is a full PBWT phaser. +//! changes. One is a Mendelian trio phaser, for a workspace that holds a parent sample. The other +//! is a full PBWT phaser. **Nobody has written either one.** [`ReferencePhaser`] is the only +//! implementation today, and the beam search above is what stands in for the PBWT. use std::collections::HashMap; @@ -49,8 +50,8 @@ pub struct PhasedGenotypes { pub sites: Vec, } -/// The phasing strategy. Reference-based statistical phasing by default; a Mendelian trio phaser -/// when a parent is available (see [`TrioPhaser`]). +/// The phasing strategy. [`ReferencePhaser`] is the only implementation. A Mendelian trio phaser, +/// for a workspace that holds a parent sample, goes behind this same trait when someone writes it. pub trait Phaser { /// Phase the sample's genotypes into two consistent parental sides. fn phase(&self, genotypes: &[SiteGenotype]) -> PhasedGenotypes; From ec965c375019cb64e2fc74f1a297efa0f6faf95a Mon Sep 17 00:00:00 2001 From: James Kane Date: Sun, 23 Aug 2026 14:56:35 -0500 Subject: [PATCH 3/4] docs: correct five stale status headers, and give archaic Tier B its missing chapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An audit of all 32 docs in documents/design/ against the tree found five status headers wrong, three of them claiming *less* than the tree has. A header goes stale the moment a branch merges, because the merge updates code and not the header. ArchaicAncestry_Design.md "Tier B GATED OFF" -> ARCHAIC_SEGMENTS_ENABLED = true realignment-module.md "not yet merged" -> bf576ab, v0.1.0-alpha.17 project-block-tree.md "15 commits, not pushed" -> 4cb9eca (#45), v0.1.0-alpha.16 BACKLOG.md 1.1 Tier B gated off -> both tiers ship BACKLOG.md 1.7 "code signing deferred" -> macOS done 2026-08-15, Windows open BACKLOG.md 2.1 "Designed, not started" -> shipped; 2.2 is unblocked by it The archaic doc needed more than a header. PR #42 (906b9ee) carried the *Why it failed* diagnosis **and** a full Tier B rebuild — 909 lines of `archaic_match.rs`, ten validation scripts — but touched no design document at all, so the whole record has lived in a module doc-comment for three weeks. Section 11 is the pointer that was missing: what changed (matching the archaic genomes, not counting private-variant density), the held-out numbers (r = +0.710 against -0.018 for v1), and the cross-population limit that is the reason the report is worded as a within-population measure. Every figure is copied from the module doc-comment, which stays authoritative. BACKLOG 2.1 is compressed rather than deleted. Each measurement it carried — the 1,157 MAPQ deltas, the `SEQ: *` CRAM panic, the 4 h 44 m sort, the single-core `open_seq` — was checked to survive in realignment-module.md before the text came out. Three Scala-era documents get stale-warning headers in the style of the one already on IBD_Matching_Implementation_Plan.md, rather than a rewrite or a delete: Edge_Client_Implementation_Status.md (~55%, last touched 2025-12-08, six months before the Rust cutover), UI_Redesign_Proposal.md and UI_i18n_Guidelines.md (both December 2024). The i18n guidelines still hold as architecture; only their examples and paths are dead. Also noted, not fixed: several passages inside archaic_match.rs still say the module "stays gated", overtaken by b39db0b and 9fca4c1. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SequryftjcFkLnjQRMKCx3 --- documents/BACKLOG.md | 89 ++++++--------- .../Edge_Client_Implementation_Status.md | 15 ++- documents/UI_Redesign_Proposal.md | 13 ++- documents/UI_i18n_Guidelines.md | 11 +- documents/design/ArchaicAncestry_Design.md | 105 ++++++++++++++++-- documents/design/project-block-tree.md | 9 +- documents/design/realignment-module.md | 8 +- 7 files changed, 172 insertions(+), 78 deletions(-) diff --git a/documents/BACKLOG.md b/documents/BACKLOG.md index a6529a67..2c7fa865 100644 --- a/documents/BACKLOG.md +++ b/documents/BACKLOG.md @@ -23,10 +23,13 @@ Code exists or the design is settled; these are the near-term threads. ### 1.1 Archaic ancestry (Neanderthal / Denisovan) - **Design:** [`design/ArchaicAncestry_Design.md`](design/ArchaicAncestry_Design.md) -- **Status (corrected 2026-08-02):** **Tier A shipped** (`230353b`, `#34`) and reports a *count*, - never a % Neanderthal. **Tier B is built but gated OFF** (`#35`, `#40`) — the diagnosis is that it - measured the wrong observable, not that the HMM is broken; read `#41`/`#42` before reopening. The - "design draft, no code" status below was already stale when this file was written. +- **Status (corrected 2026-08-23):** **Both tiers shipped and ON.** Tier A (`230353b`, `#34`) + reports a *count*, never a % Neanderthal. **Tier B was rebuilt and re-enabled** in `#42` + (`906b9ee`, released in `v0.1.0-alpha.15`): `ARCHAIC_SEGMENTS_ENABLED = true`. The v1 density + caller was gated off in `#40` for measuring the wrong observable; v2 (`archaic_match`) matches the + archaic genomes directly and reaches r = +0.710 on held-out Europeans against −0.018 for v1. It is + a **within-population** measure only — our four archaic genomes under-represent East Asian archaic + diversity, so cross-ancestry comparison orders populations backwards. See design §11. - **Scope:** Phase 1 = compute our own marker panel (EVA archaic VCFs + Ensembl-75 ancestral alleles + 1kGP AFR outgroup) + Tier A `count_archaic_markers` + domain/store/UI card — the 23andMe equivalent, for chip *and* WGS, reusing the ancestry-panel machinery. Phase 2 = Tier B segment HMM @@ -96,72 +99,42 @@ Code exists or the design is settled; these are the near-term threads. - **Design:** [`design/packaging-and-release.md`](design/packaging-and-release.md) - **Status:** Shipping (all four installers build on a `v*` tag; assets fetched on demand from the GitHub asset release). -- **Scope:** code signing + notarization (Apple Developer ID $99/yr; a Windows cert) — deferred for - alpha with a documented Gatekeeper work-around; the Linux glibc-2.28 container CI is authored but - **has never been run**; `default_reference_sha` is still `None` for all four builds - (`navigator-refgenome/src/registry.rs:172`), awaiting confirmed publisher checksums. +- **Scope:** **macOS signing + notarization are DONE** (2026-08-15) — only **Windows** code signing + (Authenticode / Azure Trusted Signing) is still open, deliberately deferred for alpha with a + documented SmartScreen work-around; the Linux glibc-2.28 container CI is authored but **has never + been run**; `default_reference_sha` is still `None` for all four builds + (`navigator-refgenome/src/registry.rs:191`), awaiting confirmed publisher checksums. --- ## Tier 2 — Designed, not started -Verified 2026-07-26 to have no implementation in the tree. - -### 2.1 Realignment module — **in progress** (phase 1 landed 2026-08-08) -- **Design:** [`design/realignment-module.md`](design/realignment-module.md) — revised 2026-08-08 - after a phase 0 spike that **retracted the module's motivating premise** (ancestry is *not* - build-locked; off-build samples already estimate ancestry through the multi-build IBD panel) and - reversed the backend decision to pure-Rust `minimap2-pure-rs`. Read the correction blocks before - planning further work — whether the remaining payoff justifies the module is an open product - question. -- **Scope:** revert + realign GRCh37/38 vendor WGS to CHM13v2 / hs1; aligner-index cache in - `navigator-refgenome`, job orchestration + provenance, opt-in background job with warnings. -- **Done:** phase 0 spikes; **phase 1** (stage A, `navigator-analysis/src/revert/`) — primaries-only - revert with orientation restore and `OQ` preference, a disk-backed external merge sort that - collates by read name, synchronized paired-FASTQ output; **phase 2** (stage B, - `navigator-align`) — pure-Rust minimap2 backend, preset selection, RAM-sized part-by-part index - build and map with cross-part merge, single- and paired-end, BAM/CRAM output via noodles; - **phase 3** (stages C and D) — coordinate sort, short-read duplicate marking, CRAM + `.crai`, - and the provenance migration with registration in `navigator-app::realign`. - **Phase 4** — the cancellable job with preflight, the per-alignment and per-project cards, the - realigned badge, and selector preference — is built too. -- **Phase 5 in progress, and it found something.** Backend parity on 168k real WGS229 chrY reads: - 99.2% identical placements, but `minimap2-pure-rs` reports **systematically higher MAPQ** than C - minimap2 (1,157 up vs 47 down, median +10), concentrated on chrY at 15x the off-chrY rate, and - 111 reads per 168k cross the MQ>=20 callable threshold upward. That is the input to the - private-Y filter stack, so it is material. **Paired-end re-run against upstream 2.31 widened the - gap** (1.09% of records differ in MAPQ vs 0.72% single-end; 96.6% of differences are Rust - higher), and does not reproduce the upstream crate's own claim of exact `sr` PAF parity. - **But it does not reach the output**: run through the shipped stage C and de-novo caller over - the worst-case window, upstream produced 232 calls and the Rust backend 233 — every upstream - call reproduced, one extra at depth 3 that the `depth >= 4` callable gate removes. Decision 1 - re-settled on the pure-Rust backend with the divergence documented. -- **First WGS-scale run (2026-08-12): 10 h 41 m, and it did not finish.** WGS229's 17.3 GB CRAM - (615.6M reads) died in stage 7 of 8 on a CRAM-encoding panic — a secondary alignment carries - `SEQ: *`, which is legal SAM and what minimap2 emits, and CRAM cannot store a read it has no - bases for. Fixed: such non-primary records are dropped and counted, a primary of that shape - errors. Two other findings: **the sort is the most expensive stage** (4 h 44 m, 44% of wall - clock — more than mapping's 3 h 40 m), and **stage A runs on one core** because `open_seq` only - threads the BAM path, fixable by reverting per-contig in parallel. The run also proved - resumability is not optional — `JobScratch` discarded seven working stages on the way out, so - `NAVIGATOR_REALIGN_KEEP_SCRATCH=1` now inverts that for pipeline work. -- **Purpose:** Y-chromosome variant discovery. A private-Y call set is only usable on CHM13 — - the callable mask, non-PAR restriction, recurrent blocklist, and de-novo tree are all defined - there — and liftover cannot help with *discovery*, where the variant is not in any site list - yet. Autosomal fixed-site matching was never the motivation; two earlier revisions of the - design said otherwise and have been corrected. -- **Settled:** whole-genome is the only correct scope. A Y-only mode was proposed and withdrawn — - the reads that need realigning are the ones GRCh38 placed wrongly, so selecting them by source - coordinate requires the answer being computed. Revert reads the whole file anyway. +Verified 2026-08-23 to have no implementation in the tree, **except 2.1, which shipped** and is kept +here under its old number so that existing references still resolve. + +### 2.1 Realignment module — **SHIPPED** (no longer a backlog item) +- **Design:** [`design/realignment-module.md`](design/realignment-module.md) — the full record: + phase 0 spike, backend parity, the three whole-genome runs, and the phase 5 acceptance result. +- **Status:** merged to `main` as `bf576ab` (2026-08-14), released in **`v0.1.0-alpha.17`**. All + five phases are done: revert (stage A), pure-Rust minimap2 mapping (stage B), sort + duplicate + marking + CRAM (stages C/D), the cancellable job with preflight and provenance, and end-to-end + validation on WGS229 (private-Y 438 → 11, chrY breadth 41 % → 98 %). Simple-mode offer and the + Windows disk preflight came with it. +- **Purpose:** Y-chromosome variant discovery. A private-Y call set is only usable on CHM13, and + liftover cannot help with *discovery*, where the variant is in no site list yet. Two earlier + revisions of the design argued it from autosomal ancestry and were wrong; they are corrected in + place. +- **Eligibility lives in one place** — `realignable_for_subject`. Do not re-derive it at a call site. - **Do not confuse** with `navigator-analysis/src/realign.rs`, which is *indel local realignment* - (plan §4b) and is a different thing entirely. + (plan §4b) and a different thing entirely. +- **Unblocks 2.2**, which depended on this module. ### 2.2 Distributed compute grid - **Design:** [`design/distributed-compute-grid.md`](design/distributed-compute-grid.md) - **Scope:** a Seti@Home-style layer — the AppView publishes public-ENA work units, Navigator instances reserve a lease, fetch, realign to CHM13, run the analysis stack, submit signed results, and earn capped compute credit. Cross-repo (Navigator worker + AppView coordinator + shared wire - records). Depends on 2.1. + records). **Depended on 2.1, which has now shipped — this is unblocked for the first time.** ### 2.3 Academic / public-dataset (ENA) import - **Design:** [`design/academic-ena-import.md`](design/academic-ena-import.md) diff --git a/documents/Edge_Client_Implementation_Status.md b/documents/Edge_Client_Implementation_Status.md index 61010226..a7a5e2ef 100755 --- a/documents/Edge_Client_Implementation_Status.md +++ b/documents/Edge_Client_Implementation_Status.md @@ -1,8 +1,21 @@ # Edge Client Implementation Status +> **STALE — do not trust the numbers below (header added 2026-08-23).** This is a **Scala-era** +> document, last updated 2025-12-08, more than six months before the Rust cutover (`0dee32c`, +> 2026-06-19). Every percentage and every file path in it refers to code that no longer exists. +> +> Atmosphere lexicon alignment **completed** (phases A–D) during the rewrite, and the federation +> surface has moved well past what this file describes: `sync_outbox` with idempotent putRecord at +> TID and PULL reconcile, signed IBD attestations over an X3DH/AES-GCM exchange channel, feed posts, +> peer DMs, and recruitment invitations are all built and merged. +> +> For the current picture use [`BACKLOG.md`](BACKLOG.md) (the "Social layer — deferred slices" and +> "AppView-side" sections) and the per-topic agent memory. Kept only for the historical record of +> what the Scala edge client had reached. + Navigator Desktop implementation status against the Atmosphere Lexicon specification. -**Overall Completion: ~55%** +**Overall Completion: ~55%** *(as of 2025-12-08, Scala codebase — see the warning above)* Last updated: 2025-12-08 diff --git a/documents/UI_Redesign_Proposal.md b/documents/UI_Redesign_Proposal.md index 10b0d63f..7a1d101c 100755 --- a/documents/UI_Redesign_Proposal.md +++ b/documents/UI_Redesign_Proposal.md @@ -1,8 +1,19 @@ # UI Redesign Proposal: DUNavigator +> **STALE — Scala-era document (header added 2026-08-23).** Last touched December 2024, before the +> Rust cutover (`0dee32c`, 2026-06-19). "Phases 1-3 Complete" refers to the **ScalaFX** UI, which was +> deleted. The redesign it proposes was carried into the Rust app and **shipped**: the egui Workbench +> (dark theme, tabs, virtualized subjects table, cards) is live, and Simple mode (left rail + panels, +> deepest-past → present, with AI narration) went in on top of it. +> +> Read this for the *design intent* — the dashboard-centric, entity-focused direction it argues for +> is still the direction — and not for status, component names, or file paths. Current UI work is +> tracked in [`BACKLOG.md`](BACKLOG.md) and +> [`design/subject-brief-simple-mode.md`](design/subject-brief-simple-mode.md). + **Date:** December 2024 **Last Updated:** December 17, 2024 -**Status:** In Progress - Phases 1-3 Complete +**Status:** Superseded — the ScalaFX phases below were overtaken by the Rust rewrite (see above) **Target Users:** Genetic genealogists and scientists managing large subject collections --- diff --git a/documents/UI_i18n_Guidelines.md b/documents/UI_i18n_Guidelines.md index 38f68455..8ce5839f 100755 --- a/documents/UI_i18n_Guidelines.md +++ b/documents/UI_i18n_Guidelines.md @@ -1,7 +1,16 @@ # Internationalization (i18n) Guidelines +> **Partly stale — Scala-era document (header added 2026-08-23).** Written December 2024, before +> the Rust cutover (`0dee32c`, 2026-06-19). The **architecture and the guidelines still hold**; the +> code examples and file paths do not. +> +> As built in Rust: lookup is `self.tr("key")`, catalogues are plain text at +> `crates/navigator-domain/locales/{en,es}.txt`, and a parity test (`every_es_key_exists_in_en`) +> gates them. en and es are at key parity. The known tail is that transient `self.status` strings +> and `format!` dynamics are still English — see [`BACKLOG.md`](BACKLOG.md), "Cross-cutting". + **Date:** December 2024 -**Status:** Draft Specification +**Status:** Adopted — implemented in Rust; see the note above for what changed **Related:** UI_Redesign_Proposal.md --- diff --git a/documents/design/ArchaicAncestry_Design.md b/documents/design/ArchaicAncestry_Design.md index 851c2482..da180d6c 100644 --- a/documents/design/ArchaicAncestry_Design.md +++ b/documents/design/ArchaicAncestry_Design.md @@ -1,15 +1,24 @@ # Archaic Ancestry Report (Neanderthal / Denisovan) — Design -**Status:** **Tier A SHIPPED** (`v0.1.0-alpha.14`). **Tier B GATED OFF** (`ARCHAIC_SEGMENTS_ENABLED -= false`) after per-individual validation showed the segment caller carries no per-person signal, and -diagnosed as **built on the wrong observable** — §3's choice of a method designed for people who do -*not* have archaic reference genomes, which we do. It shipped enabled in alpha.14 and was withdrawn -in the next release. Drafted 2026-07-23; plan added 2026-07-26; all three §9 questions resolved. - -> **Read *Tier B validation* and *Why it failed* (both at the end of §10) before anything else in -> this document about Tier B.** §3's method choice, §5's Tier B pipeline and M3's calibration are all -> superseded by that diagnosis. *Deviations from the plan* also qualifies §7's expected percentage -> and M3's feature-gate rule. Tier A (§5 Tier A, M1, M2) stands unaffected. +**Status (corrected 2026-08-23):** **Tier A SHIPPED** (`v0.1.0-alpha.14`). **Tier B is REBUILT and +ON again** — `ARCHAIC_SEGMENTS_ENABLED = true`, first released in **`v0.1.0-alpha.15`**. Drafted +2026-07-23; plan added 2026-07-26; all three §9 questions resolved. + +Tier B has a three-step history, and the body of this document records only the first two: + +1. It shipped enabled in alpha.14 on the strength of one aggregate number. +2. It was **withdrawn** (`#40`) when per-individual validation showed the segment caller carries no + per-person signal, and the failure was diagnosed as the **wrong observable** — §3 chose a method + designed for people who do *not* have archaic reference genomes, and we have all four (`#41`). +3. It was **rebuilt against the right observable and turned back on** (`#42`, `906b9ee`). The new + caller matches the archaic genomes directly instead of counting private-variant density. See + [§11](#11-tier-b-v2--the-rebuild-that-turned-it-back-on-2026-08-01) below. + +> **Read *Tier B validation* (§10), *Why it failed* (§10) and then §11, in that order, before +> anything else in this document about Tier B.** §3's method choice, §5's Tier B pipeline and M3's +> calibration are all superseded — first by the diagnosis, then by the §11 rebuild. *Deviations from +> the plan* also qualifies §7's expected percentage and M3's feature-gate rule. Tier A (§5 Tier A, +> M1, M2) stands unaffected throughout. **Goal:** Reconstruct a 23andMe-style Neanderthal report — and go beyond it with a Denisovan estimate and a true whole-genome introgression map — from public archaic reference genomes and recent methods, using the app's existing ancestry/panel/HMM machinery. @@ -1156,3 +1165,79 @@ The examples these numbers came from, all under `crates/navigator-analysis/examp `archaic_private_dump` (the HMM's actual input, with quality columns), `archaic_outgroup_density` (the rate-map proxy), `archaic_classify_dump` (diagnostic sites), `archaic_callable_dump` (what the caller can see at all), and `cram_query_probe` (the CRAM defect found on the way here). + +--- + +## 11. Tier B v2 — the rebuild that turned it back on (2026-08-01) + +**Section added 2026-08-23.** PR #42 (`906b9ee`) carried both the *Why it failed* diagnosis above +**and** a full rebuild of Tier B, but it touched no design document — it landed 909 lines of +`crates/navigator-analysis/src/archaic_match.rs`, ten validation scripts under +`scripts/archaic-validation/`, and the flag flip, with the record written into the module +doc-comment instead. This section is the pointer that was missing for three weeks. **The +authoritative, complete record is the module doc-comment at the top of `archaic_match.rs`** — every +table below is copied from it, and it holds more. + +### What changed + +`archaic_segments` (Skov 2018 / hmmix) removes the variants Africans also carry and looks for a +region dense in what stays. That method exists for a person who does **not** have archaic reference +genomes. We have all four, and Tier A already ships 2,031,406 sites where the archaics carry a +derived allele. + +`archaic_match` asks the other question: does this stretch **match** an archaic genome? It is a +two-state HMM whose observation is one bit at each diagnostic site — the subject carries the archaic +allele, or does not — with Bernoulli emissions and transitions that scale with recombination. **It +indexes over sites, not over base pairs**, so the uneven density of the diagnostic sites cancels and +the rate map that the density model needed disappears from the problem. + +The evidence per tract is what decides this, and the two observables differ 30-fold at the same ~3x +contrast: + +| observable | evidence in a 36 kb tract | sensitivity at 5 % false positives | +|---|---|---| +| private-variant density (v1) | ~1 variant | 14.3 % | +| archaic-allele matching (v2) | ~30 sites | 95.1 % | + +### The result, on held-out individuals + +60 Europeans on chr21+22, split by fixed seed into 30 train / 30 test. Every figure is from the half +the fit never saw: + +| | density caller (v1) | v2 uncalibrated | v2 calibrated | +|---|---|---|---| +| base-level F1 | n/a | 27.9 % | **34.5 %** | +| precision | 1.5 % | 20.2 % | **34.9 %** | +| extent ratio ours/theirs | 1.45 | 2.23 | **0.98** | +| extent `r` over individuals | −0.018 (p = 0.94) | +0.520 | **+0.710 (p < 0.0001)** | + +Genome-wide on three Europeans — the configuration that ships — sensitivity is 40–43 % and precision +about 46 %, both *better* than the two-chromosome figures, and all three sit above the entire +random-placement null. A concordance filter over the called segments takes precision from **54 % to +90 %**, and validates on a genome held out of it: kept segments score 74.9 % Denisova concordance +against 21.5 % for dropped ones. + +### The limit that shipped with it, and why the report is worded as it is + +Parameters frozen at the European fit, run on 30 East Asians: 30/30 above their own null, identical +31.6 % sensitivity, *better* 41.9 % precision. **Detection transfers.** The reported extent does +not. Truth puts East Asian archaic extent at 1.217x Europe; this caller reports 0.937x — the wrong +order. + +The cause is not tunable. Our four sequenced archaic genomes under-represent the archaic diversity +of East Asia: East Asian tracts match them at 83.4 % against 89.2 % for European tracts, and +Denisova is the best match for 32.2 % of East Asian tracts against 11.2 % of European ones. Recovery +is then 46 % of European truth against 38 % of East Asian. Fixing it needs archaic genomes nearer to +the populations that introgressed into East Asia, and those do not exist. + +So Tier B ships as a **within-population** measure (`b39db0b`), and the UI states that limit rather +than implying a universal percentage. **You must not use it to compare people of different +ancestries.** The Tier A rule is unchanged and separate: Tier A reports a **count**, never a percent. + +### Known stale prose in the module + +`archaic_match.rs` was written across 13 commits, and several passages in its doc-comment still say +the module "stays gated" or that a finding is "not enough to turn the module on". Those were true +when written and were overtaken by `b39db0b` (ship as within-population) and `9fca4c1` (genome-wide +validation passes). The flag is the truth: `ARCHAIC_SEGMENTS_ENABLED = true` in +`navigator-app/src/lib.rs`, whose comment is current. diff --git a/documents/design/project-block-tree.md b/documents/design/project-block-tree.md index 6307db4d..daff43bc 100644 --- a/documents/design/project-block-tree.md +++ b/documents/design/project-block-tree.md @@ -1,9 +1,10 @@ # Project Y block tree — design -**Status:** **All three phases shipped** (branch `feat/project-block-tree`, 15 commits, not pushed) — -the aggregate + collapse (1), the `ProjectTab::Tree` canvas (2), and private-variant blocks + -candidate branches + export (3), plus four things phase 3 turned out to need: the -`private-y --project` batch, a **VCF-backed private-Y engine** for subjects with no alignment, a +**Status:** **All three phases shipped and MERGED** — squashed to `main` as `4cb9eca` (PR #45, +2026-08-06), first released in **`v0.1.0-alpha.16`**; the branch was `feat/project-block-tree`. +The three phases are the aggregate + collapse (1), the `ProjectTab::Tree` canvas (2), and +private-variant blocks + candidate branches + export (3), plus four things phase 3 turned out to +need: the `private-y --project` batch, a **VCF-backed private-Y engine** for subjects with no alignment, a **candidate review surface**, and an artefact-filter stack calibrated against R1b-CTS4466Plus. Live state there: 248/255 placed members carry private-Y (was 1 workspace-wide), **7 candidate branches** surviving the filters. Suite 797 passed. The canvas was then **redrawn to Alex diff --git a/documents/design/realignment-module.md b/documents/design/realignment-module.md index 48b0b1eb..f5194cd5 100644 --- a/documents/design/realignment-module.md +++ b/documents/design/realignment-module.md @@ -1,11 +1,13 @@ # Realignment module — design & options -Status: **built and validated — phases 1–5 complete (2026-08-14).** Branch: `worktree-realignment`, -not yet merged. The measurements from the phase 0 spike (2026-08-08), the backend parity work +Status: **MERGED and SHIPPED** — phases 1–5 complete (2026-08-14), merged to `main` as `bf576ab` +(2026-08-14), first released in **`v0.1.0-alpha.17`**. The branch was `worktree-realignment`; a +follow-up simplification pass ran on `chore/realign-simplify-pass`. The measurements from the +phase 0 spike (2026-08-08), the backend parity work (2026-08-10), the first whole-genome run (2026-08-12), the second (2026-08-13) and the third — which passed every acceptance criterion — are folded in below. See [Phase 5 result](#phase-5-result--wgs229-end-to-end-2026-08-14). -Scope if built: `navigator-analysis` (revert + post-process), a new `navigator-align` crate +Scope as built: `navigator-analysis` (revert + post-process), a new `navigator-align` crate (the mapper — see Decision 1), `navigator-refgenome` (aligner-index cache), `navigator-app` (job orchestration + provenance), `navigator-store` (alignment provenance migration), `navigator-ui` (opt-in background job + warnings). From 9e0e633f1cc7cf465b37ee22dc13b732b5b7b407 Mon Sep 17 00:00:00 2001 From: James Kane Date: Sun, 23 Aug 2026 23:07:37 -0500 Subject: [PATCH 4/4] docs(analysis): archaic_match no longer ends on a verdict two commits overtook The module doc-comment is chronological, and it also *ended* chronologically. Its last word was "**This is still not enough to turn the module on**", written while the module was off. `b39db0b` turned it on as a within-population measure and `9fca4c1` answered the cohort objection, and neither touched the prose. A reader who reached the end of the file came away believing Tier B does not ship. Two stale verdicts, both replaced with what happened: - "the one reason that this module stays gated", on the population-ordering result. That ordering is real and unchanged. It is the reason the report is within-population and the reason the UI carries the caveat, not a reason the module is off. - The closing three-reasons paragraph. Two of the three now have an answer, so the section says which. *The cohort was chr21+22 alone* is answered by the genome-wide run already written up earlier in the same doc-comment: 40-43 % sensitivity and ~46 % precision on all 22 autosomes, better than the two-chromosome figures. *Precision was 34.9 % without the filter* is answered by the shipped path applying the filter, which measures 90 %. *The reference callset has weak support* stands, and is marked as standing. A new closing section carries the outcome: the module is on, it shipped in v0.1.0-alpha.15, the measure is within-population and must not be compared across ancestries, and Tier A's count rule is untouched. It also says plainly that the sections above stop before the decision, so the next reader knows the body is a work record rather than a current-state description. Nothing measured is edited or removed. Every table, figure and caveat stands as written. `filter_by_concordance`, `MIN_CONCORDANCE` and `call_from_observations` are now intra-doc links and resolve; the crate's nine pre-existing unresolved links are unchanged. ARCHAIC_SEGMENTS_ENABLED is referenced as plain code, since navigator-analysis sits below navigator-app and cannot link to it. Design section 11 pointed at this stale prose; it now describes how the doc-comment reads. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01SequryftjcFkLnjQRMKCx3 --- .../navigator-analysis/src/archaic_match.rs | 39 ++++++++++++++++--- documents/design/ArchaicAncestry_Design.md | 23 +++++++---- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/crates/navigator-analysis/src/archaic_match.rs b/crates/navigator-analysis/src/archaic_match.rs index 9d12645e..0d0a2a42 100644 --- a/crates/navigator-analysis/src/archaic_match.rs +++ b/crates/navigator-analysis/src/archaic_match.rs @@ -103,7 +103,10 @@ //! **But the reported extent puts the two populations in the wrong order.** The truth puts the //! archaic extent of East Asia at **1.217x** that of Europe. The extent that this caller reports //! is **0.937x**. A user would read that an East Asian carries *less* archaic ancestry than a -//! European. That is the wrong way round, and it is the one reason that this module stays gated. +//! European. That is the wrong way round. +//! +//! This is the reason that the shipped report is a **within-population** measure, and the reason +//! that the UI states the limit under the number. See *Where this landed* at the end. //! //! Here is the cause. The reported extent is the true positives *plus* the false positives. The //! load of false positives depends on the population, at a precision of 32.2 % against 41.9 %. @@ -176,14 +179,40 @@ //! number that you can compare across populations is not possible this way at present.** The //! caller is defensible inside one population, and not between two. //! -//! **This is still not enough to turn the module on.** Beyond the order of the populations, there -//! are three more reasons. +//! ## What this left open, and what answered it +//! +//! The module was still off at this point. Three things counted against it. Two of them now have +//! an answer, and the third stands. +//! +//! **The cohort was chr21 and chr22 alone.** The genome-wide run answers this. Read *Across the +//! genome* above: three Europeans, all 22 autosomes, sensitivity of 40 to 43 % and precision of +//! about 46 %. Both are better than the two-chromosome figures. So those figures are careful, and +//! the caller keeps its accuracy outside the two chromosomes that the fit used. //! -//! The precision is 34.9 % without the filter, on held-out Europeans. The cohort is **chr21 and -//! chr22 alone**. And the reference callset itself has weak support: the tracts of hmmix show an +//! **The precision was 34.9 % without the filter.** The path that ships applies the filter. The +//! app calls [`filter_by_concordance`] at [`MIN_CONCORDANCE`] after [`call_from_observations`], +//! and that configuration measures 90 %. +//! +//! **The reference callset has weak support.** This one stands. The tracts of hmmix show an //! enrichment of only 1.84x for their own archaic SNPs. Agreement with that callset then stops //! well below 100 %, even for a caller that is correct. F1 alone can not tell you when this work //! reaches its end. +//! +//! ## Where this landed +//! +//! **The module is on.** `ARCHAIC_SEGMENTS_ENABLED` in `navigator-app` is `true`, and Tier B +//! shipped in `v0.1.0-alpha.15`. The sections above read as a record of the work, and they stop +//! before that decision, so read this one for the result. +//! +//! It reports a **within-population** measure, for the reason that *A concordance filter* gives +//! above. The UI states that limit under the number, and gives no universal percentage. **You +//! must not use this number to compare people of different ancestries.** +//! +//! Tier A is a different measure, and its rule does not change. Tier A reports a **count** of +//! marker copies, and never a percentage. +//! +//! § 11 of `documents/design/ArchaicAncestry_Design.md` records the same history for a reader who +//! starts from the design. This doc-comment stays the detailed record. use std::collections::BTreeMap; diff --git a/documents/design/ArchaicAncestry_Design.md b/documents/design/ArchaicAncestry_Design.md index da180d6c..630385f2 100644 --- a/documents/design/ArchaicAncestry_Design.md +++ b/documents/design/ArchaicAncestry_Design.md @@ -1234,10 +1234,19 @@ So Tier B ships as a **within-population** measure (`b39db0b`), and the UI state than implying a universal percentage. **You must not use it to compare people of different ancestries.** The Tier A rule is unchanged and separate: Tier A reports a **count**, never a percent. -### Known stale prose in the module - -`archaic_match.rs` was written across 13 commits, and several passages in its doc-comment still say -the module "stays gated" or that a finding is "not enough to turn the module on". Those were true -when written and were overtaken by `b39db0b` (ship as within-population) and `9fca4c1` (genome-wide -validation passes). The flag is the truth: `ARCHAIC_SEGMENTS_ENABLED = true` in -`navigator-app/src/lib.rs`, whose comment is current. +### A note on how to read the module doc-comment + +`archaic_match.rs` was written across 13 commits, and its doc-comment is chronological: it walks +the evidence in the order the work produced it. Until 2026-08-23 it also *ended* in the order the +work produced it, which meant it closed on "this is still not enough to turn the module on" — +a verdict that `b39db0b` (ship as within-population) and `9fca4c1` (genome-wide validation passes) +had already overtaken. + +It now closes with two sections that carry the outcome: **What this left open, and what answered +it** (the chr21+22 cohort and the unfiltered precision were both answered; the weak reference +callset stands) and **Where this landed** (the module is on, the measure is within-population, and +Tier A's count rule is unchanged). Read those two first if you want the result rather than the +history. + +The flag remains the final authority: `ARCHAIC_SEGMENTS_ENABLED = true` in +`navigator-app/src/lib.rs`.