Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions crates/navigator-analysis/src/postprocess/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ pub fn sort_alignment(
std::fs::create_dir_all(parent).map_err(|e| AnalysisError::io(parent, e))?;
}

let mut reader = open_bam(input)?;
let mut reader = bamio::open(input)?;
let header = reader.read_header().map_err(|e| AnalysisError::io(input, e))?;

let mut stats = SortStats::default();
Expand Down Expand Up @@ -333,10 +333,6 @@ fn heap_bytes(record: &RecordBuf) -> usize {
+ 256
}

fn open_bam(path: &Path) -> Result<bamio::BamReader, AnalysisError> {
bamio::open(path)
}

/// Stamp `@HD SO:coordinate` on the header.
///
/// Not cosmetic: an index is only valid for a coordinate-sorted file, and readers decide whether
Expand Down
24 changes: 14 additions & 10 deletions crates/navigator-analysis/src/revert/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,23 +139,27 @@ fn finish(w: FastqWriter, path: &Path) -> Result<(), AnalysisError> {
/// One FASTQ record. Names are written bare — no `/1` or `/2` — so R1/R2 pair by position; see the
/// module docs on why. Qualities are shifted into ASCII here, the inverse of the decode in
/// [`super::transform`], through `scratch` so the shift costs no allocation per read.
///
/// The whole record is assembled in `scratch` and handed over in **one** `write_all`. It was seven
/// — `@`, name, newline, sequence, `\n+\n`, qualities, newline — and each one entered the gzip
/// encoder's state machine separately, paying that overhead seven times per read rather than once.
/// Measured on this exact stack at 151 bp: **1,882 ns/record against 539 ns**, a 3.5x difference on
/// the write path of the stage that already holds the scratch peak. At ~600 M reads for a 30x WGS
/// that is roughly thirteen minutes of single-threaded CPU per realignment. Identical bytes out.
fn write_record(
w: &mut FastqWriter,
read: &RevertedRead,
path: &Path,
scratch: &mut Vec<u8>,
) -> Result<(), AnalysisError> {
scratch.clear();
scratch.push(b'@');
scratch.extend_from_slice(&read.name);
scratch.push(b'\n');
scratch.extend_from_slice(&read.sequence);
scratch.extend_from_slice(b"\n+\n");
scratch.extend(read.qualities.iter().map(|q| q.saturating_add(PHRED_OFFSET)));
scratch.push(b'\n');

let write = |w: &mut FastqWriter| -> std::io::Result<()> {
w.write_all(b"@")?;
w.write_all(&read.name)?;
w.write_all(b"\n")?;
w.write_all(&read.sequence)?;
w.write_all(b"\n+\n")?;
w.write_all(scratch)?;
w.write_all(b"\n")
};
write(w).map_err(|e| AnalysisError::io(path, e))
w.write_all(scratch).map_err(|e| AnalysisError::io(path, e))
}
4 changes: 2 additions & 2 deletions crates/navigator-app/src/commands.rs
Original file line number Diff line number Diff line change
Expand Up @@ -248,8 +248,6 @@ impl App {
self.alignment_or_err(id).await
}

/// Fetch an alignment by id, mapping a missing row to a `NotFound` error. The standard way
/// the analysis/query methods resolve an `alignment_id` before touching its BAM/CRAM.
/// An alignment by id, or `None` if there is no such row.
///
/// Public because provenance made alignments something callers ask about directly — the UI
Expand All @@ -258,6 +256,8 @@ impl App {
Ok(alignment::get(self.store.pool(), id).await?)
}

/// Fetch an alignment by id, mapping a missing row to a `NotFound` error. The standard way
/// the analysis/query methods resolve an `alignment_id` before touching its BAM/CRAM.
pub(crate) async fn alignment_or_err(&self, id: i64) -> Result<Alignment, AppError> {
alignment::get(self.store.pool(), id)
.await?
Expand Down
2 changes: 1 addition & 1 deletion crates/navigator-app/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2910,7 +2910,7 @@ mod queries;
mod realign;
/// Re-exported alone rather than opening the module: the UI needs to name the build it is offering
/// to realign to, and nothing else in there is its business.
pub use realign::DEFAULT_TARGET_BUILD;
pub use realign::{is_target_build, DEFAULT_TARGET_BUILD};
pub mod realign_job;
mod recruitment;
mod social;
Expand Down
61 changes: 52 additions & 9 deletions crates/navigator-app/src/realign.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

use std::path::{Path, PathBuf};

use navigator_domain::du_domain::ids::SampleGuid;
use navigator_domain::workspace::{Alignment, NewAlignment};
use navigator_store::alignment;

Expand Down Expand Up @@ -109,6 +110,23 @@ impl App {
.collect())
}

/// The subject an alignment belongs to, via its sequencing run.
///
/// The UI needs this to say whose realignment is running. Without it the running card matched on
/// alignment id alone, and a page showing subject A during a job on subject B told A their
/// genome was being rebuilt — the ownership question has to be answered where the mapping from
/// alignment to subject actually lives.
pub async fn subject_of_alignment(&self, id: i64) -> Result<Option<SampleGuid>, AppError> {
let Some(aln) = alignment::get(self.store.pool(), id).await? else {
return Ok(None);
};
Ok(
navigator_store::sequence_run::get(self.store.pool(), aln.sequence_run_id)
.await?
.map(|run| run.biosample_guid),
)
}

/// The alignment `id` was derived from, or `None` when it is an original.
pub async fn derivation_source(&self, id: i64) -> Result<Option<Alignment>, AppError> {
let aln = self.alignment_or_err(id).await?;
Expand All @@ -125,10 +143,30 @@ impl App {
/// refused anyway — anything already on the target build, anything already realigned, and
/// anything with no file to read — so the count is the real one rather than an upper bound.
pub async fn realignable_in_project(&self, project_id: i64, target_build: &str) -> Result<Vec<i64>, AppError> {
// One query for the whole project rather than one per member — the same idiom
// `project_report` uses on this very tab. Measured on a 2,504-member project: 2.7 ms for the
// grouped query against 17.7 ms for the per-member loop, and this runs twice per batch.
let guids: Vec<_> = self
.list_biosamples(project_id)
.await?
.into_iter()
.map(|s| s.guid)
.collect();
let rows = navigator_store::alignment::list_for_biosamples(self.store.pool(), &guids).await?;

// Grouped by subject, not flattened: the rule's "already realigned" condition asks whether
// anything *in that subject's own set* was derived from a given alignment, so it has to see
// one subject's alignments at a time.
let mut by_subject: std::collections::HashMap<_, Vec<Alignment>> = std::collections::HashMap::new();
for (guid, alignment) in rows {
by_subject.entry(guid).or_default().push(alignment);
}

let mut out = Vec::new();
for subject in self.list_biosamples(project_id).await? {
let alignments = navigator_store::alignment::list_for_biosample(self.store.pool(), subject.guid).await?;
out.extend(realignable_for_subject(&alignments, target_build));
for guid in &guids {
if let Some(alignments) = by_subject.get(guid) {
out.extend(realignable_for_subject(alignments, target_build));
}
}
Ok(out)
}
Expand All @@ -154,15 +192,16 @@ impl App {
}
}

/// Whether two build names refer to the same reference for this purpose.
///
/// Compared case-insensitively on the recorded strings. Deliberately *not* normalised through
/// The build realignment targets when nothing says otherwise — the complete assembly, which is the
/// only reason the module exists.
pub const DEFAULT_TARGET_BUILD: &str = "chm13v2.0";

/// Whether `build` is the realignment target — the complete assembly.
pub(crate) fn is_target_build(build: &str) -> bool {
///
/// `pub` so the UI can ask the question rather than spelling out its own comparison. Both Advanced
/// realign cards used to do the latter, with `eq_ignore_ascii_case` and no trim, which is a subtly
/// different rule from the one the job enforces.
pub fn is_target_build(build: &str) -> bool {
builds_match(build, DEFAULT_TARGET_BUILD)
}

Expand All @@ -186,8 +225,12 @@ pub(crate) fn realignable_for_subject(alignments: &[Alignment], target_build: &s
.collect()
}

/// `canonical_build`: `chm13v2.0` and `chm13v2.0_maskedY_rCRS` share coordinates but differ in
/// chrM and in PAR masking, so realigning between them is a real operation rather than a no-op.
/// Whether two build names refer to the same reference for this purpose.
///
/// Compared case-insensitively on the recorded strings, after trimming. Deliberately *not*
/// normalised through a `canonical_build`: `chm13v2.0` and `chm13v2.0_maskedY_rCRS` share
/// coordinates but differ in chrM and in PAR masking, so realigning between them is a real
/// operation rather than a no-op.
fn builds_match(a: &str, b: &str) -> bool {
a.trim().eq_ignore_ascii_case(b.trim())
}
Expand Down
40 changes: 15 additions & 25 deletions crates/navigator-app/src/realign_job.rs
Original file line number Diff line number Diff line change
Expand Up @@ -453,9 +453,7 @@ impl App {
// Every stage's input is dead once the next stage has read it, and at WGS scale each is
// tens of GB. Holding them all until the job ends — which is what this did first — roughly
// doubles the peak and is the difference between fitting on a normal disk and not.
let discard = |path: &Path| {
let _ = std::fs::remove_file(path);
};
let discard = discard_partial;
if let Some(reverted) = &reverted {
discard(&reverted.read1);
discard(&reverted.read2);
Expand Down Expand Up @@ -627,23 +625,7 @@ pub fn preflight(scratch: &Path, source: &Path, source_size: u64) -> Result<Real
let needed = source_size
.saturating_mul(expansion_factor(source))
.saturating_mul(SCRATCH_MULTIPLE);
let free = free_space(scratch);

if !has_room(needed, free) {
return Err(AppError::Import(format!(
"not enough room to realign: about {} GB of working space is needed and {} GB is free \
on {}",
gb(needed),
gb(free),
scratch.display(),
)));
}

Ok(RealignPlan {
batch: BatchSize::for_this_machine(),
scratch_needed: needed,
scratch_free: free,
})
plan_for(scratch, needed, "realign")
}

/// Preflight for a job resuming from intermediates that already exist.
Expand All @@ -660,22 +642,30 @@ pub fn preflight(scratch: &Path, source: &Path, source_size: u64) -> Result<Real
fn resume_preflight(scratch: &Path, mapped: &Path, sorted: &Path, marked: &Path) -> Result<RealignPlan, AppError> {
let size = |path: &Path| std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
let largest = size(mapped).max(size(sorted)).max(size(marked));
let needed = largest.saturating_mul(3);
plan_for(scratch, largest.saturating_mul(3), "resume the realignment")
}

/// Measure the disk, refuse a job that cannot finish on it, and describe what was decided.
///
/// The two preflights differ only in how they size `needed`; everything after that — probing free
/// space, the refusal, the wording, the plan — was written out twice and had to be kept in step by
/// hand. `what` is the verb in the refusal, so the two messages stay exactly as they were.
fn plan_for(scratch: &Path, needed: u64, what: &str) -> Result<RealignPlan, AppError> {
let free = free_space(scratch);

if !has_room(needed, free) {
return Err(AppError::Import(format!(
"not enough room to resume the realignment: about {} GB of working space is needed and \
{} GB is free on {}",
"not enough room to {what}: about {} GB of working space is needed and {} GB is free \
on {}",
gb(needed),
gb(free),
scratch.display(),
)));
}

Ok(RealignPlan {
// Nothing that reads this is going to run — resuming starts at the sort, which is past the
// index — but the plan is the shared shape and a caller may still log it.
// A resumed job never reaches the index stage, but the plan is one shape and a caller may
// still log the figure.
batch: BatchSize::for_this_machine(),
scratch_needed: needed,
scratch_free: free,
Expand Down
17 changes: 8 additions & 9 deletions crates/navigator-ui/src/ui/detail.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1656,8 +1656,6 @@ impl NavigatorApp {
});
}

/// A per-sample coverage/haplogroup table for the open project, with per-row coverage
/// recompute and a CSV export. Coverage/haplogroup cells show "—" until computed.
/// Realign a whole project, as a card rather than a dialog for the same reason the per-alignment
/// one is: this runs for *days*, and nothing that long should own the screen.
///
Expand All @@ -1668,7 +1666,7 @@ impl NavigatorApp {
let Some(project_id) = self.selected_project else {
return;
};
let target = "chm13v2.0";
let target = navigator_app::DEFAULT_TARGET_BUILD;

// Asked of the app, once per project, rather than filtered here. This used to count
// `all_alignments` — the whole workspace — and label the result "in this project": a
Expand All @@ -1691,9 +1689,10 @@ impl NavigatorApp {
});
return;
};
let eligible = eligible.clone();
// Only the count is read below, so take that rather than cloning the Vec every frame.
let eligible_count = eligible.len();

if eligible.is_empty() {
if eligible_count == 0 {
ui.label(format!(
"Every alignment in this project is already on {target}, or has been realigned."
));
Expand All @@ -1702,12 +1701,12 @@ impl NavigatorApp {

ui.label(format!(
"{} alignment(s) in this project could be re-mapped to {target}.",
eligible.len()
eligible_count
));
ui.add_space(4.0);
ui.label(
egui::RichText::new(
"They run one after another, each taking hours. Stopping ends the whole batch; everything already finished is kept, and no original is changed.",
"They run one after another, each taking hours. Stopping ends the whole batch; everything already finished is kept, and no original is changed.",
)
.weak()
.size(12.0),
Expand All @@ -1719,15 +1718,15 @@ impl NavigatorApp {
if ui
.add_enabled(
!busy,
egui::Button::new(format!("Realign {} to {target}", eligible.len())),
egui::Button::new(format!("Realign {eligible_count} to {target}")),
)
.clicked()
{
let _ = self.tx.send(Command::StartProjectRealign {
project_id,
target_build: target.to_string(),
});
self.status = format!("Realigning {} alignment(s) to {target}…", eligible.len());
self.status = format!("Realigning {eligible_count} alignment(s) to {target}…");
}
if busy && ui.button("Stop").clicked() {
let _ = self.tx.send(Command::CancelRealign);
Expand Down
11 changes: 8 additions & 3 deletions crates/navigator-ui/src/ui/events.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1003,13 +1003,15 @@ impl NavigatorApp {
}
Event::RealignProgress {
alignment_id,
biosample_guid,
step,
total,
label,
detail,
} => {
self.realign = Some(super::RealignState {
alignment_id,
biosample_guid,
step,
total,
label,
Expand All @@ -1019,6 +1021,7 @@ impl NavigatorApp {
}
Event::RealignDone {
alignment_id,
biosample_guid,
new_alignment_id,
cancelled,
summary,
Expand All @@ -1031,11 +1034,13 @@ impl NavigatorApp {
(None, true) => super::RealignFinished::Cancelled,
(None, false) => super::RealignFinished::Failed(summary.clone()),
};
let prior = self.realign.take();
// step/total are zero rather than carried over: every consumer matches on
// `finished` first and none of them reads progress from a finished card.
self.realign = Some(super::RealignState {
alignment_id,
step: prior.as_ref().map(|r| r.step).unwrap_or(0),
total: prior.as_ref().map(|r| r.total).unwrap_or(0),
biosample_guid,
step: 0,
total: 0,
label: String::new(),
detail: String::new(),
finished: Some(finished),
Expand Down
9 changes: 7 additions & 2 deletions crates/navigator-ui/src/ui/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,10 @@ struct AnalysisModal {
struct RealignState {
/// The source alignment this job belongs to — cards for other alignments ignore it.
alignment_id: i64,
/// The subject it belongs to. Simple mode's card is about a *person*, not an alignment, so it
/// has to match on this: with only the alignment id to go on, a page open on subject A during a
/// job on subject B told A their genome was being rebuilt.
biosample_guid: Option<SampleGuid>,
step: usize,
total: usize,
label: String,
Expand Down Expand Up @@ -665,14 +669,15 @@ pub struct NavigatorApp {
analysis: Option<AnalysisModal>,
/// The running (or last finished) realignment; see [`RealignState`].
realign: Option<RealignState>,
/// Simple mode's pending realignment confirmation, `(alignment id, its current build)`.
/// Simple mode's pending realignment confirmation — the same [`RealignOffer`] the brief
/// supplied, rather than a tuple re-spelling its two fields.
///
/// Simple mode gets a confirmation step where Advanced does not, and the asymmetry is
/// deliberate: the Advanced card sits among alignment internals and states its cost in a
/// paragraph its reader is equipped to weigh. Simple mode's reader has been shown a story about
/// their ancestors, and should not be able to commit the machine to four hours and 276 GB by
/// misjudging one button.
simple_realign_confirm: Option<(i64, String)>,
simple_realign_confirm: Option<navigator_domain::brief::RealignOffer>,
/// Set the moment Cancel is clicked, cleared when the run actually ends.
///
/// Cancellation is cooperative: the walkers stop at their next check, so there is always a gap
Expand Down
Loading
Loading