From b8da308cde0f38e0656bb3b22c80b99ddd88feb8 Mon Sep 17 00:00:00 2001 From: "J. Sebastian Paez" Date: Thu, 24 Sep 2026 10:03:22 -0700 Subject: [PATCH 1/5] Preserve library RT coordinates and calibrate search dimensions independently --- Cargo.lock | 11 +- Cargo.toml | 18 +- docs/development.md | 28 +- python/timsquery_pyo3/src/target.rs | 30 +- rust/calib_dash/src/bin/calib_dash.rs | 4 + ...alib_dash__ui__tests__fit_layer_Curve.snap | 2 +- ...calib_dash__ui__tests__fit_layer_None.snap | 2 +- ...calib_dash__ui__tests__fit_layer_Path.snap | 2 +- ...r_and_a_different_grid_when_scrubbing.snap | 2 +- ...every_binding_over_the_tab_beneath_it.snap | 2 +- rust/calib_dash/src/ui.rs | 2 +- rust/calibrt/src/lib.rs | 24 +- rust/timsquery/src/chemistry/analyte.rs | 4 +- rust/timsquery/src/lib.rs | 9 +- .../models/aggregators/chromatogram_agg.rs | 17 +- .../src/models/aggregators/point_agg.rs | 14 +- .../src/models/aggregators/spectrum_agg.rs | 11 +- rust/timsquery/src/models/indexed_data.rs | 19 +- rust/timsquery/src/models/mod.rs | 5 + rust/timsquery/src/models/query_handle.rs | 14 +- rust/timsquery/src/models/retention_time.rs | 107 ++++ rust/timsquery/src/models/source_id.rs | 4 +- rust/timsquery/src/models/target.rs | 60 ++- rust/timsquery/src/models/target_columns.rs | 67 ++- rust/timsquery/src/serde/diann_io.rs | 16 +- rust/timsquery/src/serde/diann_speclib_io.rs | 49 +- .../src/serde/elution_group_inputs.rs | 12 +- rust/timsquery/src/serde/library_file.rs | 77 ++- rust/timsquery/src/serde/mzspeclib_io.rs | 270 +++++++--- rust/timsquery/src/serde/skyline_io.rs | 9 +- rust/timsquery/src/serde/spectronaut_io.rs | 23 +- rust/timsquery/src/traits/mod.rs | 2 +- rust/timsquery/src/traits/queriable_data.rs | 10 +- rust/timsquery/src/traits/query_geom.rs | 36 +- rust/timsquery/tests/carafe_contract/main.rs | 2 +- rust/timsquery_cli/src/commands.rs | 8 +- rust/timsquery_cli/src/processing.rs | 17 +- rust/timsquery_viewer/src/app.rs | 99 ++-- rust/timsquery_viewer/src/calibration.rs | 91 +++- rust/timsquery_viewer/src/computed_state.rs | 12 +- rust/timsquery_viewer/src/file_loader.rs | 20 +- .../src/data_sources/reference_library.rs | 46 +- rust/timsseek/src/ml/qvalues.rs | 75 ++- rust/timsseek/src/rt_calibration.rs | 300 +++++++---- rust/timsseek/src/scoring/apex_finding.rs | 2 +- rust/timsseek/src/scoring/extraction.rs | 116 ++++- rust/timsseek/src/scoring/offsets.rs | 22 +- rust/timsseek/src/scoring/parquet_writer.rs | 14 +- rust/timsseek/src/scoring/pipeline.rs | 139 ++++-- rust/timsseek/src/scoring/plan.rs | 27 +- rust/timsseek/src/utils/elution_group_ops.rs | 6 +- rust/timsseek_cli/src/predicted_library.rs | 135 +++-- rust/timsseek_cli/src/processing.rs | 62 ++- .../src/processing/calibration.rs | 467 +++++++++++------- 54 files changed, 1920 insertions(+), 702 deletions(-) create mode 100644 rust/timsquery/src/models/retention_time.rs diff --git a/Cargo.lock b/Cargo.lock index 9716d6b3..033fa9e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4925,11 +4925,12 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] name = "mzannotate" version = "0.2.0" -source = "git+https://github.com/jspaezp/rustyms?rev=142c96ed1d31c56936c003af5f599d0c8f740005#142c96ed1d31c56936c003af5f599d0c8f740005" +source = "git+https://github.com/jspaezp/rustyms?rev=c033092561166e1b9960fa2a6678299af52a94e1#c033092561166e1b9960fa2a6678299af52a94e1" dependencies = [ "context_error", "indexmap", - "itertools 0.14.0", + "itertools 0.13.0", + "memchr", "mzcore", "mzcv", "mzdata", @@ -4944,12 +4945,12 @@ dependencies = [ [[package]] name = "mzcore" version = "0.2.0" -source = "git+https://github.com/jspaezp/rustyms?rev=142c96ed1d31c56936c003af5f599d0c8f740005#142c96ed1d31c56936c003af5f599d0c8f740005" +source = "git+https://github.com/jspaezp/rustyms?rev=c033092561166e1b9960fa2a6678299af52a94e1#c033092561166e1b9960fa2a6678299af52a94e1" dependencies = [ "bincode", "context_error", "flate2", - "itertools 0.14.0", + "itertools 0.13.0", "mzcv", "ordered-float 5.3.0", "roxmltree", @@ -4963,7 +4964,7 @@ dependencies = [ [[package]] name = "mzcv" version = "0.3.0" -source = "git+https://github.com/jspaezp/rustyms?rev=142c96ed1d31c56936c003af5f599d0c8f740005#142c96ed1d31c56936c003af5f599d0c8f740005" +source = "git+https://github.com/jspaezp/rustyms?rev=c033092561166e1b9960fa2a6678299af52a94e1#c033092561166e1b9960fa2a6678299af52a94e1" dependencies = [ "bincode", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 6d622b81..fcde6247 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,18 +77,12 @@ insta = { version = "1.34.0" } bon = "3.8.1" tinyvec = { features = ["alloc", "serde"], version = "1.10.0" } smallvec = { version = "1.13", features = ["const_generics", "union"] } -# `mzannotate` carries the mzSpecLib reader. Two fixes are not in a release yet: -# named attribute sets were never applied, so every decoy in a target/decoy -# library read back as a target, and retention time was dropped for any spectrum -# whose writer grouped other attributes with it. `mzcore` and `mzcv` come from -# the same source so the graph holds one copy of each and the ontology types -# unify. -# -# Pinned by revision, not by branch: a branch can be force-pushed or deleted. -# All three become version numbers once the fixes ship. -mzannotate = { git = "https://github.com/jspaezp/rustyms", rev = "142c96ed1d31c56936c003af5f599d0c8f740005" } -mzcore = { git = "https://github.com/jspaezp/rustyms", rev = "142c96ed1d31c56936c003af5f599d0c8f740005" } -mzcv = { git = "https://github.com/jspaezp/rustyms", rev = "142c96ed1d31c56936c003af5f599d0c8f740005" } +# mzSpecLib record API preserves RT terms, grouped units and attribute-set +# provenance before projection (jspaezp/rustyms PR #6). Keep all ontology crates +# on the same reviewed revision so their types unify. +mzannotate = { git = "https://github.com/jspaezp/rustyms", rev = "c033092561166e1b9960fa2a6678299af52a94e1" } +mzcore = { git = "https://github.com/jspaezp/rustyms", rev = "c033092561166e1b9960fa2a6678299af52a94e1" } +mzcv = { git = "https://github.com/jspaezp/rustyms", rev = "c033092561166e1b9960fa2a6678299af52a94e1" } csv = "1.3" tempfile = "3.23.0" diff --git a/docs/development.md b/docs/development.md index f8441048..7c62a4c0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -31,6 +31,32 @@ q-value columns (`result_mode=raw` Parquet metadata). It bypasses calibration, rescoring and q-value filtering. Supplied decoys do not by themselves validate a decoy strategy for a new analyte class. +Library RT is declared once per library: seconds, normalized index, unspecified +coordinates, or absent. Measured minutes convert to seconds; normalized indices +retain their values, including zero and negative values. Declared normalized-scale +metadata is preserved from mzSpecLib headers and prediction provenance; records +select the actual axis. Mixed RT availability or +axes are rejected. An RT-free library searches unrestricted on RT, omits RT +prediction/residual features, and retains decoy competition, rescoring and q-values +when decoys are available. RT, m/z and mobility calibrate independently. Without +an RT fit, score-selected apexes still supply m/z and mobility measurements, +without ridge filtering. RT-free prescoring retains bounded candidates across +observed RT bands. Mobility calibration requires a searchable run axis; FAIMS and +absent mobility do not contribute sentinel measurements. An axis without measurements keeps its configured tolerance +in primary and secondary extraction. A failed RT fit never reinterprets an index +as seconds; initial extraction searches unrestricted RT. Secondary queries still +center on detected apex RT and observed mobility. + +Results format 5 includes `library_rt_axis` Parquet metadata. Unavailable library +RT, calibrated RT and RT residuals are NaN; observed apex RT remains seconds. +Calibration format v4 records the input library axis and effective tolerance +enums; an empty RT snapshot means no RT fit, while residuals can still contain +m/z and mobility calibration. Older calibration files +must be regenerated. Explicit `rt_seconds` JSON/Python inputs remain seconds. JSON `rt_axis` is +preserved whether optional precursor/fragment labels are supplied or filled in. +Rust callers use the `Target` geometry trait, `OwnedTarget` for owned data, and +`AtObservedRt` to borrow library geometry at an observed extraction time. + `run_report.json` records the resolved `scoring_plan` once for the search library, plus `calibration_scoring_plan` when a separate calibration library is supplied. These are the same plans serialized in Parquet metadata: sequence-operation @@ -49,7 +75,7 @@ intensities, so those reader routes remain extraction-only. Standalone `calib_dash` reads saved `calibration.json`, not a spectral library. [Analyte module documentation](../rust/timsquery/src/chemistry/analyte.rs) documents chemistry storage, reader mappings, -library-wide sequence eligibility, and results format version 4. +library-wide sequence eligibility, and results format version 5. Precursor isotope envelopes retain the three-bin C/S approximation. Scoring finalization includes known modification C/S deltas or an explicitly based diff --git a/python/timsquery_pyo3/src/target.rs b/python/timsquery_pyo3/src/target.rs index f785d9b4..7e0d740f 100644 --- a/python/timsquery_pyo3/src/target.rs +++ b/python/timsquery_pyo3/src/target.rs @@ -1,5 +1,5 @@ use pyo3::prelude::*; -use timsquery::Target; +use timsquery::OwnedTarget; use timsquery::tinyvec::tiny_vec; /// A query target consists of one precursor and its fragments. @@ -11,7 +11,7 @@ use timsquery::tinyvec::tiny_vec; #[pyclass(skip_from_py_object)] #[derive(Debug, Clone)] pub struct PyTarget { - pub(crate) inner: Target, + pub(crate) inner: OwnedTarget, } #[pymethods] @@ -45,7 +45,7 @@ impl PyTarget { }; let fragment_labels_tv = fragment_labels.into_iter().collect(); - let target = Target::builder() + let target = OwnedTarget::builder() .id(id) .precursor(precursor_mz, precursor_charge) .mobility_ook0(mobility) @@ -75,8 +75,23 @@ impl PyTarget { } #[getter] - fn rt_seconds(&self) -> f32 { - self.inner.rt_seconds() + fn rt_seconds(&self) -> Option { + timsquery::Target::observed_rt_seconds(&self.inner) + } + + #[getter] + fn library_rt(&self) -> Option { + self.inner.rt().map(|rt| rt.value) + } + + #[getter] + fn rt_axis(&self) -> &'static str { + match self.inner.rt().map(|rt| rt.axis) { + None => "absent", + Some(timsquery::RtAxis::Seconds) => "seconds", + Some(timsquery::RtAxis::NormalizedIndex { .. }) => "normalized_index", + _ => "unspecified", + } } #[getter] @@ -96,11 +111,12 @@ impl PyTarget { fn __repr__(&self) -> String { format!( - "Target(id={}, mz={:.4}, charge={}, rt={:.1}s, mob={:.3}, frags={}, precs={})", + "Target(id={}, mz={:.4}, charge={}, rt={:?} ({}), mob={:.3}, frags={}, precs={})", self.inner.id(), self.inner.precursor_mz(), self.inner.precursor_charge(), - self.inner.rt_seconds(), + self.library_rt(), + self.rt_axis(), self.inner.mobility_ook0(), self.inner.fragment_count(), self.inner.precursor_count(), diff --git a/rust/calib_dash/src/bin/calib_dash.rs b/rust/calib_dash/src/bin/calib_dash.rs index 0fedce0a..00e96860 100644 --- a/rust/calib_dash/src/bin/calib_dash.rs +++ b/rust/calib_dash/src/bin/calib_dash.rs @@ -28,6 +28,10 @@ fn main() { std::process::exit(1); } }; + if snapshot.points.is_empty() { + eprintln!("No RT fit in {}; no RT curve to replay.", path.display()); + return; + } if let Err(e) = validate_snapshot(&snapshot) { eprintln!("invalid calibration snapshot in {}: {e}", path.display()); std::process::exit(1); diff --git a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Curve.snap b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Curve.snap index 3b96dd1e..dbceb77b 100644 --- a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Curve.snap +++ b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Curve.snap @@ -2,7 +2,7 @@ source: rust/calib_dash/src/ui.rs expression: out --- - ┌ Fit -- observed RT (s) ↑ vs library RT (s) → ──────────────────────────────────────────── b0 ┐ + ┌ Fit -- observed RT (s) ↑ vs library RT coordinate → ───────────────────────────────────── b0 ┐ │ Showing: curve -- fitted calibration │ │ │ │ ▄▄▄▄▄▄│ diff --git a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_None.snap b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_None.snap index 38162e04..d9deb187 100644 --- a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_None.snap +++ b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_None.snap @@ -2,7 +2,7 @@ source: rust/calib_dash/src/ui.rs expression: out --- - ┌ Fit -- observed RT (s) ↑ vs library RT (s) → ──────────────────────────────────────────── b0 ┐ + ┌ Fit -- observed RT (s) ↑ vs library RT coordinate → ───────────────────────────────────── b0 ┐ │ Showing: none -- density only │ │ │ │ ▄▄▄▄▄▄│ diff --git a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Path.snap b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Path.snap index 045ae801..432e31e9 100644 --- a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Path.snap +++ b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_layer_Path.snap @@ -2,7 +2,7 @@ source: rust/calib_dash/src/ui.rs expression: out --- - ┌ Fit -- observed RT (s) ↑ vs library RT (s) → ──────────────────────────────────────────── b0 ┐ + ┌ Fit -- observed RT (s) ↑ vs library RT coordinate → ───────────────────────────────────── b0 ┐ │ Showing: path -- O chosen, X greedy tail │ │ │ │ ▄▄▄▄▄▄│ diff --git a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_tab_shows_a_banner_and_a_different_grid_when_scrubbing.snap b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_tab_shows_a_banner_and_a_different_grid_when_scrubbing.snap index 205bf24e..93aea185 100644 --- a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_tab_shows_a_banner_and_a_different_grid_when_scrubbing.snap +++ b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__fit_tab_shows_a_banner_and_a_different_grid_when_scrubbing.snap @@ -3,7 +3,7 @@ source: rust/calib_dash/src/ui.rs expression: "render_snapshot(&mut app, 100, 30)" --- SCRUBBED -- retained frame 3/5, batch 17 (not live; `>` returns to now) - ┌ Fit -- observed RT (s) ↑ vs library RT (s) → ──────────────────────────────────────────── b0 ┐ + ┌ Fit -- observed RT (s) ↑ vs library RT coordinate → ───────────────────────────────────── b0 ┐ │ Showing: none -- density only │ │ ████████████│ 15┤ ████████████│ diff --git a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__the_keys_overlay_lists_every_binding_over_the_tab_beneath_it.snap b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__the_keys_overlay_lists_every_binding_over_the_tab_beneath_it.snap index 48742401..fb117b1b 100644 --- a/rust/calib_dash/src/snapshots/calib_dash__ui__tests__the_keys_overlay_lists_every_binding_over_the_tab_beneath_it.snap +++ b/rust/calib_dash/src/snapshots/calib_dash__ui__tests__the_keys_overlay_lists_every_binding_over_the_tab_beneath_it.snap @@ -3,7 +3,7 @@ source: rust/calib_dash/src/ui.rs expression: "render(&mut app, 100, 30)" --- Fit │ Convergence │ Tolerances - ┌ Fit -- observed RT (s) ↑ vs library RT (s) → ──────────────────────────────────────────── b0 ┐ + ┌ Fit -- observed RT (s) ↑ vs library RT coordinate → ───────────────────────────────────── b0 ┐ │ Showing: none -- density only │ │ │ │ ▄▄▄▄▄▄│ diff --git a/rust/calib_dash/src/ui.rs b/rust/calib_dash/src/ui.rs index fc1abc14..797277c8 100644 --- a/rust/calib_dash/src/ui.rs +++ b/rust/calib_dash/src/ui.rs @@ -457,7 +457,7 @@ fn draw_heatmap(frame: &mut Frame, area: Rect, app: &App) { return; }; - let title_left = " Fit -- observed RT (s) \u{2191} vs library RT (s) \u{2192} "; + let title_left = " Fit -- observed RT (s) \u{2191} vs library RT coordinate \u{2192} "; // `batch` only -- the active layer gets its own subtitle row below. let title_right = format!(" b{} ", app.batch()); frame.render_widget( diff --git a/rust/calibrt/src/lib.rs b/rust/calibrt/src/lib.rs index 0c1faf98..deaa8b80 100644 --- a/rust/calibrt/src/lib.rs +++ b/rust/calibrt/src/lib.rs @@ -231,9 +231,9 @@ pub struct CalibrationSnapshot { /// The only version [`SavedCalibration`] reads or writes. Named once so the /// writer and the reader's gate cannot disagree. -pub const CALIBRATION_FORMAT_VERSION: &str = "v3"; +pub const CALIBRATION_FORMAT_VERSION: &str = "v4"; -/// JSON v3 calibration file format -- shared between CLI and viewer. +/// JSON v4 calibration file format -- shared between CLI and viewer. /// /// `calibration` is the grid's own snapshot and the only record of the fit: the /// curve and the ridge widths are recomputed by refitting it, so the file cannot @@ -245,6 +245,9 @@ pub const CALIBRATION_FORMAT_VERSION: &str = "v3"; pub struct SavedCalibration { pub version: String, pub rt_range_seconds: [f64; 2], + /// Input-axis descriptor supplied by the library consumer. The fitter maps + /// numerical library coordinates to observed seconds without interpreting units. + pub library_rt_axis: serde_json::Value, pub calibration: CalibrationSnapshot, /// The uniform RT tolerance. Every writer has one -- it is what a query falls /// back to where the grid measured no ridge. @@ -269,6 +272,7 @@ impl SavedCalibration { ) -> Self { Self { version: CALIBRATION_FORMAT_VERSION.to_string(), + library_rt_axis: serde_json::json!({"kind": "unspecified"}), rt_range_seconds, calibration, rt_tolerance_minutes, @@ -277,6 +281,11 @@ impl SavedCalibration { } } + pub fn with_library_rt_axis(mut self, axis: serde_json::Value) -> Self { + self.library_rt_axis = axis; + self + } + /// Serialize to `path` in the layout [`Self::read`] expects. pub fn write(&self, path: &std::path::Path) -> Result<(), String> where @@ -299,13 +308,18 @@ impl SavedCalibration { R: serde::de::DeserializeOwned, { let json = std::fs::read_to_string(path).map_err(|e| e.to_string())?; - let saved: Self = serde_json::from_str(&json).map_err(|e| e.to_string())?; - if saved.version != CALIBRATION_FORMAT_VERSION { + #[derive(serde::Deserialize)] + struct Version { + version: String, + } + let version: Version = serde_json::from_str(&json).map_err(|e| e.to_string())?; + if version.version != CALIBRATION_FORMAT_VERSION { return Err(format!( "Unsupported calibration version: {} (expected {CALIBRATION_FORMAT_VERSION})", - saved.version + version.version )); } + let saved: Self = serde_json::from_str(&json).map_err(|e| e.to_string())?; let warning = saved.provenance_warning(raw_rt_range); Ok((saved, warning)) } diff --git a/rust/timsquery/src/chemistry/analyte.rs b/rust/timsquery/src/chemistry/analyte.rs index 9a28697b..f581a4b7 100644 --- a/rust/timsquery/src/chemistry/analyte.rs +++ b/rust/timsquery/src/chemistry/analyte.rs @@ -55,7 +55,7 @@ //! modification spellings therefore collapse. Incomparable entries remain separate; //! formula equality and labels do not establish chemical identity or competition. //! -//! Results format version 4 resolves metadata from the library. `sequence` is nullable +//! Results format version 5 resolves metadata from the library. `sequence` is nullable //! and canonically formatted; `entry_name`, `molecular_formula`, and `formula_basis` //! are nullable columns. Unformattable properties produce null. The viewer's Analyte //! column displays sequence or formula. `Analyte` serialization preserves property @@ -871,7 +871,7 @@ mod tests { TargetCapabilities, TargetColumnsBuilder, }; - use crate::traits::QueryGeom; + use crate::traits::Target; #[test] fn geometry_and_formula_rows_need_no_peptide() { diff --git a/rust/timsquery/src/lib.rs b/rust/timsquery/src/lib.rs index 96023678..1e4cc630 100644 --- a/rust/timsquery/src/lib.rs +++ b/rust/timsquery/src/lib.rs @@ -17,7 +17,7 @@ pub use crate::models::base::{ MzMajorIntensityArray, RTMajorIntensityArray, }; -pub use crate::models::target::Target; +pub use crate::models::target::OwnedTarget; // Re-export traits pub use crate::models::PeakAddable; @@ -71,3 +71,10 @@ pub use crate::serde::index_serde::{ LoadIndexError, load_index, }; + +pub use crate::models::retention_time::{ + AtObservedRt, + RtAxis, + RtCoordinate, +}; +pub use crate::traits::Target; diff --git a/rust/timsquery/src/models/aggregators/chromatogram_agg.rs b/rust/timsquery/src/models/aggregators/chromatogram_agg.rs index 948bbc68..5be66257 100644 --- a/rust/timsquery/src/models/aggregators/chromatogram_agg.rs +++ b/rust/timsquery/src/models/aggregators/chromatogram_agg.rs @@ -6,7 +6,7 @@ use crate::models::base::{ Chromatogram, MzMajorIntensityArray, }; -use crate::traits::QueryGeom; +use crate::traits::Target; use crate::traits::queriable_data::HasQueryData; use crate::{ KeyLike, @@ -63,7 +63,7 @@ pub struct ChromatogramCollector { impl ChromatogramCollector { pub fn new( - eg: &impl QueryGeom