diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b4346..2656ba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,13 @@ Sections commonly used: Features, Bug fixes, Other changes. ### Bug fixes +- **Multi-member gzip input is no longer truncated.** Compressed input was + decoded with `flate2::read::GzDecoder`, which stops at the end of the first + gzip member; a `.gz` made of several concatenated members (bcl2fastq output, + `cat a.fq.gz b.fq.gz`, any BGZF file) was read partially with no error and no + warning. All four read paths now use `MultiGzDecoder`: FASTQ input, the solo + barcode whitelist, solo counting, and the `emptydrops` binary. + - Read names are cut at `--readNameSeparator` (default `/`), as STAR does. A read named `foo/1` was previously emitted as `foo/1` where STAR emits `foo`. diff --git a/src/bin/emptydrops.rs b/src/bin/emptydrops.rs index 29913cd..4cac8f4 100644 --- a/src/bin/emptydrops.rs +++ b/src/bin/emptydrops.rs @@ -19,7 +19,7 @@ use std::fs::File; use std::io::{BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -use flate2::read::GzDecoder; +use flate2::read::MultiGzDecoder; use rustar_aligner::rng::{SplitMix64, cumulative_weights, sample_cumulative}; struct Args { @@ -89,7 +89,7 @@ fn find(d: &Path, base: &str) -> PathBuf { fn reader(p: &Path) -> Box { let f = File::open(p).unwrap(); if p.extension().is_some_and(|e| e == "gz") { - Box::new(BufReader::new(GzDecoder::new(f))) + Box::new(BufReader::new(MultiGzDecoder::new(f))) } else { Box::new(BufReader::new(f)) } diff --git a/src/io/fastq.rs b/src/io/fastq.rs index 0c795de..93c48d9 100644 --- a/src/io/fastq.rs +++ b/src/io/fastq.rs @@ -1,6 +1,6 @@ /// FASTQ reader with base encoding and decompression support use crate::error::Error; -use flate2::read::GzDecoder; +use flate2::read::MultiGzDecoder; use noodles::fastq; use std::fs::File; use std::io::{BufRead, BufReader, BufWriter, Write}; @@ -116,7 +116,7 @@ impl FastqReader { let buffered = BufReader::with_capacity(DECODE_BUF, file); Box::new(BufReader::with_capacity( DECODE_BUF, - GzDecoder::new(buffered), + MultiGzDecoder::new(buffered), )) } else { // Plain text FASTQ @@ -569,6 +569,54 @@ mod tests { assert_eq!(read1.quality.len(), 4); } + /// A `.gz` written as several concatenated gzip members — what `bcl2fastq` + /// emits, what `cat a.fq.gz b.fq.gz` produces, and what every BGZF file is. + /// `flate2::read::GzDecoder` stops after the first member and reports EOF, + /// so reading such a file used to drop reads with no error at all. + #[test] + fn test_fastq_reader_gzip_multi_member() { + use flate2::Compression; + use flate2::write::GzEncoder; + + let mut tmpfile = tempfile::Builder::new() + .suffix(".fastq.gz") + .tempfile() + .unwrap(); + + // Member 1: read1. Each `finish()` closes a complete gzip stream, so + // the next encoder appends a second member rather than continuing. + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + writeln!(encoder, "@read1").unwrap(); + writeln!(encoder, "ACGT").unwrap(); + writeln!(encoder, "+").unwrap(); + writeln!(encoder, "IIII").unwrap(); + tmpfile.write_all(&encoder.finish().unwrap()).unwrap(); + + // Member 2: read2. + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + writeln!(encoder, "@read2").unwrap(); + writeln!(encoder, "TGCA").unwrap(); + writeln!(encoder, "+").unwrap(); + writeln!(encoder, "HHHH").unwrap(); + tmpfile.write_all(&encoder.finish().unwrap()).unwrap(); + tmpfile.flush().unwrap(); + + let mut reader = FastqReader::open(tmpfile.path(), None).unwrap(); + + let read1 = reader.next_encoded().unwrap().unwrap(); + assert_eq!(read1.name, "read1"); + assert_eq!(read1.sequence, vec![0, 1, 2, 3]); // ACGT + + let read2 = reader + .next_encoded() + .unwrap() + .expect("second gzip member must be decoded, not silently truncated"); + assert_eq!(read2.name, "read2"); + assert_eq!(read2.sequence, vec![3, 2, 1, 0]); // TGCA + + assert!(reader.next_encoded().unwrap().is_none()); + } + #[test] fn test_strip_mate_suffix_slash() { assert_eq!(strip_mate_suffix("read123/1"), "read123"); diff --git a/src/ruSTAR.code-workspace b/src/ruSTAR.code-workspace new file mode 100644 index 0000000..3e9ebe5 --- /dev/null +++ b/src/ruSTAR.code-workspace @@ -0,0 +1,11 @@ +{ + "folders": [ + { + "path": "../../STAR-rs" + }, + { + "path": ".." + } + ], + "settings": {} +} \ No newline at end of file diff --git a/src/solo/count.rs b/src/solo/count.rs index 4ddb414..cda22e7 100644 --- a/src/solo/count.rs +++ b/src/solo/count.rs @@ -2320,7 +2320,9 @@ fn open_maybe_gz(path: &Path) -> Result, Error> { .extension() .is_some_and(|e| e.eq_ignore_ascii_case("gz")) { - Ok(Box::new(BufReader::new(flate2::read::GzDecoder::new(file)))) + Ok(Box::new(BufReader::new(flate2::read::MultiGzDecoder::new( + file, + )))) } else { Ok(Box::new(BufReader::new(file))) } diff --git a/src/solo/whitelist.rs b/src/solo/whitelist.rs index 1d882ae..8c87c23 100644 --- a/src/solo/whitelist.rs +++ b/src/solo/whitelist.rs @@ -13,7 +13,7 @@ use crate::error::Error; use crate::io::fastq::{decode_base, encode_base}; -use flate2::read::GzDecoder; +use flate2::read::MultiGzDecoder; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::Path; @@ -535,7 +535,7 @@ fn open_maybe_gzip(path: &Path) -> Result, Error> { .extension() .is_some_and(|e| e.eq_ignore_ascii_case("gz")); if is_gz { - Ok(Box::new(BufReader::new(GzDecoder::new(file)))) + Ok(Box::new(BufReader::new(MultiGzDecoder::new(file)))) } else { Ok(Box::new(BufReader::new(file))) }