From 517fbd092d74afc51136ebc615aa832f43d3b4dd Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 4 Sep 2026 14:24:28 -0400 Subject: [PATCH 1/9] Pass the genome fasta to the ngs-samples nextflow workflow MakeNgsSamplesNextflowConfig now takes a genomeFile param, digests it to its cluster path, and writes it into the generated config as genomeFastaFile (same name dnaseq-nextflow uses). All six callers pass $$genomeSymLink$$ and now depend on symLinkGenome. The fasta already reached the cluster -- copyToCluster mirrors all of dataDir, which is where symLinkGenome puts it -- but the config step only depended on makeResultDir, so it could run before the symlink existed. Co-Authored-By: Claude Opus 5 (1M context) --- Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm | 3 +++ Main/lib/xml/workflow/bulkrnaseq.xml | 2 ++ Main/lib/xml/workflow/longReadRnaSeq.xml | 2 ++ Main/lib/xml/workflow/processChipSeqExperiment.xml | 2 ++ Main/lib/xml/workflow/processOriginsExperiment.xml | 2 ++ .../workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml | 2 ++ Main/lib/xml/workflow/snpAndCnvDNASeq.xml | 2 ++ 7 files changed, 15 insertions(+) diff --git a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm index 253c2354cc..2a2987688e 100644 --- a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm @@ -19,6 +19,7 @@ sub run { my $sampleSheetName = $self->getParamValue("sampleSheetName"); my $assayType = $self->getParamValue("assayType"); my $organismAbbrev = $self->getParamValue('organismAbbrev'); + my $genomeFile = $self->getParamValue("genomeFile"); my $fromSRA = $self->getBooleanParamValue("fromSRA") ? "true" : "false"; # Max SRA file size prefetch will download. If a data load fails because prefetch @@ -34,6 +35,7 @@ sub run { my $digestedFinalDirPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $finalDir); my $digestedAnalysisDirPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $analysisDirectory); my $digestedOutputDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $resultsDirectory); + my $digestedGenomeFile = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $genomeFile); my $clusterServer = $self->getSharedConfig('clusterServer'); my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); @@ -79,6 +81,7 @@ params { fromSra = $fromSRA outDir = "$digestedOutputDir" genomeSize = $genomeSize + genomeFastaFile = "$digestedGenomeFile" assayType = "$assayType" maxDownloadSize = "$maxDownloadSize" } diff --git a/Main/lib/xml/workflow/bulkrnaseq.xml b/Main/lib/xml/workflow/bulkrnaseq.xml index dab70476ba..d2aff6d913 100644 --- a/Main/lib/xml/workflow/bulkrnaseq.xml +++ b/Main/lib/xml/workflow/bulkrnaseq.xml @@ -74,7 +74,9 @@ $$fromSRA$$ RNASeq $$organismAbbrev$$ + $$genomeSymLink$$ + diff --git a/Main/lib/xml/workflow/longReadRnaSeq.xml b/Main/lib/xml/workflow/longReadRnaSeq.xml index 6ff229b780..37d37d9620 100644 --- a/Main/lib/xml/workflow/longReadRnaSeq.xml +++ b/Main/lib/xml/workflow/longReadRnaSeq.xml @@ -73,7 +73,9 @@ $$fromSRA$$ RNASeq $$organismAbbrev$$ + $$genomeSymLink$$ + diff --git a/Main/lib/xml/workflow/processChipSeqExperiment.xml b/Main/lib/xml/workflow/processChipSeqExperiment.xml index 6598e10dba..be94890b19 100644 --- a/Main/lib/xml/workflow/processChipSeqExperiment.xml +++ b/Main/lib/xml/workflow/processChipSeqExperiment.xml @@ -63,7 +63,9 @@ $$fromSRA$$ ChipSeq $$organismAbbrev$$ + $$genomeSymLink$$ + diff --git a/Main/lib/xml/workflow/processOriginsExperiment.xml b/Main/lib/xml/workflow/processOriginsExperiment.xml index ab0f9baf87..5e3833f554 100644 --- a/Main/lib/xml/workflow/processOriginsExperiment.xml +++ b/Main/lib/xml/workflow/processOriginsExperiment.xml @@ -63,7 +63,9 @@ $$fromSRA$$ DNASeq $$organismAbbrev$$ + $$genomeSymLink$$ + diff --git a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml index 4b960b45ea..e8297e7678 100644 --- a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml +++ b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml @@ -60,7 +60,9 @@ $$fromSRA$$ RNASeq $$organismAbbrev$$ + $$genomeSymLink$$ + diff --git a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml index 9cc41e8bda..b6a5f9482c 100644 --- a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml +++ b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml @@ -104,7 +104,9 @@ $$fromSRA$$ DNASeq $$organismAbbrev$$ + $$genomeSymLink$$ + From 640512efd461fad7b27fe233a2bf2ef9b197a3f6 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 4 Sep 2026 14:40:51 -0400 Subject: [PATCH 2/9] Track the contamination-aware subsampling params in ngs-samples config Matches the param changes in ngs-samples-nextflow: retired genomeSize measured from referenceFasta now added referenceFasta required; pipeline fails fast without it added targetCoverage 60, was hardcoded in calculateMaxReads() added minOnTargetFraction 0.05 fraction floor / 20x inflation cap added minPlausibleFraction 0.01 wrong-genome flag threshold added pilotSize 100000 pilot draw Retiring genomeSize drops the getGenomeSize DB lookup, so this step no longer touches the database. gusConfigFile and organismAbbrev fed only that lookup and are removed from the six callers. The gate thresholds are constants here rather than XML params: the design treats them as one gate with explicit thresholds, not per-graph knobs, and six duplicated paramValues is how they become tribal knowledge. Co-Authored-By: Claude Opus 5 (1M context) --- .../MakeNgsSamplesNextflowConfig.pm | 20 +++++++++++++------ Main/lib/xml/workflow/bulkrnaseq.xml | 2 -- Main/lib/xml/workflow/longReadRnaSeq.xml | 2 -- .../xml/workflow/processChipSeqExperiment.xml | 2 -- .../xml/workflow/processOriginsExperiment.xml | 2 -- ...eqSplicedLeaderAndPolyASitesExperiment.xml | 2 -- Main/lib/xml/workflow/snpAndCnvDNASeq.xml | 2 -- 7 files changed, 14 insertions(+), 18 deletions(-) diff --git a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm index 2a2987688e..0b7c96e5e2 100644 --- a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm @@ -18,7 +18,6 @@ sub run { my $nextflowConfigFile = $self->getParamValue("nextflowConfigFile"); my $sampleSheetName = $self->getParamValue("sampleSheetName"); my $assayType = $self->getParamValue("assayType"); - my $organismAbbrev = $self->getParamValue('organismAbbrev'); my $genomeFile = $self->getParamValue("genomeFile"); my $fromSRA = $self->getBooleanParamValue("fromSRA") ? "true" : "false"; @@ -27,15 +26,21 @@ sub run { # config on the cluster and re-run (default sra-tools limit is 20G). my $maxDownloadSize = "50G"; - my $gusConfig = $self->getWorkflowDataDir() . "/" . $self->getParamValue('gusConfigFile'); - my $genomeSize = $self->getGenomeSize($test, $organismAbbrev, $gusConfig); + # Depth gate. minOnTargetFraction doubles as the inflation cap: a 0.05 floor means a + # contaminated sample is asked for at most 20x the reads a clean one would get. + # Below minPlausibleFraction the sample is flagged as probably-wrong-genome, not failed. + my $targetCoverage = 60; + my $minOnTargetFraction = 0.05; + my $minPlausibleFraction = 0.01; + my $pilotSize = 100000; + my $workflowDataDir = $self->getWorkflowDataDir(); my $workingDirRelativePath = $self->getParamValue("workingDirRelativePath"); my $digestedFinalDirPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $finalDir); my $digestedAnalysisDirPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $analysisDirectory); my $digestedOutputDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $resultsDirectory); - my $digestedGenomeFile = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $genomeFile); + my $digestedReferenceFasta = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $genomeFile); my $clusterServer = $self->getSharedConfig('clusterServer'); my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); @@ -80,9 +85,12 @@ params { samplesheetName = "$sampleSheetName" fromSra = $fromSRA outDir = "$digestedOutputDir" - genomeSize = $genomeSize - genomeFastaFile = "$digestedGenomeFile" + referenceFasta = "$digestedReferenceFasta" assayType = "$assayType" + targetCoverage = $targetCoverage + minOnTargetFraction = $minOnTargetFraction + minPlausibleFraction = $minPlausibleFraction + pilotSize = $pilotSize maxDownloadSize = "$maxDownloadSize" } diff --git a/Main/lib/xml/workflow/bulkrnaseq.xml b/Main/lib/xml/workflow/bulkrnaseq.xml index d2aff6d913..d091c85a24 100644 --- a/Main/lib/xml/workflow/bulkrnaseq.xml +++ b/Main/lib/xml/workflow/bulkrnaseq.xml @@ -64,7 +64,6 @@ - $$gusConfigFile$$ $$analysisDirectory$$ $$finalSymLink$$ $$analysisDirectory$$/ngs-samples-results @@ -73,7 +72,6 @@ samplesheet.csv $$fromSRA$$ RNASeq - $$organismAbbrev$$ $$genomeSymLink$$ diff --git a/Main/lib/xml/workflow/longReadRnaSeq.xml b/Main/lib/xml/workflow/longReadRnaSeq.xml index 37d37d9620..97bbe5a204 100644 --- a/Main/lib/xml/workflow/longReadRnaSeq.xml +++ b/Main/lib/xml/workflow/longReadRnaSeq.xml @@ -63,7 +63,6 @@ - $$gusConfigFile$$ $$analysisDirectory$$ $$finalSymLink$$ $$analysisDirectory$$/ngs-samples-results @@ -72,7 +71,6 @@ samplesheet.csv $$fromSRA$$ RNASeq - $$organismAbbrev$$ $$genomeSymLink$$ diff --git a/Main/lib/xml/workflow/processChipSeqExperiment.xml b/Main/lib/xml/workflow/processChipSeqExperiment.xml index be94890b19..cbe12c9e16 100644 --- a/Main/lib/xml/workflow/processChipSeqExperiment.xml +++ b/Main/lib/xml/workflow/processChipSeqExperiment.xml @@ -53,7 +53,6 @@ - $$gusConfigFile$$ $$analysisDirectory$$ $$finalSymLink$$ $$analysisDirectory$$/ngs-samples-results @@ -62,7 +61,6 @@ samplesheet.csv $$fromSRA$$ ChipSeq - $$organismAbbrev$$ $$genomeSymLink$$ diff --git a/Main/lib/xml/workflow/processOriginsExperiment.xml b/Main/lib/xml/workflow/processOriginsExperiment.xml index 5e3833f554..ce8648934b 100644 --- a/Main/lib/xml/workflow/processOriginsExperiment.xml +++ b/Main/lib/xml/workflow/processOriginsExperiment.xml @@ -53,7 +53,6 @@ - $$gusConfigFile$$ $$analysisDirectory$$ $$finalSymLink$$ $$analysisDirectory$$/ngs-samples-results @@ -62,7 +61,6 @@ samplesheet.csv $$fromSRA$$ DNASeq - $$organismAbbrev$$ $$genomeSymLink$$ diff --git a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml index e8297e7678..47b68caa16 100644 --- a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml +++ b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml @@ -50,7 +50,6 @@ - $$gusConfigFile$$ $$analysisDirectory$$ $$finalSymLink$$ $$analysisDirectory$$/ngs-samples-results @@ -59,7 +58,6 @@ samplesheet.csv $$fromSRA$$ RNASeq - $$organismAbbrev$$ $$genomeSymLink$$ diff --git a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml index b6a5f9482c..b8290b07c5 100644 --- a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml +++ b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml @@ -94,7 +94,6 @@ - $$gusConfigFile$$ $$analysisDirectory$$ $$finalSymLink$$ $$analysisDirectory$$/ngs-samples-results @@ -103,7 +102,6 @@ samplesheet.csv $$fromSRA$$ DNASeq - $$organismAbbrev$$ $$genomeSymLink$$ From 44d1a070bedd7cbda894e73992f03ef143cd183d Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 4 Sep 2026 16:39:12 -0400 Subject: [PATCH 3/9] Run ngs-samples and the analysis nextflow from a single workflow step runNextflowOnCluster.xml ran ngs-samples-nextflow and the analysis workflow as two steps sharing the runNextflowOnCluster load type. Each released its throttle slot on completion, so after ngs-samples fetched an experiment's reads the analysis had to re-queue behind other experiments' fetches. Running both from one step makes the pair hold a single slot, which gives the throttle its back-pressure, and drops a step for the 24 callers that pass no ngs-samples config (there RunAndMonitorNextflowNGSSamples did nothing at all). Each run keeps its own job info, log, trace and stdout file names, unchanged from before, so a re-entered step still detects an already finished run from its own log and in-flight cluster state stays resumable. Replaces RunAndMonitorNextflowNGSSamples, whose only job was to rename those files and no-op on an empty config. Co-Authored-By: Claude Opus 5 (1M context) --- .../RunAndMonitorNextflowNGSSamples.pm | 30 ----- .../WorkflowSteps/RunNgsSamplesAndNextflow.pm | 124 ++++++++++++++++++ .../lib/xml/workflow/runNextflowOnCluster.xml | 21 +-- Main/t/run_ngs_samples_and_nextflow.t | 59 +++++++++ 4 files changed, 189 insertions(+), 45 deletions(-) delete mode 100644 Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm create mode 100644 Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm create mode 100644 Main/t/run_ngs_samples_and_nextflow.t diff --git a/Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm b/Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm deleted file mode 100644 index f61013c051..0000000000 --- a/Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm +++ /dev/null @@ -1,30 +0,0 @@ -package ApiCommonWorkflow::Main::WorkflowSteps::RunAndMonitorNextflowNGSSamples; - -@ISA = (ReFlow::StepClasses::RunAndMonitorNextflow); - -use strict; -use ReFlow::StepClasses::RunAndMonitorNextflow; - -sub clusterJobInfoFileName { return "ngs-samples-clusterJobInfo.txt"} -sub logFileName { return "ngs-samples-nextflow.log" } -sub traceFileName { return "ngs-samples-trace.txt" } -sub nextflowStdoutFileName { return "ngs-samples-nextflow.txt" } - - -sub run { - my ($self, $test, $undo) = @_; - - my $nextflowConfigFile = $self->getParamValue('nextflowConfigFile'); - my $entry = $self->getParamValue('entry'); - - # if the xml has set this config file.. run the cluster job - if($nextflowConfigFile ne "") { - $self->SUPER::run($test, $undo); - } - - # otherwise nothing to see here -} - - - -1; diff --git a/Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm b/Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm new file mode 100644 index 0000000000..abaaa43176 --- /dev/null +++ b/Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm @@ -0,0 +1,124 @@ +package ApiCommonWorkflow::Main::WorkflowSteps::RunNgsSamplesAndNextflow; + +@ISA = (ReFlow::StepClasses::RunAndMonitorNextflow); + +use strict; +use warnings; +use ReFlow::StepClasses::RunAndMonitorNextflow; + +# ngs-samples-nextflow fetches the reads; the analysis workflow then consumes them. +# Both run from this single step so the pair holds one throttle slot: as separate +# steps the analysis had to re-queue behind other experiments' fetches. +sub ngsSamplesWorkflow { return "VEuPathDB/ngs-samples-nextflow" } + +# Each run keeps its own job info, log, trace and stdout files. Crash recovery +# depends on that: a re-entered step detects an already-finished run from its log +# and moves on to the next one. +sub nextflowRuns { + my ($self) = @_; + + my @runs; + + # optional. analyses with no samples to fetch leave it empty + my $ngsSamplesConfigFile = $self->getParamValue("ngsSamplesNextflowConfigFile"); + + push(@runs, { label => "ngs-samples", + workflow => $self->ngsSamplesWorkflow(), + configFile => $ngsSamplesConfigFile, + resultsDir => $self->getParamValue("ngsSamplesResultsDir"), + entry => "", + clusterJobInfoFileName => "ngs-samples-clusterJobInfo.txt", + logFileName => "ngs-samples-nextflow.log", + traceFileName => "ngs-samples-trace.txt", + nextflowStdoutFileName => "ngs-samples-nextflow.txt", + }) if $ngsSamplesConfigFile; + + push(@runs, { label => $self->getParamValue("nextflowWorkflow"), + workflow => $self->getParamValue("nextflowWorkflow"), + configFile => $self->getParamValue("nextflowConfigFile"), + resultsDir => $self->getParamValue("resultsDir"), + entry => $self->getParamValue("entry"), + clusterJobInfoFileName => $self->clusterJobInfoFileName(), + logFileName => $self->logFileName(), + traceFileName => $self->traceFileName(), + nextflowStdoutFileName => $self->nextflowStdoutFileName(), + }); + + return @runs; +} + +sub run { + my ($self, $test, $undo) = @_; + + my $clusterServer = $self->getSharedConfig('clusterServer'); + my $clusterTransferServer = $self->getSharedConfig('clusterFileTransferServer'); + my $userName = $self->getSharedConfig("$clusterServer.clusterLogin"); + my $clusterQueue = $self->getSharedConfig("$clusterServer.clusterQueue"); + my $maxTimeMins = $self->getSharedConfig("$clusterServer.maxAllowedRuntimeDays") * 24 * 60; + + my $isGitRepo = $self->getBooleanParamValue("isGitRepo"); + + my $workingDirRelativePath = $self->getParamValue("workingDirRelativePath"); + my $clusterWorkingDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath, + $self->getParamValue("workingDir")); + + foreach my $nextflowRun ($self->nextflowRuns()) { + + my $clusterResultsDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath, + $nextflowRun->{resultsDir}); + my $clusterNextflowConfigFile = $self->relativePathToNextflowClusterPath($workingDirRelativePath, + $nextflowRun->{configFile}); + + my $jobInfoFile = "$clusterWorkingDir/" . $nextflowRun->{clusterJobInfoFileName}; + my $logFile = "$clusterWorkingDir/" . $nextflowRun->{logFileName}; + my $traceFile = "$clusterWorkingDir/" . $nextflowRun->{traceFileName}; + my $nextflowStdoutFile = "$clusterWorkingDir/" . $nextflowRun->{nextflowStdoutFileName}; + + if ($undo) { + $self->runCmdOnClusterTransferServer(0, "rm -fr $clusterWorkingDir/work"); + $self->runCmdOnClusterTransferServer(0, "rm -fr $clusterResultsDir/*"); + $self->runCmdOnClusterTransferServer(0, "rm -fr $traceFile"); + $self->runCmdOnClusterTransferServer(0, "rm -fr $logFile"); + $self->runCmdOnClusterTransferServer(0, "rm -fr $nextflowStdoutFile"); + $self->log("Removing log file at: $logFile"); + next; + } + + my $success = $self->runAndMonitor($test, $userName, $clusterServer, $clusterTransferServer, + $jobInfoFile, $logFile, $nextflowStdoutFile, $clusterWorkingDir, + $maxTimeMins, $clusterQueue, $nextflowRun->{workflow}, $isGitRepo, + $clusterNextflowConfigFile, $nextflowRun->{entry}); + + $self->error($self->nextflowFailureMessage($nextflowRun->{label}, $logFile, $traceFile)) unless $success; + + # remove the work directory + $self->runCmdOnClusterTransferServer(0, "rm -fr $clusterWorkingDir/work"); + } +} + +sub nextflowFailureMessage { + my ($self, $label, $logFile, $traceFile) = @_; + + return +" +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +The '$label' nextflow run in this step did not successfully run. Check its task +log file on the cluster: + $logFile + +If the task log file ends in a perl error, that suggests an unusual controller failure. Often those are recoverable by setting the step to ready and trying again. + +Otherwise, to diagnose the problem, look in the scheduler and nextflow step logs to see what command is executed on the nodes. Find those logs at: + $traceFile + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +"; +} + +sub getConfigDeclaration { + return ( + # [name, default, description] + ); +} + +1; diff --git a/Main/lib/xml/workflow/runNextflowOnCluster.xml b/Main/lib/xml/workflow/runNextflowOnCluster.xml index 7c98c3248c..237fbc237a 100644 --- a/Main/lib/xml/workflow/runNextflowOnCluster.xml +++ b/Main/lib/xml/workflow/runNextflowOnCluster.xml @@ -33,29 +33,20 @@ - - - $$parentDataDir$$ - $$analysisDir$$ - $$analysisDir$$/ngs-samples-results - $$ngsSamplesNextflowConfigFile$$ - VEuPathDB/ngs-samples-nextflow - true - $$organismAbbrev$$ - - - - - + + $$parentDataDir$$ $$analysisDir$$ $$analysisDir$$/results $$nextflowConfigFile$$ $$nextflowWorkflow$$ + $$analysisDir$$/ngs-samples-results + $$ngsSamplesNextflowConfigFile$$ true $$organismAbbrev$$ $$entry$$ - diff --git a/Main/t/run_ngs_samples_and_nextflow.t b/Main/t/run_ngs_samples_and_nextflow.t new file mode 100644 index 0000000000..53f4ce413c --- /dev/null +++ b/Main/t/run_ngs_samples_and_nextflow.t @@ -0,0 +1,59 @@ +use strict; +use warnings; + +use lib "$ENV{GUS_HOME}/lib/perl"; + +use ApiCommonWorkflow::Main::WorkflowSteps::RunNgsSamplesAndNextflow; +use Test::More; + +# the step class only needs paramValues to build its run list +package TestStep; +our @ISA = ('ApiCommonWorkflow::Main::WorkflowSteps::RunNgsSamplesAndNextflow'); +sub new { my ($class, $paramValues) = @_; return bless({paramValues => $paramValues}, $class) } +sub log { } + +package main; + +my %params = (nextflowWorkflow => 'VEuPathDB/dnaseq-nextflow', + nextflowConfigFile => '/data/exp/dnaseqNextflow/analysisDir/nextflow.config', + resultsDir => '/data/exp/dnaseqNextflow/analysisDir/results', + entry => 'processSingleExperiment', + ngsSamplesNextflowConfigFile => '/data/exp/dnaseqNextflow/analysisDir/ngs-samples-nextflow.config', + ngsSamplesResultsDir => '/data/exp/dnaseqNextflow/analysisDir/ngs-samples-results'); + +my @runs = TestStep->new({%params})->nextflowRuns(); + +is(scalar(@runs), 2, "ngs-samples run precedes the analysis run"); + +is($runs[0]->{workflow}, 'VEuPathDB/ngs-samples-nextflow', "ngs-samples workflow"); +is($runs[0]->{entry}, '', "ngs-samples takes no entry"); +is($runs[0]->{configFile}, $params{ngsSamplesNextflowConfigFile}, "ngs-samples config"); +is($runs[0]->{resultsDir}, $params{ngsSamplesResultsDir}, "ngs-samples results dir"); + +is($runs[1]->{workflow}, 'VEuPathDB/dnaseq-nextflow', "analysis workflow"); +is($runs[1]->{entry}, 'processSingleExperiment', "analysis entry"); +is($runs[1]->{configFile}, $params{nextflowConfigFile}, "analysis config"); + +# the on-cluster file names must not change: crash recovery finds an already +# finished run by its own log, and in-flight runs must stay resumable +is($runs[0]->{clusterJobInfoFileName}, 'ngs-samples-clusterJobInfo.txt', "ngs job info file"); +is($runs[0]->{logFileName}, 'ngs-samples-nextflow.log', "ngs log file"); +is($runs[0]->{traceFileName}, 'ngs-samples-trace.txt', "ngs trace file"); +is($runs[0]->{nextflowStdoutFileName}, 'ngs-samples-nextflow.txt', "ngs stdout file"); + +is($runs[1]->{clusterJobInfoFileName}, 'clusterJobInfo.txt', "analysis job info file"); +is($runs[1]->{logFileName}, 'nextflow.log', "analysis log file"); +is($runs[1]->{traceFileName}, 'trace.txt', "analysis trace file"); +is($runs[1]->{nextflowStdoutFileName}, 'nextflow.txt', "analysis stdout file"); + +# no distinct log files would mean run 2 could never resume past run 1 +isnt($runs[0]->{logFileName}, $runs[1]->{logFileName}, "each run gets its own log"); +isnt($runs[0]->{clusterJobInfoFileName}, $runs[1]->{clusterJobInfoFileName}, "each run gets its own job info"); + +# the 24 annotation callers pass no ngs-samples config: they must run one workflow +my @oneRun = TestStep->new({%params, ngsSamplesNextflowConfigFile => ''})->nextflowRuns(); +is(scalar(@oneRun), 1, "empty ngs-samples config drops that run"); +is($oneRun[0]->{workflow}, 'VEuPathDB/dnaseq-nextflow', "the remaining run is the analysis"); +is($oneRun[0]->{logFileName}, 'nextflow.log', "and it keeps the unprefixed log name"); + +done_testing; From 35f11195a5baafedfa12871604c8f8e680a16c25 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Fri, 4 Sep 2026 16:46:52 -0400 Subject: [PATCH 4/9] Give each ngs assay type its own throttle bucket for its nextflow run runNextflowOnCluster.xml is shared by 30 callers, so the runClusterTask step it declares can only name one bucket for all of them. stepLoadTypes takes no variables -- neither substituteValues nor substituteMacros touches the attribute -- so tag the step from each call site instead, which is what the path form of stepLoadTypes on a is for. Load types AND together in okToRun, so these run alongside the shared runNextflowOnCluster cap rather than replacing it. The 24 callers with no samples to fetch add nothing and are unaffected. REQUIRES loadThrottle.prop in each workflow config to define these keys, since checkLoadTypes fails the graph build on an unknown stepLoadType: dnaseqNextflow, rnaseqNextflow, longReadRnaSeqNextflow, chipseqNextflow, originsNextflow, splicedLeaderNextflow Co-Authored-By: Claude Opus 5 (1M context) --- Main/lib/xml/workflow/bulkrnaseq.xml | 2 +- Main/lib/xml/workflow/longReadRnaSeq.xml | 2 +- Main/lib/xml/workflow/processChipSeqExperiment.xml | 2 +- Main/lib/xml/workflow/processOriginsExperiment.xml | 2 +- .../xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml | 2 +- Main/lib/xml/workflow/snpAndCnvDNASeq.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Main/lib/xml/workflow/bulkrnaseq.xml b/Main/lib/xml/workflow/bulkrnaseq.xml index d091c85a24..302a22eed5 100644 --- a/Main/lib/xml/workflow/bulkrnaseq.xml +++ b/Main/lib/xml/workflow/bulkrnaseq.xml @@ -92,7 +92,7 @@ - + $$gusConfigFile$$ $$projectName$$ $$dataDir$$ diff --git a/Main/lib/xml/workflow/longReadRnaSeq.xml b/Main/lib/xml/workflow/longReadRnaSeq.xml index 97bbe5a204..fe35dd8e2c 100644 --- a/Main/lib/xml/workflow/longReadRnaSeq.xml +++ b/Main/lib/xml/workflow/longReadRnaSeq.xml @@ -96,7 +96,7 @@ - + $$gusConfigFile$$ $$projectName$$ $$dataDir$$ diff --git a/Main/lib/xml/workflow/processChipSeqExperiment.xml b/Main/lib/xml/workflow/processChipSeqExperiment.xml index cbe12c9e16..04e335904d 100644 --- a/Main/lib/xml/workflow/processChipSeqExperiment.xml +++ b/Main/lib/xml/workflow/processChipSeqExperiment.xml @@ -83,7 +83,7 @@ - + $$gusConfigFile$$ $$projectName$$ $$dataDir$$ diff --git a/Main/lib/xml/workflow/processOriginsExperiment.xml b/Main/lib/xml/workflow/processOriginsExperiment.xml index ce8648934b..9430797d95 100644 --- a/Main/lib/xml/workflow/processOriginsExperiment.xml +++ b/Main/lib/xml/workflow/processOriginsExperiment.xml @@ -81,7 +81,7 @@ - + $$gusConfigFile$$ $$projectName$$ $$dataDir$$ diff --git a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml index 47b68caa16..27acaa6458 100644 --- a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml +++ b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml @@ -76,7 +76,7 @@ - + $$gusConfigFile$$ $$projectName$$ $$dataDir$$ diff --git a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml index b8290b07c5..2961db8bdb 100644 --- a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml +++ b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml @@ -131,7 +131,7 @@ - + $$gusConfigFile$$ $$projectName$$ $$dataDir$$ From 12d5f52f8dbcc7712802ce52a5dd10ab060c781f Mon Sep 17 00:00:00 2001 From: Sufen Hu Date: Tue, 8 Sep 2026 11:56:12 -0400 Subject: [PATCH 5/9] ignore depends name="makeDataDir", because it is not need anymore --- Main/lib/xml/workflow/mapWithBowtie.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/Main/lib/xml/workflow/mapWithBowtie.xml b/Main/lib/xml/workflow/mapWithBowtie.xml index 8440b065ff..f79af6950d 100644 --- a/Main/lib/xml/workflow/mapWithBowtie.xml +++ b/Main/lib/xml/workflow/mapWithBowtie.xml @@ -17,7 +17,6 @@ $$relativeWebServicesDir$$ $$genomeExtDbRlsSpec$$ $$genomeFastaFile$$ - @@ -57,7 +56,6 @@ $$relativeWebServicesDir$$ $$genomeExtDbRlsSpec$$ $$genomeFastaFile$$ - @@ -71,6 +69,5 @@ $$genomeExtDbRlsSpec$$ $$gusConfigFile$$ $$genomeFastaFile$$ - From 5e0cd0f69dff0045f036518049ebbd20fe7b1151 Mon Sep 17 00:00:00 2001 From: John Brestelli Date: Tue, 8 Sep 2026 13:22:02 -0400 Subject: [PATCH 6/9] add queue for ngssamples --- Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm index 0b7c96e5e2..ebf08c21c7 100644 --- a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm @@ -11,6 +11,8 @@ sub run { #NOTE: the subset size here would run "X" number of genomic sequences at a time on the cluster (chromosomes or contigs) my $fastaSubsetSize = 5; + my $queue = $self->getClusterQueue(); + my $finalDir = $self->getParamValue("finalDirectory"); my $resultsDirectory = $self->getParamValue("resultsDirectory"); my $analysisDirectory = $self->getParamValue("analysisDirectory"); @@ -96,6 +98,7 @@ params { process { maxForks = $maxForks + queue = \'$queue\' } includeConfig "$clusterConfigFile" From 2184fe60d9cddc27331b21ea9bca887cdb9403d8 Mon Sep 17 00:00:00 2001 From: rdemko2332 Date: Tue, 8 Sep 2026 13:50:19 -0400 Subject: [PATCH 7/9] Adding module load apptainer line to avoid FIPS error --- Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm index ebf08c21c7..5af18b42e3 100644 --- a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm @@ -97,6 +97,7 @@ params { } process { + beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH' maxForks = $maxForks queue = \'$queue\' } From 48bfa88564a38c910254af32d265a11bb550df05 Mon Sep 17 00:00:00 2001 From: rdemko2332 Date: Tue, 8 Sep 2026 16:46:31 -0400 Subject: [PATCH 8/9] Adding module load line to dnaseq nextflow configs --- Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm | 1 + .../WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm | 1 + 2 files changed, 2 insertions(+) diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm index 67a8d301c6..dca4fc578c 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm @@ -76,6 +76,7 @@ params { } process { + beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH' executor = '$executor' queue = '$queue' } diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm index 77d500edbc..8677c8ea31 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm @@ -101,6 +101,7 @@ params { } process { + beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH' executor = \'$executor\' queue = \'$queue\' withName: 'gatk' { From 45ed999705642bdbd4b1020e41aa4af07642ebfc Mon Sep 17 00:00:00 2001 From: rdemko2332 Date: Tue, 8 Sep 2026 16:49:51 -0400 Subject: [PATCH 9/9] Adding module load line to dnaseq nextflow configs --- .../WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm index 96826797f3..cc13413f36 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm @@ -85,6 +85,7 @@ params { // so maxForks only gates the few independent early branches. Serialize them to // keep peak memory flat (snpEff JVM, Julia processSeqVars). process { + beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH' maxForks = 1 }