diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm
index 96826797f3..cc13413f36 100644
--- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm
+++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm
@@ -85,6 +85,7 @@ params {
// so maxForks only gates the few independent early branches. Serialize them to
// keep peak memory flat (snpEff JVM, Julia processSeqVars).
process {
+ beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH'
maxForks = 1
}
diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm
index 67a8d301c6..dca4fc578c 100644
--- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm
+++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm
@@ -76,6 +76,7 @@ params {
}
process {
+ beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH'
executor = '$executor'
queue = '$queue'
}
diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm
index 77d500edbc..8677c8ea31 100644
--- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm
+++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm
@@ -101,6 +101,7 @@ params {
}
process {
+ beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH'
executor = \'$executor\'
queue = \'$queue\'
withName: 'gatk' {
diff --git a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm
index 57dae80bb8..6add6cf347 100644
--- a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm
+++ b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm
@@ -11,6 +11,8 @@ sub run {
#NOTE: the subset size here would run "X" number of genomic sequences at a time on the cluster (chromosomes or contigs)
my $fastaSubsetSize = 5;
+ my $queue = $self->getClusterQueue();
+
my $finalDir = $self->getParamValue("finalDirectory");
my $resultsDirectory = $self->getParamValue("resultsDirectory");
my $analysisDirectory = $self->getParamValue("analysisDirectory");
@@ -18,7 +20,7 @@ sub run {
my $nextflowConfigFile = $self->getParamValue("nextflowConfigFile");
my $sampleSheetName = $self->getParamValue("sampleSheetName");
my $assayType = $self->getParamValue("assayType");
- my $organismAbbrev = $self->getParamValue('organismAbbrev');
+ my $genomeFile = $self->getParamValue("genomeFile");
my $fromSRA = $self->getBooleanParamValue("fromSRA") ? "true" : "false";
# Max SRA file size prefetch will download. If a data load fails because prefetch
@@ -26,14 +28,21 @@ sub run {
# config on the cluster and re-run (default sra-tools limit is 20G).
my $maxDownloadSize = "50G";
- my $gusConfig = $self->getWorkflowDataDir() . "/" . $self->getParamValue('gusConfigFile');
- my $genomeSize = $self->getGenomeSize($test, $organismAbbrev, $gusConfig);
+ # Depth gate. minOnTargetFraction doubles as the inflation cap: a 0.05 floor means a
+ # contaminated sample is asked for at most 20x the reads a clean one would get.
+ # Below minPlausibleFraction the sample is flagged as probably-wrong-genome, not failed.
+ my $targetCoverage = 60;
+ my $minOnTargetFraction = 0.05;
+ my $minPlausibleFraction = 0.01;
+ my $pilotSize = 100000;
+
my $workflowDataDir = $self->getWorkflowDataDir();
my $workingDirRelativePath = $self->getParamValue("workingDirRelativePath");
my $digestedFinalDirPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $finalDir);
my $digestedAnalysisDirPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $analysisDirectory);
my $digestedOutputDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $resultsDirectory);
+ my $digestedReferenceFasta = $self->relativePathToNextflowClusterPath($workingDirRelativePath, $genomeFile);
my $clusterServer = $self->getSharedConfig('clusterServer');
my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir();
@@ -79,14 +88,18 @@ params {
samplesheetName = "$sampleSheetName"
fromSra = $fromSRA
outDir = "$digestedOutputDir"
- genomeSize = $genomeSize
+ referenceFasta = "$digestedReferenceFasta"
assayType = "$assayType"
+ targetCoverage = $targetCoverage
+ minOnTargetFraction = $minOnTargetFraction
+ minPlausibleFraction = $minPlausibleFraction
+ pilotSize = $pilotSize
maxDownloadSize = "$maxDownloadSize"
}
process {
- queue = '$queue'
maxForks = $maxForks
+ queue = \'$queue\'
}
includeConfig "$clusterConfigFile"
diff --git a/Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm b/Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm
deleted file mode 100644
index f61013c051..0000000000
--- a/Main/lib/perl/WorkflowSteps/RunAndMonitorNextflowNGSSamples.pm
+++ /dev/null
@@ -1,30 +0,0 @@
-package ApiCommonWorkflow::Main::WorkflowSteps::RunAndMonitorNextflowNGSSamples;
-
-@ISA = (ReFlow::StepClasses::RunAndMonitorNextflow);
-
-use strict;
-use ReFlow::StepClasses::RunAndMonitorNextflow;
-
-sub clusterJobInfoFileName { return "ngs-samples-clusterJobInfo.txt"}
-sub logFileName { return "ngs-samples-nextflow.log" }
-sub traceFileName { return "ngs-samples-trace.txt" }
-sub nextflowStdoutFileName { return "ngs-samples-nextflow.txt" }
-
-
-sub run {
- my ($self, $test, $undo) = @_;
-
- my $nextflowConfigFile = $self->getParamValue('nextflowConfigFile');
- my $entry = $self->getParamValue('entry');
-
- # if the xml has set this config file.. run the cluster job
- if($nextflowConfigFile ne "") {
- $self->SUPER::run($test, $undo);
- }
-
- # otherwise nothing to see here
-}
-
-
-
-1;
diff --git a/Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm b/Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm
new file mode 100644
index 0000000000..abaaa43176
--- /dev/null
+++ b/Main/lib/perl/WorkflowSteps/RunNgsSamplesAndNextflow.pm
@@ -0,0 +1,124 @@
+package ApiCommonWorkflow::Main::WorkflowSteps::RunNgsSamplesAndNextflow;
+
+@ISA = (ReFlow::StepClasses::RunAndMonitorNextflow);
+
+use strict;
+use warnings;
+use ReFlow::StepClasses::RunAndMonitorNextflow;
+
+# ngs-samples-nextflow fetches the reads; the analysis workflow then consumes them.
+# Both run from this single step so the pair holds one throttle slot: as separate
+# steps the analysis had to re-queue behind other experiments' fetches.
+sub ngsSamplesWorkflow { return "VEuPathDB/ngs-samples-nextflow" }
+
+# Each run keeps its own job info, log, trace and stdout files. Crash recovery
+# depends on that: a re-entered step detects an already-finished run from its log
+# and moves on to the next one.
+sub nextflowRuns {
+ my ($self) = @_;
+
+ my @runs;
+
+ # optional. analyses with no samples to fetch leave it empty
+ my $ngsSamplesConfigFile = $self->getParamValue("ngsSamplesNextflowConfigFile");
+
+ push(@runs, { label => "ngs-samples",
+ workflow => $self->ngsSamplesWorkflow(),
+ configFile => $ngsSamplesConfigFile,
+ resultsDir => $self->getParamValue("ngsSamplesResultsDir"),
+ entry => "",
+ clusterJobInfoFileName => "ngs-samples-clusterJobInfo.txt",
+ logFileName => "ngs-samples-nextflow.log",
+ traceFileName => "ngs-samples-trace.txt",
+ nextflowStdoutFileName => "ngs-samples-nextflow.txt",
+ }) if $ngsSamplesConfigFile;
+
+ push(@runs, { label => $self->getParamValue("nextflowWorkflow"),
+ workflow => $self->getParamValue("nextflowWorkflow"),
+ configFile => $self->getParamValue("nextflowConfigFile"),
+ resultsDir => $self->getParamValue("resultsDir"),
+ entry => $self->getParamValue("entry"),
+ clusterJobInfoFileName => $self->clusterJobInfoFileName(),
+ logFileName => $self->logFileName(),
+ traceFileName => $self->traceFileName(),
+ nextflowStdoutFileName => $self->nextflowStdoutFileName(),
+ });
+
+ return @runs;
+}
+
+sub run {
+ my ($self, $test, $undo) = @_;
+
+ my $clusterServer = $self->getSharedConfig('clusterServer');
+ my $clusterTransferServer = $self->getSharedConfig('clusterFileTransferServer');
+ my $userName = $self->getSharedConfig("$clusterServer.clusterLogin");
+ my $clusterQueue = $self->getSharedConfig("$clusterServer.clusterQueue");
+ my $maxTimeMins = $self->getSharedConfig("$clusterServer.maxAllowedRuntimeDays") * 24 * 60;
+
+ my $isGitRepo = $self->getBooleanParamValue("isGitRepo");
+
+ my $workingDirRelativePath = $self->getParamValue("workingDirRelativePath");
+ my $clusterWorkingDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath,
+ $self->getParamValue("workingDir"));
+
+ foreach my $nextflowRun ($self->nextflowRuns()) {
+
+ my $clusterResultsDir = $self->relativePathToNextflowClusterPath($workingDirRelativePath,
+ $nextflowRun->{resultsDir});
+ my $clusterNextflowConfigFile = $self->relativePathToNextflowClusterPath($workingDirRelativePath,
+ $nextflowRun->{configFile});
+
+ my $jobInfoFile = "$clusterWorkingDir/" . $nextflowRun->{clusterJobInfoFileName};
+ my $logFile = "$clusterWorkingDir/" . $nextflowRun->{logFileName};
+ my $traceFile = "$clusterWorkingDir/" . $nextflowRun->{traceFileName};
+ my $nextflowStdoutFile = "$clusterWorkingDir/" . $nextflowRun->{nextflowStdoutFileName};
+
+ if ($undo) {
+ $self->runCmdOnClusterTransferServer(0, "rm -fr $clusterWorkingDir/work");
+ $self->runCmdOnClusterTransferServer(0, "rm -fr $clusterResultsDir/*");
+ $self->runCmdOnClusterTransferServer(0, "rm -fr $traceFile");
+ $self->runCmdOnClusterTransferServer(0, "rm -fr $logFile");
+ $self->runCmdOnClusterTransferServer(0, "rm -fr $nextflowStdoutFile");
+ $self->log("Removing log file at: $logFile");
+ next;
+ }
+
+ my $success = $self->runAndMonitor($test, $userName, $clusterServer, $clusterTransferServer,
+ $jobInfoFile, $logFile, $nextflowStdoutFile, $clusterWorkingDir,
+ $maxTimeMins, $clusterQueue, $nextflowRun->{workflow}, $isGitRepo,
+ $clusterNextflowConfigFile, $nextflowRun->{entry});
+
+ $self->error($self->nextflowFailureMessage($nextflowRun->{label}, $logFile, $traceFile)) unless $success;
+
+ # remove the work directory
+ $self->runCmdOnClusterTransferServer(0, "rm -fr $clusterWorkingDir/work");
+ }
+}
+
+sub nextflowFailureMessage {
+ my ($self, $label, $logFile, $traceFile) = @_;
+
+ return
+"
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+The '$label' nextflow run in this step did not successfully run. Check its task
+log file on the cluster:
+ $logFile
+
+If the task log file ends in a perl error, that suggests an unusual controller failure. Often those are recoverable by setting the step to ready and trying again.
+
+Otherwise, to diagnose the problem, look in the scheduler and nextflow step logs to see what command is executed on the nodes. Find those logs at:
+ $traceFile
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+";
+}
+
+sub getConfigDeclaration {
+ return (
+ # [name, default, description]
+ );
+}
+
+1;
diff --git a/Main/lib/xml/workflow/bulkrnaseq.xml b/Main/lib/xml/workflow/bulkrnaseq.xml
index dab70476ba..302a22eed5 100644
--- a/Main/lib/xml/workflow/bulkrnaseq.xml
+++ b/Main/lib/xml/workflow/bulkrnaseq.xml
@@ -64,7 +64,6 @@
- $$gusConfigFile$$
$$analysisDirectory$$
$$finalSymLink$$
$$analysisDirectory$$/ngs-samples-results
@@ -73,8 +72,9 @@
samplesheet.csv
$$fromSRA$$
RNASeq
- $$organismAbbrev$$
+ $$genomeSymLink$$
+
@@ -92,7 +92,7 @@
-
+
$$gusConfigFile$$
$$projectName$$
$$dataDir$$
diff --git a/Main/lib/xml/workflow/longReadRnaSeq.xml b/Main/lib/xml/workflow/longReadRnaSeq.xml
index 6ff229b780..fe35dd8e2c 100644
--- a/Main/lib/xml/workflow/longReadRnaSeq.xml
+++ b/Main/lib/xml/workflow/longReadRnaSeq.xml
@@ -63,7 +63,6 @@
- $$gusConfigFile$$
$$analysisDirectory$$
$$finalSymLink$$
$$analysisDirectory$$/ngs-samples-results
@@ -72,8 +71,9 @@
samplesheet.csv
$$fromSRA$$
RNASeq
- $$organismAbbrev$$
+ $$genomeSymLink$$
+
@@ -96,7 +96,7 @@
-
+
$$gusConfigFile$$
$$projectName$$
$$dataDir$$
diff --git a/Main/lib/xml/workflow/mapWithBowtie.xml b/Main/lib/xml/workflow/mapWithBowtie.xml
index 8440b065ff..f79af6950d 100644
--- a/Main/lib/xml/workflow/mapWithBowtie.xml
+++ b/Main/lib/xml/workflow/mapWithBowtie.xml
@@ -17,7 +17,6 @@
$$relativeWebServicesDir$$
$$genomeExtDbRlsSpec$$
$$genomeFastaFile$$
-
@@ -57,7 +56,6 @@
$$relativeWebServicesDir$$
$$genomeExtDbRlsSpec$$
$$genomeFastaFile$$
-
@@ -71,6 +69,5 @@
$$genomeExtDbRlsSpec$$
$$gusConfigFile$$
$$genomeFastaFile$$
-
diff --git a/Main/lib/xml/workflow/processChipSeqExperiment.xml b/Main/lib/xml/workflow/processChipSeqExperiment.xml
index 6598e10dba..04e335904d 100644
--- a/Main/lib/xml/workflow/processChipSeqExperiment.xml
+++ b/Main/lib/xml/workflow/processChipSeqExperiment.xml
@@ -53,7 +53,6 @@
- $$gusConfigFile$$
$$analysisDirectory$$
$$finalSymLink$$
$$analysisDirectory$$/ngs-samples-results
@@ -62,8 +61,9 @@
samplesheet.csv
$$fromSRA$$
ChipSeq
- $$organismAbbrev$$
+ $$genomeSymLink$$
+
@@ -83,7 +83,7 @@
-
+
$$gusConfigFile$$
$$projectName$$
$$dataDir$$
diff --git a/Main/lib/xml/workflow/processOriginsExperiment.xml b/Main/lib/xml/workflow/processOriginsExperiment.xml
index ab0f9baf87..9430797d95 100644
--- a/Main/lib/xml/workflow/processOriginsExperiment.xml
+++ b/Main/lib/xml/workflow/processOriginsExperiment.xml
@@ -53,7 +53,6 @@
- $$gusConfigFile$$
$$analysisDirectory$$
$$finalSymLink$$
$$analysisDirectory$$/ngs-samples-results
@@ -62,8 +61,9 @@
samplesheet.csv
$$fromSRA$$
DNASeq
- $$organismAbbrev$$
+ $$genomeSymLink$$
+
@@ -81,7 +81,7 @@
-
+
$$gusConfigFile$$
$$projectName$$
$$dataDir$$
diff --git a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml
index 4b960b45ea..27acaa6458 100644
--- a/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml
+++ b/Main/lib/xml/workflow/rnaSeqSplicedLeaderAndPolyASitesExperiment.xml
@@ -50,7 +50,6 @@
- $$gusConfigFile$$
$$analysisDirectory$$
$$finalSymLink$$
$$analysisDirectory$$/ngs-samples-results
@@ -59,8 +58,9 @@
samplesheet.csv
$$fromSRA$$
RNASeq
- $$organismAbbrev$$
+ $$genomeSymLink$$
+
@@ -76,7 +76,7 @@
-
+
$$gusConfigFile$$
$$projectName$$
$$dataDir$$
diff --git a/Main/lib/xml/workflow/runNextflowOnCluster.xml b/Main/lib/xml/workflow/runNextflowOnCluster.xml
index 7c98c3248c..237fbc237a 100644
--- a/Main/lib/xml/workflow/runNextflowOnCluster.xml
+++ b/Main/lib/xml/workflow/runNextflowOnCluster.xml
@@ -33,29 +33,20 @@
-
-
- $$parentDataDir$$
- $$analysisDir$$
- $$analysisDir$$/ngs-samples-results
- $$ngsSamplesNextflowConfigFile$$
- VEuPathDB/ngs-samples-nextflow
- true
- $$organismAbbrev$$
-
-
-
-
-
+
+
$$parentDataDir$$
$$analysisDir$$
$$analysisDir$$/results
$$nextflowConfigFile$$
$$nextflowWorkflow$$
+ $$analysisDir$$/ngs-samples-results
+ $$ngsSamplesNextflowConfigFile$$
true
$$organismAbbrev$$
$$entry$$
-
diff --git a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml
index 9cc41e8bda..2961db8bdb 100644
--- a/Main/lib/xml/workflow/snpAndCnvDNASeq.xml
+++ b/Main/lib/xml/workflow/snpAndCnvDNASeq.xml
@@ -94,7 +94,6 @@
- $$gusConfigFile$$
$$analysisDirectory$$
$$finalSymLink$$
$$analysisDirectory$$/ngs-samples-results
@@ -103,8 +102,9 @@
samplesheet.csv
$$fromSRA$$
DNASeq
- $$organismAbbrev$$
+ $$genomeSymLink$$
+
@@ -131,7 +131,7 @@
-
+
$$gusConfigFile$$
$$projectName$$
$$dataDir$$
diff --git a/Main/t/run_ngs_samples_and_nextflow.t b/Main/t/run_ngs_samples_and_nextflow.t
new file mode 100644
index 0000000000..53f4ce413c
--- /dev/null
+++ b/Main/t/run_ngs_samples_and_nextflow.t
@@ -0,0 +1,59 @@
+use strict;
+use warnings;
+
+use lib "$ENV{GUS_HOME}/lib/perl";
+
+use ApiCommonWorkflow::Main::WorkflowSteps::RunNgsSamplesAndNextflow;
+use Test::More;
+
+# the step class only needs paramValues to build its run list
+package TestStep;
+our @ISA = ('ApiCommonWorkflow::Main::WorkflowSteps::RunNgsSamplesAndNextflow');
+sub new { my ($class, $paramValues) = @_; return bless({paramValues => $paramValues}, $class) }
+sub log { }
+
+package main;
+
+my %params = (nextflowWorkflow => 'VEuPathDB/dnaseq-nextflow',
+ nextflowConfigFile => '/data/exp/dnaseqNextflow/analysisDir/nextflow.config',
+ resultsDir => '/data/exp/dnaseqNextflow/analysisDir/results',
+ entry => 'processSingleExperiment',
+ ngsSamplesNextflowConfigFile => '/data/exp/dnaseqNextflow/analysisDir/ngs-samples-nextflow.config',
+ ngsSamplesResultsDir => '/data/exp/dnaseqNextflow/analysisDir/ngs-samples-results');
+
+my @runs = TestStep->new({%params})->nextflowRuns();
+
+is(scalar(@runs), 2, "ngs-samples run precedes the analysis run");
+
+is($runs[0]->{workflow}, 'VEuPathDB/ngs-samples-nextflow', "ngs-samples workflow");
+is($runs[0]->{entry}, '', "ngs-samples takes no entry");
+is($runs[0]->{configFile}, $params{ngsSamplesNextflowConfigFile}, "ngs-samples config");
+is($runs[0]->{resultsDir}, $params{ngsSamplesResultsDir}, "ngs-samples results dir");
+
+is($runs[1]->{workflow}, 'VEuPathDB/dnaseq-nextflow', "analysis workflow");
+is($runs[1]->{entry}, 'processSingleExperiment', "analysis entry");
+is($runs[1]->{configFile}, $params{nextflowConfigFile}, "analysis config");
+
+# the on-cluster file names must not change: crash recovery finds an already
+# finished run by its own log, and in-flight runs must stay resumable
+is($runs[0]->{clusterJobInfoFileName}, 'ngs-samples-clusterJobInfo.txt', "ngs job info file");
+is($runs[0]->{logFileName}, 'ngs-samples-nextflow.log', "ngs log file");
+is($runs[0]->{traceFileName}, 'ngs-samples-trace.txt', "ngs trace file");
+is($runs[0]->{nextflowStdoutFileName}, 'ngs-samples-nextflow.txt', "ngs stdout file");
+
+is($runs[1]->{clusterJobInfoFileName}, 'clusterJobInfo.txt', "analysis job info file");
+is($runs[1]->{logFileName}, 'nextflow.log', "analysis log file");
+is($runs[1]->{traceFileName}, 'trace.txt', "analysis trace file");
+is($runs[1]->{nextflowStdoutFileName}, 'nextflow.txt', "analysis stdout file");
+
+# no distinct log files would mean run 2 could never resume past run 1
+isnt($runs[0]->{logFileName}, $runs[1]->{logFileName}, "each run gets its own log");
+isnt($runs[0]->{clusterJobInfoFileName}, $runs[1]->{clusterJobInfoFileName}, "each run gets its own job info");
+
+# the 24 annotation callers pass no ngs-samples config: they must run one workflow
+my @oneRun = TestStep->new({%params, ngsSamplesNextflowConfigFile => ''})->nextflowRuns();
+is(scalar(@oneRun), 1, "empty ngs-samples config drops that run");
+is($oneRun[0]->{workflow}, 'VEuPathDB/dnaseq-nextflow', "the remaining run is the analysis");
+is($oneRun[0]->{logFileName}, 'nextflow.log', "and it keeps the unprefixed log name");
+
+done_testing;