From 1dfac52fad1c44c861e40ccdeffd7aa07860b0e2 Mon Sep 17 00:00:00 2001 From: rdemko2332 Date: Wed, 9 Sep 2026 15:22:27 -0400 Subject: [PATCH 1/3] Add shared, opt-in cluster nextflow config via includeConfig New WorkflowStep.pm helper getSharedClusterNextflowConfigIncludeBlock(): reads $clusterServer.sharedNextflowConfig from stepsShared.prop (via getSharedConfigRelaxed, so an unset value is a silent no-op) and, if set, returns an includeConfig line pulling in that cluster-provided config file. Lets a cluster-wide nextflow process{} setting (e.g. beforeScript = 'module load apptainer/1.4.1 && unset LD_LIBRARY_PATH') be rolled out via a one-line stepsShared.prop change instead of hand-editing every Make*NextflowConfig.pm generator -- 37 of them exist, and this session's ortho work already needed the same beforeScript line added to 7 by hand. Mirrors the existing getNextflowLsfScratchEnvBlock() pattern in the same file (same shared-helper shape, already called identically from 28 of the 37 generators). Also fixed getNextflowLsfScratchEnvBlock() while here: NXF_SCRATCH = '$LSF_TMPDIR' had no default, crashing under nextflow's own set -u if LSF doesn't export LSF_TMPDIR -- the same bug independently found and hand-fixed in 7 ortho-specific generators on another branch this session. Centralizing the fix here covers all 28 current callers. --- Main/lib/perl/WorkflowSteps/WorkflowStep.pm | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Main/lib/perl/WorkflowSteps/WorkflowStep.pm b/Main/lib/perl/WorkflowSteps/WorkflowStep.pm index 998887ce1b..6ab10a99bb 100644 --- a/Main/lib/perl/WorkflowSteps/WorkflowStep.pm +++ b/Main/lib/perl/WorkflowSteps/WorkflowStep.pm @@ -348,7 +348,24 @@ sub getClusterExecutor { sub getNextflowLsfScratchEnvBlock { my ($self) = @_; return '' unless $self->getClusterExecutor() eq 'lsf'; - return "\nenv {\n NXF_SCRATCH = '\$LSF_TMPDIR'\n}\n"; + # LSF_TMPDIR isn't guaranteed to be exported by every LSF cluster/job; without the + # ':-' default this crashes under nextflow's own 'set -u' if it's ever unset. + return "\nenv {\n NXF_SCRATCH = '\${LSF_TMPDIR:-}'\n}\n"; +} + +# Opt-in include of a shared, cluster-provided nextflow config file (e.g. one that sets +# process { beforeScript = '...' } for module loads that shouldn't have to be repeated in +# every Make*NextflowConfig.pm generator by hand). Reads $clusterServer.sharedNextflowConfig +# from stepsShared.prop; if it isn't set, this is a silent no-op -- existing builds that +# don't configure it are unaffected. If it is set, the returned line pulls that config in +# via nextflow's own includeConfig directive, so a single shared file can add or change a +# cluster-wide setting without touching every step class that generates a nextflow config. +sub getSharedClusterNextflowConfigIncludeBlock { + my ($self) = @_; + my $clusterServer = $self->getSharedConfig('clusterServer'); + my $path = $self->getSharedConfigRelaxed("$clusterServer.sharedNextflowConfig"); + return '' unless $path; + return "includeConfig '$path'\n"; } From da29ef678ba8dbeded4105b472c50a84555b730e Mon Sep 17 00:00:00 2001 From: rdemko2332 Date: Wed, 9 Sep 2026 15:27:56 -0400 Subject: [PATCH 2/3] Wire the shared cluster nextflow config include into every config generator Every Make*NextflowConfig.pm step class (38 total) now computes $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock() and interpolates it at the very top of its generated nextflow config, before params {}. When $clusterServer.sharedNextflowConfig isn't set in stepsShared.prop this renders as an empty string -- verified byte-identical output to before this change for that (default, current) case. When it is set, an includeConfig '' line appears at the top, pulling in whatever cluster-wide process{} settings that shared file declares. Mechanical, per-file changes only: add the one declaration line (anchored next to each file's existing $lsfEnv/$lsfScratch computation, or near the top of run() for the 5 files with neither), and prepend $sharedClusterConfig to whatever the file already prints (either interpolated into a heredoc string, or concatenated onto a $configString variable) -- every file's own existing style preserved as-is, nothing else changed. The 5 ortho-specific generators here still hand-roll their own $lsfScratch (predates the shared getNextflowLsfScratchEnvBlock() helper 28 of the others already use) -- left untouched, out of scope for this change. --- Main/lib/perl/WorkflowSteps/MakeAntismashNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeArbaNextflowConfig.pm | 3 ++- .../WorkflowSteps/MakeArrayProbeMappingNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeBlastPPdbNextflowConfig.pm | 3 ++- .../perl/WorkflowSteps/MakeBlastSimilarityNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeBlatNextflowConfig.pm | 3 ++- .../perl/WorkflowSteps/MakeBowtieMappingNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeBulkRnaSeqNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeBuscoNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeChipSeqNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeDnaSeqLoadNextflowConfig.pm | 3 ++- .../MakeDnaSeqMergeExperimentsNextflowConfig.pm | 4 +++- Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm | 3 ++- .../MakeDnaSeqSingleExperimentNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeDustNextflowConfig.pm | 3 ++- .../perl/WorkflowSteps/MakeEpitopeMappingNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeExportpredNextflowConfig.pm | 3 ++- .../MakeGenomicArrayExperimentNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeIprscan5NextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeLoadCNVNextflowConfig.pm | 5 +++-- .../perl/WorkflowSteps/MakeLongReadRnaSeqNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeMassSpecNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm | 3 ++- .../WorkflowSteps/MakeOriginsOfReplicationNextflowConfig.pm | 3 ++- .../perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm | 3 ++- .../MakeOrthoFinderPeripheralEntryNextflowConfig.pm | 3 ++- .../MakeOrthoFinderPostProcessingEntryNextflowConfig.pm | 3 ++- .../MakeOrthoFinderPostResidualEntryNextflowConfig.pm | 3 ++- .../MakeOrthoFinderResidualEntryNextflowConfig.pm | 3 ++- .../WorkflowSteps/MakeProteinAttributesNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakePsiPredNextflowConfig.pm | 3 ++- .../lib/perl/WorkflowSteps/MakeRepeatMaskerNextflowConfig.pm | 3 ++- .../perl/WorkflowSteps/MakeRnaSeqAnalysisNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeSegNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeSignalPNextflowConfig.pm | 3 ++- .../MakeSplicedLeaderAndPolyASitesNextflowConfig.pm | 3 ++- .../WorkflowSteps/MakeStudyAssayResultsNextflowConfig.pm | 3 ++- Main/lib/perl/WorkflowSteps/MakeTmhmmNextflowConfig.pm | 3 ++- 38 files changed, 78 insertions(+), 39 deletions(-) diff --git a/Main/lib/perl/WorkflowSteps/MakeAntismashNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeAntismashNextflowConfig.pm index 62e0798340..c4bcaaf44a 100644 --- a/Main/lib/perl/WorkflowSteps/MakeAntismashNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeAntismashNextflowConfig.pm @@ -28,6 +28,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -36,7 +37,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { fasta = \"$digestedFasta\" gff = \"$digestedGff\" diff --git a/Main/lib/perl/WorkflowSteps/MakeArbaNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeArbaNextflowConfig.pm index 96ae7f6574..8d581a1764 100644 --- a/Main/lib/perl/WorkflowSteps/MakeArbaNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeArbaNextflowConfig.pm @@ -26,6 +26,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -34,7 +35,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { interproResults = \"$digestedInterproResults\" outputDir = \"$digestedOutputDir\" diff --git a/Main/lib/perl/WorkflowSteps/MakeArrayProbeMappingNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeArrayProbeMappingNextflowConfig.pm index 708c514b1d..e6ef3cc5af 100644 --- a/Main/lib/perl/WorkflowSteps/MakeArrayProbeMappingNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeArrayProbeMappingNextflowConfig.pm @@ -71,6 +71,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -121,7 +122,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeBlastPPdbNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeBlastPPdbNextflowConfig.pm index 72bb881532..fc69e1b7cc 100644 --- a/Main/lib/perl/WorkflowSteps/MakeBlastPPdbNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeBlastPPdbNextflowConfig.pm @@ -31,6 +31,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -70,7 +71,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeBlastSimilarityNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeBlastSimilarityNextflowConfig.pm index 146951f896..a6fda4b9bd 100644 --- a/Main/lib/perl/WorkflowSteps/MakeBlastSimilarityNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeBlastSimilarityNextflowConfig.pm @@ -38,6 +38,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -46,7 +47,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { blastProgram = \"$blastProgram\" seqFile = \"$seqFile\" diff --git a/Main/lib/perl/WorkflowSteps/MakeBlatNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeBlatNextflowConfig.pm index 5eb18bb34d..80880b10b8 100644 --- a/Main/lib/perl/WorkflowSteps/MakeBlatNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeBlatNextflowConfig.pm @@ -31,6 +31,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -67,7 +68,7 @@ process { includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeBowtieMappingNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeBowtieMappingNextflowConfig.pm index 019fb29d82..d5ed0d1f7d 100644 --- a/Main/lib/perl/WorkflowSteps/MakeBowtieMappingNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeBowtieMappingNextflowConfig.pm @@ -32,6 +32,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -40,7 +41,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { preconfiguredDatabase = $preconfiguredDatabase writeBedFile = $writeBedFile diff --git a/Main/lib/perl/WorkflowSteps/MakeBulkRnaSeqNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeBulkRnaSeqNextflowConfig.pm index 82d9172dee..3cf90aeb0d 100644 --- a/Main/lib/perl/WorkflowSteps/MakeBulkRnaSeqNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeBulkRnaSeqNextflowConfig.pm @@ -36,6 +36,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -101,7 +102,7 @@ def check_max(obj, type) { NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeBuscoNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeBuscoNextflowConfig.pm index 1818fe9cd1..1db868bfab 100644 --- a/Main/lib/perl/WorkflowSteps/MakeBuscoNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeBuscoNextflowConfig.pm @@ -30,6 +30,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -67,7 +68,7 @@ process { includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeChipSeqNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeChipSeqNextflowConfig.pm index cad7e2c449..4a82290fee 100644 --- a/Main/lib/perl/WorkflowSteps/MakeChipSeqNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeChipSeqNextflowConfig.pm @@ -40,6 +40,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -83,7 +84,7 @@ process { NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqLoadNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqLoadNextflowConfig.pm index 33b1569eb9..02d8d2db35 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqLoadNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqLoadNextflowConfig.pm @@ -14,13 +14,14 @@ sub run { my $genomeExtDbRlsSpec = $self->getParamValue("genomeExtDbRlsSpec"); my $configPath = $self->getWorkflowDataDir() . "/" . $self->getParamValue("nextflowConfigFile"); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); if ($undo) { $self->runCmd(0, "rm -rf $configPath"); } else { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { indelDir = \"$indelDir\" extDbRlsSpec = '\"$extDbRlsSpec\"' diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm index 96826797f3..4078f5962e 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqMergeExperimentsNextflowConfig.pm @@ -63,9 +63,11 @@ sub run { my @experimentConfigs = glob($experimentConfigGlob); my $ploidy = parsePloidyFromConfigs(@experimentConfigs); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); + open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { outputDir = \"$outputDir\" diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm index 67a8d301c6..546fafe539 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqNextflowConfig.pm @@ -39,6 +39,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $queue = $self->getClusterQueue(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $genomeFastaFile = $self->getWorkflowDataDir() . "/" . $self->getParamValue("genomeFastaFile"); @@ -59,7 +60,7 @@ sub run { $self->runCmd(0, "rm -rf $nextflowConfigFile"); } else { open(F, ">", $nextflowConfigFile) or die "$! :Can't open config file '$nextflowConfigFile' for writing"; - print F " + print F "$sharedClusterConfig params { samplesheet = \"$digestedSampleSheet\" bwaThreads = $bwaThreads diff --git a/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm index 77d500edbc..45c7f91cbf 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDnaSeqSingleExperimentNextflowConfig.pm @@ -42,6 +42,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $gusConfigFile = $ENV{GUS_HOME}."/config/gus.config"; @@ -75,7 +76,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { input = \"$input\" fromBAM = $fromBAM diff --git a/Main/lib/perl/WorkflowSteps/MakeDustNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeDustNextflowConfig.pm index bed70acdb9..20080dc8bf 100644 --- a/Main/lib/perl/WorkflowSteps/MakeDustNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeDustNextflowConfig.pm @@ -26,6 +26,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -53,7 +54,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeEpitopeMappingNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeEpitopeMappingNextflowConfig.pm index 25e62038f4..85009ac9af 100644 --- a/Main/lib/perl/WorkflowSteps/MakeEpitopeMappingNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeEpitopeMappingNextflowConfig.pm @@ -36,6 +36,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -92,7 +93,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeExportpredNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeExportpredNextflowConfig.pm index 70b7cc7dec..ffe64f4aa9 100644 --- a/Main/lib/perl/WorkflowSteps/MakeExportpredNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeExportpredNextflowConfig.pm @@ -21,6 +21,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -51,7 +52,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeGenomicArrayExperimentNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeGenomicArrayExperimentNextflowConfig.pm index f51dc51b3e..ae8126b756 100644 --- a/Main/lib/perl/WorkflowSteps/MakeGenomicArrayExperimentNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeGenomicArrayExperimentNextflowConfig.pm @@ -47,6 +47,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -78,7 +79,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeIprscan5NextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeIprscan5NextflowConfig.pm index 26ea98f04c..369d1ca2e5 100644 --- a/Main/lib/perl/WorkflowSteps/MakeIprscan5NextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeIprscan5NextflowConfig.pm @@ -25,6 +25,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -33,7 +34,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { input = \"$digestedInput\" outputDir = \"$digestedOutputDir\" diff --git a/Main/lib/perl/WorkflowSteps/MakeLoadCNVNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeLoadCNVNextflowConfig.pm index 27a40f55f5..e835a134be 100644 --- a/Main/lib/perl/WorkflowSteps/MakeLoadCNVNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeLoadCNVNextflowConfig.pm @@ -16,7 +16,8 @@ sub run { my $footprintFile = join("/", $workflowDataDir, $self->getParamValue("footprintFile")); my $ploidy = $self->getParamValue("ploidy"); my $taxonId = $self->getParamValue("taxonId"); - my $outputDir = join("/", $workflowDataDir, $self->getParamValue("clusterResultDir")); + my $outputDir = join("/", $workflowDataDir, $self->getParamValue("clusterResultDir")); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); if ($undo) { $self->runCmd(0,"rm -rf $configPath"); @@ -24,7 +25,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { input = \"$input\" outputDir = \"$outputDir\" diff --git a/Main/lib/perl/WorkflowSteps/MakeLongReadRnaSeqNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeLongReadRnaSeqNextflowConfig.pm index 1b5fe3bcf2..d90169733e 100644 --- a/Main/lib/perl/WorkflowSteps/MakeLongReadRnaSeqNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeLongReadRnaSeqNextflowConfig.pm @@ -42,6 +42,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -52,7 +53,7 @@ sub run { } else { open(F, ">", $nextflowConfigFile) or die "$! :Can't open config file '$nextflowConfigFile' for writing"; print F - " + "$sharedClusterConfig params { input = \"$digestedInputDirPath\" samplesheetFileName = \"$sampleSheet\" diff --git a/Main/lib/perl/WorkflowSteps/MakeMassSpecNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeMassSpecNextflowConfig.pm index de70b5b198..1d901307d4 100644 --- a/Main/lib/perl/WorkflowSteps/MakeMassSpecNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeMassSpecNextflowConfig.pm @@ -34,6 +34,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -64,7 +65,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm index 57dae80bb8..036c74f6ee 100644 --- a/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeNgsSamplesNextflowConfig.pm @@ -39,6 +39,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -100,7 +101,7 @@ workDir = "$digestedAnalysisDirPath/ngs-samples-work" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeOriginsOfReplicationNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOriginsOfReplicationNextflowConfig.pm index c136215632..6ac48b8bdc 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOriginsOfReplicationNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOriginsOfReplicationNextflowConfig.pm @@ -38,6 +38,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -82,7 +83,7 @@ process { NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm index 5e2ee0680f..94022e40ae 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm @@ -31,6 +31,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -39,7 +40,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { proteomes = \"$proteomesInNextflowWorkingDirOnCluster\" outputDir = \"$resultsDirectoryInNextflowWorkingDirOnCluster\" diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm index f1dfda6769..9297b7652f 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm @@ -37,6 +37,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -45,7 +46,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { outputDir = \"$resultsDirectoryInNextflowWorkingDirOnCluster\" coreProteomes = \"$coreProteomesInNextflowWorkingDirOnCluster\" diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm index de0dd87491..9a1052e695 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm @@ -42,6 +42,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -50,7 +51,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { outputDir = \"$resultsDirectoryInNextflowWorkingDirOnCluster\" residualFasta = \"$residualFastaInNextflowWorkingDirOnCluster\" diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm index 86b43545ed..0349af1779 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm @@ -33,6 +33,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -41,7 +42,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { outputDir = \"$resultsDirectoryInNextflowWorkingDirOnCluster\" residualFasta = \"$residualFastaInNextflowWorkingDirOnCluster\" diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm index 0a88f349c0..e944b38dbe 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm @@ -23,6 +23,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); if ($undo) { @@ -31,7 +32,7 @@ sub run { open(F, ">", $configPath) or die "$! :Can't open config file '$configPath' for writing"; print F -" +"$sharedClusterConfig params { outputDir = \"$resultsDirectoryInNextflowWorkingDirOnCluster\" residualFastaDir = \"$residualFastaDirInNextflowWorkingDirOnCluster\" diff --git a/Main/lib/perl/WorkflowSteps/MakeProteinAttributesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeProteinAttributesNextflowConfig.pm index cb06d8b4c6..d5cb1bdfcf 100644 --- a/Main/lib/perl/WorkflowSteps/MakeProteinAttributesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeProteinAttributesNextflowConfig.pm @@ -23,6 +23,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -53,7 +54,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakePsiPredNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakePsiPredNextflowConfig.pm index 867ef563fe..e53219e6dc 100644 --- a/Main/lib/perl/WorkflowSteps/MakePsiPredNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakePsiPredNextflowConfig.pm @@ -23,6 +23,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -56,7 +57,7 @@ process { includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeRepeatMaskerNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeRepeatMaskerNextflowConfig.pm index f1ed5da6c9..7e2629f577 100644 --- a/Main/lib/perl/WorkflowSteps/MakeRepeatMaskerNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeRepeatMaskerNextflowConfig.pm @@ -39,6 +39,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -87,7 +88,7 @@ singularity { NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } diff --git a/Main/lib/perl/WorkflowSteps/MakeRnaSeqAnalysisNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeRnaSeqAnalysisNextflowConfig.pm index 462684452a..8445662d85 100644 --- a/Main/lib/perl/WorkflowSteps/MakeRnaSeqAnalysisNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeRnaSeqAnalysisNextflowConfig.pm @@ -25,6 +25,7 @@ sub run { my $workflowDataDir = $self->getWorkflowDataDir(); my $baseConfigFile = "\$baseDir/conf/singularity.config"; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); if ($undo) { $self->runCmd(0, "rm $workflowDataDir/$nextflowConfigFile"); @@ -48,7 +49,7 @@ includeConfig "$baseConfigFile" NEXTFLOW - print F $configString; + print F $sharedClusterConfig . $configString; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeSegNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeSegNextflowConfig.pm index a63c2889b6..4352825982 100644 --- a/Main/lib/perl/WorkflowSteps/MakeSegNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeSegNextflowConfig.pm @@ -22,6 +22,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -52,7 +53,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeSignalPNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeSignalPNextflowConfig.pm index bde902ca0f..36ebdff9eb 100644 --- a/Main/lib/perl/WorkflowSteps/MakeSignalPNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeSignalPNextflowConfig.pm @@ -36,6 +36,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -79,7 +80,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeSplicedLeaderAndPolyASitesNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeSplicedLeaderAndPolyASitesNextflowConfig.pm index 153b3e73e1..4d5d42ce3e 100644 --- a/Main/lib/perl/WorkflowSteps/MakeSplicedLeaderAndPolyASitesNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeSplicedLeaderAndPolyASitesNextflowConfig.pm @@ -37,6 +37,7 @@ sub run { my $clusterWorkflowDataDir = $self->getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -77,7 +78,7 @@ process { NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } diff --git a/Main/lib/perl/WorkflowSteps/MakeStudyAssayResultsNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeStudyAssayResultsNextflowConfig.pm index fb9d9550a7..c47c54365f 100644 --- a/Main/lib/perl/WorkflowSteps/MakeStudyAssayResultsNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeStudyAssayResultsNextflowConfig.pm @@ -24,6 +24,7 @@ sub run { my $workflowDataDir = $self->getWorkflowDataDir(); my $baseConfigFile = "\$baseDir/conf/singularity.config"; + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); # Determine the input file based on platform mapping (logic from DoStudyAssayResults) my $inputFile; @@ -74,7 +75,7 @@ $configString .= <getClusterWorkflowDataDir(); my $executor = $self->getClusterExecutor(); my $lsfEnv = $self->getNextflowLsfScratchEnvBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); my $queue = $self->getClusterQueue(); my $clusterConfigFile = "\$baseDir/conf/${executor}.config"; @@ -64,7 +65,7 @@ includeConfig "$clusterConfigFile" NEXTFLOW - print F $configString . $lsfEnv; + print F $sharedClusterConfig . $configString . $lsfEnv; close(F); } } From d1c70181c6e98d4fb2cbfddbefc368e351728e6c Mon Sep 17 00:00:00 2001 From: rdemko2332 Date: Thu, 10 Sep 2026 12:50:53 -0400 Subject: [PATCH 3/3] Shared nextflow config lives on the workflow server, symlinked into analysisDir Reworks getSharedClusterNextflowConfigIncludeBlock() (added earlier this branch): the shared config file now lives on the workflow server (yew), read from a flat 'sharedNextflowConfig' key in stepsShared.prop, not a $clusterServer-scoped cluster path. Decouples the file's location from which cluster runs the work. Mechanism: the helper symlinks the yew-side file into the run's analysisDir as sharedNextflow.config, then returns an includeConfig line pointing at that file's cluster-side path (translated via relativePathToNextflowClusterPath, same as every other path these generators emit). CopyNextflowWorkingDirToCluster later tars analysisDir to the cluster with 'tar cfh' -- the -h dereferences the symlink into real content there, so the includeConfig target exists on the cluster. Signature gains ($analysisDir, $workingDirRelativePath) -- the caller's own locals, needed to place the symlink and translate the path. A caller that can't supply them (the 33 non-ortho generators, still calling zero-arg from the earlier sweep) gets a silent no-op, same as when the key is unset. Wired for real only in the 5 ortho generators for now; extending to other workflows is a separate change. Errors early with a clear message if sharedNextflowConfig points at a nonexistent file, rather than failing later in an opaque tar error. --- .../MakeOrthoFinderCoreNextflowConfig.pm | 2 +- ...rthoFinderPeripheralEntryNextflowConfig.pm | 2 +- ...FinderPostProcessingEntryNextflowConfig.pm | 2 +- ...hoFinderPostResidualEntryNextflowConfig.pm | 2 +- ...eOrthoFinderResidualEntryNextflowConfig.pm | 2 +- Main/lib/perl/WorkflowSteps/WorkflowStep.pm | 39 +++++++++++++------ 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm index 94022e40ae..c96c689fab 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderCoreNextflowConfig.pm @@ -31,7 +31,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; - my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock($analysisDir, $workingDirRelativePath); my $queue = $self->getClusterQueue(); if ($undo) { diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm index 9297b7652f..7d4ac60b10 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPeripheralEntryNextflowConfig.pm @@ -37,7 +37,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; - my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock($analysisDir, $workingDirRelativePath); my $queue = $self->getClusterQueue(); if ($undo) { diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm index 9a1052e695..1d471fd396 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostProcessingEntryNextflowConfig.pm @@ -42,7 +42,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; - my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock($analysisDir, $workingDirRelativePath); my $queue = $self->getClusterQueue(); if ($undo) { diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm index 0349af1779..e336e613f2 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderPostResidualEntryNextflowConfig.pm @@ -33,7 +33,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; - my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock($analysisDir, $workingDirRelativePath); my $queue = $self->getClusterQueue(); if ($undo) { diff --git a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm index e944b38dbe..0ec308d872 100644 --- a/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm +++ b/Main/lib/perl/WorkflowSteps/MakeOrthoFinderResidualEntryNextflowConfig.pm @@ -23,7 +23,7 @@ sub run { my $executor = $self->getClusterExecutor(); my $lsfScratch = ($executor eq 'lsf') ? "\n NXF_SCRATCH = '\$LSF_TMPDIR'" : ''; - my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock(); + my $sharedClusterConfig = $self->getSharedClusterNextflowConfigIncludeBlock($analysisDir, $workingDirRelativePath); my $queue = $self->getClusterQueue(); if ($undo) { diff --git a/Main/lib/perl/WorkflowSteps/WorkflowStep.pm b/Main/lib/perl/WorkflowSteps/WorkflowStep.pm index 6ab10a99bb..84b0e69fbc 100644 --- a/Main/lib/perl/WorkflowSteps/WorkflowStep.pm +++ b/Main/lib/perl/WorkflowSteps/WorkflowStep.pm @@ -353,19 +353,36 @@ sub getNextflowLsfScratchEnvBlock { return "\nenv {\n NXF_SCRATCH = '\${LSF_TMPDIR:-}'\n}\n"; } -# Opt-in include of a shared, cluster-provided nextflow config file (e.g. one that sets +# Opt-in include of a shared nextflow config file (e.g. one that sets # process { beforeScript = '...' } for module loads that shouldn't have to be repeated in -# every Make*NextflowConfig.pm generator by hand). Reads $clusterServer.sharedNextflowConfig -# from stepsShared.prop; if it isn't set, this is a silent no-op -- existing builds that -# don't configure it are unaffected. If it is set, the returned line pulls that config in -# via nextflow's own includeConfig directive, so a single shared file can add or change a -# cluster-wide setting without touching every step class that generates a nextflow config. +# every Make*NextflowConfig.pm generator by hand). +# +# The file lives on the workflow server (yew), not the cluster, so which cluster actually +# runs the processing can change without moving it. It is symlinked into the run's +# analysisDir here; CopyNextflowWorkingDirToCluster later carries analysisDir to the +# cluster with 'tar cfh', whose -h dereferences the symlink into real content there. The +# returned line then points includeConfig at that file's cluster-side path, translated the +# same way every other path these generators emit is. +# +# Reads the flat 'sharedNextflowConfig' key from stepsShared.prop (via getSharedConfigRelaxed, +# so unset is a silent no-op -- existing builds are unaffected). $analysisDir and +# $workingDirRelativePath are the calling generator's own locals; a caller that doesn't +# have them (i.e. can't place the symlink) also gets a silent no-op. sub getSharedClusterNextflowConfigIncludeBlock { - my ($self) = @_; - my $clusterServer = $self->getSharedConfig('clusterServer'); - my $path = $self->getSharedConfigRelaxed("$clusterServer.sharedNextflowConfig"); - return '' unless $path; - return "includeConfig '$path'\n"; + my ($self, $analysisDir, $workingDirRelativePath) = @_; + + my $sharedPath = $self->getSharedConfigRelaxed('sharedNextflowConfig'); + return '' unless $sharedPath; + return '' unless $analysisDir && $workingDirRelativePath; + + $self->error("sharedNextflowConfig '$sharedPath' does not exist on the workflow server") + unless -e $sharedPath; + + my $localLink = join("/", $self->getWorkflowDataDir(), $analysisDir, "sharedNextflow.config"); + $self->runCmd(0, "ln -sfn $sharedPath $localLink"); + + my $clusterPath = $self->relativePathToNextflowClusterPath($workingDirRelativePath, "$analysisDir/sharedNextflow.config"); + return "includeConfig '$clusterPath'\n"; }