WIP: Toil times

Makefile

@@ -251,6 +251,9 @@ evolver_test_all_local: evolver_test_local evolver_test_prepare_toil evolver_tes
 yeast_test_local:
 	PYTHONPATH="${CWD}/submodules/" CACTUS_BINARIES_MODE=local CACTUS_DOCKER_MODE=0 ${PYTHON} -m pytest ${pytestOpts} -s test/evolverTest.py::TestCase::testYeastPangenomeLocal
 
+yeast_test_mgsplit_local:
+	PYTHONPATH="${CWD}/submodules/" CACTUS_BINARIES_MODE=local CACTUS_DOCKER_MODE=0 ${PYTHON} -m pytest ${pytestOpts} -s test/evolverTest.py::TestCase::testYeastPangenomeMgSplitLocal
+
 yeast_test_split_local:
 	PYTHONPATH="${CWD}/submodules/" CACTUS_BINARIES_MODE=local CACTUS_DOCKER_MODE=0 ${PYTHON} -m pytest ${pytestOpts} -s test/evolverTest.py::TestCase::testYeastPangenomeSplitLocal
 

build-tools/downloadPangenomeTools

@@ -217,7 +217,7 @@ fi
 cd ${pangenomeBuildDir}
 git clone https://github.com/ComparativeGenomicsToolkit/cactus-gfa-tools.git
 cd cactus-gfa-tools
-git checkout a32588c32fc22652c55f7bfe00d240e16ec96598
+git checkout 2dea492f60a1494fe6d0546d999b4b1f8dad3161
 make -j ${numcpu}
 if [[ $STATIC_CHECK -ne 1 || $(ldd paf2lastz | grep so | wc -l) -eq 0 ]]
 then
@@ -243,6 +243,12 @@ then
 else
 	 exit 1
 fi
+if [[ $STATIC_CHECK -ne 1 || $(ldd rgfa-collapse | grep so | wc -l) -eq 0 ]]
+then
+	 mv rgfa-collapse ${binDir}
+else
+	 exit 1
+fi
 if [[ $STATIC_CHECK -ne 1 || $(ldd rgfa-split | grep so | wc -l) -eq 0 ]]
 then
 	 mv rgfa-split ${binDir}

doc/pangenome.md

@@ -711,7 +711,7 @@ One thing to watch out for: **error intervals at contig tips interact with telom
 
 ```
 cactus-panpatch ./js chromfile.txt --outDir patched --batch --requireTelomeres \
-    --batchSystem slurm --slurmTime 10:00:00 --doubleMem true --maxMemory 1.5T --retryCount 10 \
+    --batchSystem slurm --doubleMem true --maxMemory 1.5T --retryCount 10 \
     --consCores 16 --mgCores 32 --mapCores 16
 ```
 

doc/progressive.md

@@ -232,7 +232,7 @@ cactus-hal2maf ./js evolverMammals.hal evolverMammals.maf.gz --refGenome simHuma
 Exporting a MAF for each reference in an 8-way [ape alignment](https://cglgenomics.ucsc.edu/february-2024-t2t-apes/) on UCSC Slurm cluster:
 
 ```
-for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16  --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --slurmTime 200:00:00 --slurmPartition long;done
+for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16  --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --slurmPartition long;done
 ```
 
 Note that this invocation creates two MAFs per reference (as dictated by the `--outType` option).  Notably, the `.single.maf.gz` files will be filtered so that each genome appears at most once per block, which is often required by browsers and other MAF-reading tools.
@@ -343,7 +343,7 @@ cactus-phast ./js-vgp \
     --geneAnnotation https://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/ncbiRefSeq.txt.gz \
     --bigwig --batchSystem slurm --chunkCores 32 --phyloFitCores 32 \
     --substMod REV --modFreqs --precision HIGH \
-    --slurmPartition medium --slurmTime 10:00:00 \
+    --slurmPartition medium \
     --doubleMem true
 ```
 
@@ -419,8 +419,15 @@ These are the most relevant options for running on a cluster
 
 On a cluster with partitions and/or time limits, make sure to use
 
-* `--slurmTime` to specify the time for each job.  Unfortunately cactus does not yet try to set this itself, so you need to give one value that will be applied to all jobs, ex `--slurmTime 200:00:00`
-* `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on.  Cactus will try to figure this out on its own using the `--slurmTime` value along with whether or not the job needs GPU.  But this option will allow you to override that.
+Slurm chooses a job's partition from its time limit, so giving every job one big time (the old `--slurmTime 200:00:00` advice) puts even the thousands of jobs that finish in seconds into the slowest queue.  **Cactus now estimates a walltime for each job itself**, from the size of that job's inputs, the same way it estimates memory.  There is nothing you need to set for this: the defaults below are the knobs for when the estimates do not suit your cluster.
+
+* `--doubleTime true` is **on by default** (Cactus turns it on the way it raises `--retryCount` to 5).  It is the time analogue of `--doubleMem`: if slurm kills a job for running past its walltime, it is retried with double the walltime.  This is what makes tight per-job estimates safe -- an occasional underestimate costs one retry, not the run.  Pass `--doubleTime false` to turn it off.
+* `--walltimeFactor` is the safety multiplier applied to every estimate (default `2.5`).  Raise it if your cluster is slower than the ones the estimates were measured on and jobs are being killed for running over.  Pass `--walltimeFactor 0` to switch per-job walltimes off entirely, which puts every job back on `--defaultWalltime`.
+* `--minWalltime` is the floor for any request, in seconds (default `1800`).  A short partition is typically an hour, so every request below that routes the same way and there is nothing to gain by asking for less -- while a request that is short by a minute costs a whole `--doubleTime` retry.  Half an hour leaves room for worker startup and for jobstore staging under heavy concurrency, which no size-based estimate can see.
+* `--maxWalltime` is the ceiling, in seconds.  By default, on Slurm, Cactus queries `sinfo` and caps every request at *half* the longest time limit of any partition your jobs could land on.  This matters more than it looks: when Toil cannot find a partition that fits a job's walltime it raises rather than falling back to a default, and it does so on the batch system's own thread, taking the whole workflow down rather than just the job.  Since `--doubleTime` retries a killed job with twice the walltime, an estimate sitting at the longest partition's limit would turn its own first retry into exactly that crash -- hence the half.  A job needing more than half your longest partition still gets there, it just spends one attempt on the way.  With `--doubleTime false` there is nothing to retry into and the full limit is used.
+* `--defaultWalltime` is Toil's fallback for jobs Cactus did not estimate a time for.  With the estimates on there is very little left for it to cover, so you can usually leave it alone.
+* `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on.  Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU.  But this option will allow you to override that.
+* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job estimates above (so it also disables the partition routing they exist for, and `--doubleTime` along with it).  Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`.
 
 You can also use
 
@@ -439,13 +446,13 @@ source /private/groups/cgl/cactus/venv-cactus-latest/bin/activate
 Some recommended options: 
 
 ```
-cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true --slurmTime 200:00:00
+cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true
 ```
 
 To run the same command step by step,
 
 ```
-cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --slurmTime 200:00:00 --batchSystem slurm" --alignCores 64 --script > mammals.sh
+cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --batchSystem slurm" --alignCores 64 --script > mammals.sh
 chmod+x mammals.sh
 ./mammals.sh
 ```

src/cactus/blast/cactus_blast.py

@@ -12,12 +12,12 @@
 import timeit
 
 from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree
-from cactus.shared.common import setupBinaries, importSingularityImage
+from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime
 from cactus.shared.common import cactusRootPath
 from cactus.shared.configWrapper import ConfigWrapper
 from cactus.shared.common import makeURL, catFiles
 from cactus.shared.common import enableDumpStack
-from cactus.shared.common import cactus_override_toil_options
+from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options
 from cactus.shared.common import getOptionalAttrib
 from cactus.shared.version import cactus_commit
 from cactus.progressive.cactus_prepare import human2bytesN
@@ -34,6 +34,7 @@
 
 def main():
     parser = Job.Runner.getDefaultArgumentParser()
+    add_cactus_toil_options(parser)
 
     parser.add_argument("seqFile", help="Seq file")
     parser.add_argument("outputFile", type=str, help="Output pairwise alignment file")
@@ -156,7 +157,7 @@ def runCactusBlastOnly(options):
                     input_seq_id_map[genome] = toil.importFile(seq)
 
             paf_id = toil.start(Job.wrapJobFn(sanitize_then_make_paf_alignments, NXNewick().writeString(spanning_tree),
-                                              input_seq_id_map, options.root, config_node, options.outputFile))
+                                              input_seq_id_map, options.root, config_node, options.outputFile, walltime=cactus_walltime()))
 
         # export the alignments
         toil.exportFile(paf_id, makeURL(options.outputFile))

src/cactus/cactus_progressive_config.xml

@@ -135,6 +135,20 @@
 				five="--step=2 --ambiguous=iupac,100,100 --ydrop=4000 --hspthresh=2400 --gappedthresh=2600 --queryhspbest=100000"
 				default="--step=1 --ambiguous=iupac,100,100 --ydrop=4000 --hspthresh=2200 --gappedthresh=2400 --queryhspbest=100000"
 		/>
+		<!-- Estimated runtime, in seconds, of one lastz job in each divergence class, for a
+		     pair of full-sized (chunkSize) chunks.  These are the p99 of 827k lastz runs from
+		     the VGP 577-way alignment, halved for the 2x lastz speedup since; Cactus scales
+		     them by the actual chunk sizes and applies the walltime factor on top.  Slurm picks a
+		     partition from the time limit, so these being per-class rather than one global
+		     number is what keeps the 147k closely-related pairs out of the long queue. -->
+		<lastzWalltime
+				one="350"
+				two="1400"
+				three="1650"
+				four="2000"
+				five="2800"
+				default="4300"
+		/>
 		<!-- KegAlign doesn't seem to work properly with queryhspbest -->
 		<kegalignArguments
 				one="--step=2 --ambiguous=iupac,100,100 --ydrop=3000 --notransition"
@@ -433,6 +447,8 @@
 		 minIdentity="0.5"
 		 minScore="1"
 		 removeMinigraphFromPAF="0"
+		 collapseInversions="0"
+		 collapseOptions="-k 300 -b 5000 -i 0.95 -D -A 3 -x asm20 -M 1.05"
 		 cpu="6"
 		 />
 	<!-- cactus-graphmap-split options -->
@@ -605,6 +621,13 @@
 		 memory_core_scale_baseline="64"
 		 memory_core_scale_max_pct="100"
 		 memory_poa_window_exponent="0.43"
+		 walltime_coefficient_secs="400"
+		 walltime_input_exponent="0.95"
+		 walltime_banded_coefficient_secs="550"
+		 walltime_banded_exponent="0.35"
+		 walltime_banding_threshold="100000"
+		 walltime_core_scale_baseline="24"
+		 walltime_parallel_fraction="0.64"
 		 og_size_scale_pct="75"
 		 >
 	</consolidated>