From c3080b3f166ba058b497fa64931f127a3f0414c1 Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 23 Jul 2026 11:47:14 -0400 Subject: [PATCH 01/18] cactus-panpatch: call the shared setup functions in main() cactus-panpatch's main() was missing the standard per-tool boilerplate that every other cactus entry point runs after parsing options: - cactus_override_toil_options(options): sets toil defaults (retryCount, realtime logging, default memory) and exports CACTUS_MAX_MEMORY / CACTUS_DEFAULT_MEMORY. panpatch calls cactus_clamp_memory(), which reads those env vars, so without this it would KeyError. - setupBinaries(options): configures binariesMode/docker so cactus_call works as requested (panpatch runs vg, bgzip, etc. via cactus_call). - set_logging_from_options(options) and enableDumpStack(): consistent logging setup and debug stack dumps. Co-Authored-By: Claude Opus 4.8 --- src/cactus/refmap/cactus_panpatch.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cactus/refmap/cactus_panpatch.py b/src/cactus/refmap/cactus_panpatch.py index 90fe96b04..e158edc47 100644 --- a/src/cactus/refmap/cactus_panpatch.py +++ b/src/cactus/refmap/cactus_panpatch.py @@ -34,6 +34,9 @@ from cactus.shared.common import cactus_clamp_memory from cactus.shared.common import delete_directory from cactus.shared.common import getOptionalAttrib, findRequiredNode +from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import setupBinaries +from cactus.shared.common import enableDumpStack from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.version import cactus_commit from cactus.refmap.cactus_minigraph import check_sample_names @@ -42,7 +45,7 @@ from cactus.refmap.cactus_pangenome import pangenome_config_overrides, pangenome_end_to_end_workflow from toil.job import Job from toil.common import Toil -from toil.statsAndLogging import logger +from toil.statsAndLogging import logger, set_logging_from_options from toil.realtimeLogger import RealtimeLogger def main(): @@ -107,6 +110,10 @@ def main(): options = parser.parse_args() + setupBinaries(options) + set_logging_from_options(options) + enableDumpStack() + panpatch_validate_options(options) # we need the minigraph event name to filter it out of the seqfiles below @@ -137,6 +144,9 @@ def main(): # out of the join workflow before any clipping or filtering disable_pangenome_outputs(options) + # Mess with some toil options to create useful defaults. + cactus_override_toil_options(options) + logger.info('Cactus Command: {}'.format(' '.join(sys.argv))) logger.info('Cactus Commit: {}'.format(cactus_commit)) for run in runs: From f9e94fafe374e132e57185bd918786d0850d1606 Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 23 Jul 2026 11:59:25 -0400 Subject: [PATCH 02/18] Add per-job "fast" walltime to route small coordination jobs to a fast Slurm partition Newer Toil exposes a per-job `walltime` requirement that its Slurm backend uses to pick a partition (and set --time). This gives Cactus's many small "coordination" jobs a short walltime so Slurm routes them to a fast/short partition, while everything else falls back to Toil's --defaultWalltime. Machinery (src/cactus/shared/common.py): - cactus_fast_walltime(): reads CACTUS_FAST_WALLTIME (mirrors cactus_clamp_memory). - add_cactus_toil_options(): adds --fastWalltime (default 1800 = 30 min, very conservative for these jobs; 0 disables). - cactus_override_toil_options() exports CACTUS_FAST_WALLTIME so it reaches workers. - RoundedJob / ChildTreeJob now forward walltime to Toil. Application: - Wired add_cactus_toil_options(parser) into every tool's main(). - Tagged the small coordination jobs' scheduling call sites with walltime=cactus_fast_walltime() (the FAST set plus a hand-picked subset of the ambiguous ones; compute-heavy jobs left on --defaultWalltime). Docs (doc/progressive.md, doc/pangenome.md): recommend --defaultWalltime (+ the new --fastWalltime) instead of --slurmTime, which is now framed as a global override. IMPORTANT: toil-requirement.txt is temporarily pinned to Toil master because per-job walltime is not in a Toil release yet. Revert to a released toil[aws]== before merging. Co-Authored-By: Claude Opus 4.8 --- doc/pangenome.md | 2 +- doc/progressive.md | 14 ++--- src/cactus/blast/cactus_blast.py | 7 +-- src/cactus/maf/cactus_hal2chains.py | 11 ++-- src/cactus/maf/cactus_hal2maf.py | 19 +++---- src/cactus/maf/cactus_maf2bigmaf.py | 7 +-- src/cactus/maf/cactus_phast.py | 29 ++++++----- src/cactus/paf/local_alignment.py | 20 ++++---- .../preprocessor/cactus_preprocessor.py | 15 +++--- src/cactus/preprocessor/unmasking.py | 4 +- src/cactus/progressive/cactus_prepare.py | 5 +- src/cactus/progressive/cactus_progressive.py | 27 +++++----- src/cactus/refmap/cactus_graphmap.py | 15 +++--- src/cactus/refmap/cactus_graphmap_join.py | 17 ++++--- src/cactus/refmap/cactus_graphmap_split.py | 17 ++++--- src/cactus/refmap/cactus_minigraph.py | 19 +++---- src/cactus/refmap/cactus_pangenome.py | 51 ++++++++++--------- src/cactus/refmap/cactus_panpatch.py | 15 +++--- src/cactus/refmap/cactus_refmap.py | 13 ++--- src/cactus/setup/cactus_align.py | 18 ++++--- src/cactus/shared/common.py | 39 ++++++++++++-- src/cactus/update/cactus_hal2seqfile.py | 7 +-- toil-requirement.txt | 4 +- 23 files changed, 215 insertions(+), 160 deletions(-) diff --git a/doc/pangenome.md b/doc/pangenome.md index a6f29b944..f9240ee4f 100644 --- a/doc/pangenome.md +++ b/doc/pangenome.md @@ -341,7 +341,7 @@ cactus-panpatch ./js chromfile.txt --outDir patched --batch ``` cactus-panpatch ./js chromfile.txt --outDir patched --batch --requireTelomeres \ - --batchSystem slurm --slurmTime 10:00:00 --doubleMem true --maxMemory 1.5T --retryCount 10 \ + --batchSystem slurm --defaultWalltime 36000 --doubleMem true --maxMemory 1.5T --retryCount 10 \ --consCores 16 --mgCores 32 --mapCores 16 ``` diff --git a/doc/progressive.md b/doc/progressive.md index e28220263..97dd5e8df 100644 --- a/doc/progressive.md +++ b/doc/progressive.md @@ -231,7 +231,7 @@ cactus-hal2maf ./js evolverMammals.hal evolverMammals.maf.gz --refGenome simHuma Exporting a MAF for each reference in an 8-way [ape alignment](https://cglgenomics.ucsc.edu/february-2024-t2t-apes/) on UCSC Slurm cluster: ``` -for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16 --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --slurmTime 200:00:00 --slurmPartition long;done +for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16 --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --defaultWalltime 720000 --slurmPartition long;done ``` Note that this invocation creates two MAFs per reference (as dictated by the `--outType` option). Notably, the `.single.maf.gz` files will be filtered so that each genome appears at most once per block, which is often required by browsers and other MAF-reading tools. @@ -342,7 +342,7 @@ cactus-phast ./js-vgp \ --geneAnnotation https://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/ncbiRefSeq.txt.gz \ --bigwig --batchSystem slurm --chunkCores 32 --phyloFitCores 32 \ --substMod REV --modFreqs --precision HIGH \ - --slurmPartition medium --slurmTime 10:00:00 \ + --slurmPartition medium --defaultWalltime 36000 \ --doubleMem true ``` @@ -416,8 +416,10 @@ These are the most relevant options for running on a cluster On a cluster with partitions and/or time limits, make sure to use -* `--slurmTime` to specify the time for each job. Unfortunately cactus does not yet try to set this itself, so you need to give one value that will be applied to all jobs, ex `--slurmTime 200:00:00` -* `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own using the `--slurmTime` value along with whether or not the job needs GPU. But this option will allow you to override that. +* `--defaultWalltime` to specify the time, **in seconds**, for each job, ex `--defaultWalltime 720000` (200 hours). Slurm uses each job's walltime to choose a partition, so give the big jobs plenty of time here. +* `--fastWalltime` sets the walltime, **in seconds**, that Cactus gives its many small "coordination" jobs; it defaults to `1800` (30 minutes), which is very conservative for these jobs. On a cluster with a fast/short partition, this lets those little jobs get routed there instead of tying up a long partition; everything else falls back to `--defaultWalltime`. Pass `--fastWalltime 0` to disable it. +* `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU. But this option will allow you to override that. +* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job `--defaultWalltime` / `--fastWalltime` values above (so it also disables the fast-partition routing). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. You can also use @@ -436,13 +438,13 @@ source /private/groups/cgl/cactus/venv-cactus-latest/bin/activate Some recommended options: ``` -cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true --slurmTime 200:00:00 +cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true --defaultWalltime 720000 ``` To run the same command step by step, ``` -cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --slurmTime 200:00:00 --batchSystem slurm" --alignCores 64 --script > mammals.sh +cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --defaultWalltime 720000 --batchSystem slurm" --alignCores 64 --script > mammals.sh chmod+x mammals.sh ./mammals.sh ``` diff --git a/src/cactus/blast/cactus_blast.py b/src/cactus/blast/cactus_blast.py index 275ed4f23..cc3d16593 100644 --- a/src/cactus/blast/cactus_blast.py +++ b/src/cactus/blast/cactus_blast.py @@ -12,12 +12,12 @@ import timeit from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import getOptionalAttrib from cactus.shared.version import cactus_commit from cactus.progressive.cactus_prepare import human2bytesN @@ -34,6 +34,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help="Seq file") parser.add_argument("outputFile", type=str, help="Output pairwise alignment file") @@ -156,7 +157,7 @@ def runCactusBlastOnly(options): input_seq_id_map[genome] = toil.importFile(seq) paf_id = toil.start(Job.wrapJobFn(sanitize_then_make_paf_alignments, NXNewick().writeString(spanning_tree), - input_seq_id_map, options.root, config_node, options.outputFile)) + input_seq_id_map, options.root, config_node, options.outputFile, walltime=cactus_fast_walltime())) # export the alignments toil.exportFile(paf_id, makeURL(options.outputFile)) diff --git a/src/cactus/maf/cactus_hal2chains.py b/src/cactus/maf/cactus_hal2chains.py index 4603f9f86..b0cbdce3d 100644 --- a/src/cactus/maf/cactus_hal2chains.py +++ b/src/cactus/maf/cactus_hal2chains.py @@ -14,12 +14,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.version import cactus_commit @@ -41,6 +41,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help = "HAL file to convert to MAF") parser.add_argument("outDir", help = "Output directory") @@ -154,7 +155,7 @@ def main(): config.substituteAllPredefinedConstantsWithLiterals(options) hal_id = toil.importFile(options.halFile) - chains_id_dict = toil.start(Job.wrapJobFn(hal2chains_workflow, config, options, hal_id)) + chains_id_dict = toil.start(Job.wrapJobFn(hal2chains_workflow, config, options, hal_id, walltime=cactus_fast_walltime())) #export the chains for query_genome in chains_id_dict.keys(): @@ -183,8 +184,8 @@ def hal2chains_workflow(job, config, options, hal_id): disk=int(hal_id.size * 1.2)) leaf_genomes = get_genomes_job.rv(0) distance_matrix = get_genomes_job.rv(1) - chrom_info_job = get_genomes_job.addFollowOnJobFn(hal2chains_chrom_info_all, config, options, hal_id, leaf_genomes) - hal2chains_all_job = chrom_info_job.addFollowOnJobFn(hal2chains_all, config, options, hal_id, chrom_info_job.rv(), distance_matrix) + chrom_info_job = get_genomes_job.addFollowOnJobFn(hal2chains_chrom_info_all, config, options, hal_id, leaf_genomes, walltime=cactus_fast_walltime()) + hal2chains_all_job = chrom_info_job.addFollowOnJobFn(hal2chains_all, config, options, hal_id, chrom_info_job.rv(), distance_matrix, walltime=cactus_fast_walltime()) return hal2chains_all_job.rv() def hal2chains_check_tools(job, options): diff --git a/src/cactus/maf/cactus_hal2maf.py b/src/cactus/maf/cactus_hal2maf.py index ffcba36c3..da2cda59b 100644 --- a/src/cactus/maf/cactus_hal2maf.py +++ b/src/cactus/maf/cactus_hal2maf.py @@ -15,12 +15,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import clean_jobstore_files @@ -44,6 +44,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help = "HAL file to convert to MAF") parser.add_argument("outputMAF", help = "Output MAF (will be gzipped if ends in .gz). Suffix with .taf or .taf.gz if you want TAF output") @@ -220,7 +221,7 @@ def main(): bed_id = toil.importFile(options.bedRanges) if options.bedRanges else None hal_id = toil.importFile(options.halFile) - toil.start(Job.wrapJobFn(hal2maf_workflow, hal_id, bed_id, options, config)) + toil.start(Job.wrapJobFn(hal2maf_workflow, hal_id, bed_id, options, config, walltime=cactus_fast_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -235,9 +236,9 @@ def hal2maf_workflow(job, hal_id, bed_id, options, config): hal2maf_ranges_job = job.addChildJobFn(hal2maf_ranges, hal_id, bed_id, options, cores=1, disk=hal_id.size) chunks, genome_list = hal2maf_ranges_job.rv(0), hal2maf_ranges_job.rv(1) - hal2maf_all_job = hal2maf_ranges_job.addFollowOnJobFn(hal2maf_all, hal_id, chunks, genome_list, options, config) + hal2maf_all_job = hal2maf_ranges_job.addFollowOnJobFn(hal2maf_all, hal_id, chunks, genome_list, options, config, walltime=cactus_fast_walltime()) hal2maf_merge_job = hal2maf_all_job.addFollowOnJobFn(hal2maf_merge_all, hal2maf_all_job.rv(), options, genome_list, - disk=hal_id.size) + disk=hal_id.size, walltime=cactus_fast_walltime()) hal2maf_ranges_job.addFollowOn(hal2maf_merge_job) # note: merge job also handles exporting (and some cleanup), indexing and coverage @@ -666,13 +667,13 @@ def hal2maf_merge_all(job, output_dicts, options, genome_list): output_ext += '.gz' if out_type != 'norm' and len(options.outType) > 1: output_name += '.{}'.format(out_type) - export_job = merge_job.addFollowOnJobFn(export_file, merge_job.rv(), output_name + output_ext) - merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=maf_ids) + export_job = merge_job.addFollowOnJobFn(export_file, merge_job.rv(), output_name + output_ext, walltime=cactus_fast_walltime()) + merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=maf_ids, walltime=cactus_fast_walltime()) if options.index: index_job = merge_job.addFollowOnJobFn(taffy_index, merge_job.rv(), output_name + output_ext, disk=int(1.1 * maf_size), memory=cactus_clamp_memory(maf_size / 10)) - index_job.addFollowOnJobFn(export_file, index_job.rv(), output_name + output_ext + '.tai') + index_job.addFollowOnJobFn(export_file, index_job.rv(), output_name + output_ext + '.tai', walltime=cactus_fast_walltime()) if options.coverage: @@ -680,7 +681,7 @@ def hal2maf_merge_all(job, output_dicts, options, genome_list): genome_list, options, disk=int(1.1 * maf_size), memory=cactus_clamp_memory(maf_size / 10)) - coverage_job.addFollowOnJobFn(export_file, coverage_job.rv(), output_name + output_ext + '.cov.tsv') + coverage_job.addFollowOnJobFn(export_file, coverage_job.rv(), output_name + output_ext + '.cov.tsv', walltime=cactus_fast_walltime()) return export_job.rv() diff --git a/src/cactus/maf/cactus_maf2bigmaf.py b/src/cactus/maf/cactus_maf2bigmaf.py index 4ea7a16e4..ec5aeb710 100644 --- a/src/cactus/maf/cactus_maf2bigmaf.py +++ b/src/cactus/maf/cactus_maf2bigmaf.py @@ -14,12 +14,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import cactus_clamp_memory @@ -36,6 +36,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("mafFile", help = "MAF file to convert to BigMaf (can be gzipped)") parser.add_argument("outFile", help = "Output bigMaf file (.bb)") @@ -93,7 +94,7 @@ def main(): if options.halFile: hal_id = toil.importFile(options.halFile) - bigmaf_id_dict = toil.start(Job.wrapJobFn(maf2bigmaf_workflow, config, options, maf_id, hal_id)) + bigmaf_id_dict = toil.start(Job.wrapJobFn(maf2bigmaf_workflow, config, options, maf_id, hal_id, walltime=cactus_fast_walltime())) #export the big maf out_bm_path = makeURL(options.outFile) diff --git a/src/cactus/maf/cactus_phast.py b/src/cactus/maf/cactus_phast.py index 51219ba5f..99a891dad 100644 --- a/src/cactus/maf/cactus_phast.py +++ b/src/cactus/maf/cactus_phast.py @@ -23,12 +23,12 @@ import timeit import xml.etree.ElementTree as ET -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory from cactus.shared.common import cactus_cpu_count @@ -51,6 +51,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("inMaf", help="Input alignment as produced by cactus-hal2maf. " "Accepts .maf, .maf.gz, .taf, or .taf.gz (auto-detected " @@ -415,7 +416,7 @@ def main(): toil.start(Job.wrapJobFn(phast_workflow, config, options, maf_id, tai_id, tai_built, hal_id, ann_id, model_id, - bed_id)) + bed_id, walltime=cactus_fast_walltime())) end_time = timeit.default_timer() logger.info("cactus-phast finished in {} seconds".format(end_time - start_time)) @@ -814,20 +815,20 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # Convenience sed script: maps phast canonical names back to HAL genome # names. Only relevant when at least one HAL genome name contains '.'; # the job no-ops otherwise. - setup_job.addFollowOnJobFn(export_name_map_job, options, species_list) + setup_job.addFollowOnJobFn(export_name_map_job, options, species_list, walltime=cactus_fast_walltime()) # build .tai if missing, else just chain through the existing one if tai_built: idx_job = setup_job.addFollowOnJobFn(taffy_index_job, maf_id, os.path.basename(options.inMaf), disk=int(maf_id.size * 1.1)) tai_id = idx_job.rv() - idx_job.addFollowOnJobFn(export_file, tai_id, tai_export_path(options)) + idx_job.addFollowOnJobFn(export_file, tai_id, tai_export_path(options), walltime=cactus_fast_walltime()) plan_parent = idx_job else: plan_parent = setup_job plan_job = plan_parent.addFollowOnJobFn(plan_chunks_job, options, ref_seq_lengths, tai_id, - bed_id, aligned_contigs) + bed_id, aligned_contigs, walltime=cactus_fast_walltime()) chunk_specs = plan_job.rv() # The single multi-core chunker job: localizes the source MAF once, @@ -867,7 +868,7 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # ----- phyloFit branch (also runs in phyloP mode if no model was given) ----- if need_train: train_job = chunk_job.addFollowOnJobFn(train_workflow, options, chunks, - species_list, tree_str, ann_id) + species_list, tree_str, ann_id, walltime=cactus_fast_walltime()) # train_workflow returns the model file id trained_model_id = train_job.rv() if options.mode == 'phyloFit': @@ -890,7 +891,7 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ for sub in track_inputs: score_job = score_parent.addFollowOnJobFn(phyloP_workflow, options, chunks, model_id, ref_seq_lengths, species_list, - sub, effective_root_name) + sub, effective_root_name, walltime=cactus_fast_walltime()) track_rvs.append(score_job.rv()) return track_rvs @@ -1301,7 +1302,7 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): # projection); with --root, it's restricted to clade leaves. leaves_csv = ','.join(phast_name(g) for g in newick_leaves(tree_str)) extract_job = job.addChildJobFn(extract_4d_all, options, chunks, ann_id, - species_list, leaves_csv) + species_list, leaves_csv, walltime=cactus_fast_walltime()) ss_results = extract_job.rv() # flat list of ss file ids (one per chunk-group) # 4d-site SS files are tiny even at 447-way (~300 MB aggregated). 4 GiB @@ -1310,7 +1311,7 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): species_list, tree_str, disk=4 * 1024**3) aggregate_id = aggregate_job.rv() - aggregate_job.addFollowOnJobFn(export_file, aggregate_id, ss_export_path(options)) + aggregate_job.addFollowOnJobFn(export_file, aggregate_id, ss_export_path(options), walltime=cactus_fast_walltime()) # phyloFit on 577-way × 1.4 GB SS peaked at ~300 MiB; default memory is # fine unless the user overrides via --phyloFitMemory. @@ -1329,7 +1330,7 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): else: export_parent = fit_job - export_parent.addFollowOnJobFn(export_file, model_id, model_export_path(options)) + export_parent.addFollowOnJobFn(export_file, model_id, model_export_path(options), walltime=cactus_fast_walltime()) return model_id @@ -1571,7 +1572,7 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis else: RealtimeLogger.info('phyloP track: global conservation (no --subtree)') score_job = job.addChildJobFn(phyloP_all, options, chunks, model_id, species_list, - track_subtree) + track_subtree, walltime=cactus_fast_walltime()) per_chunk_wigs = score_job.rv() # flat list of (contig, start, wig_id_or_None) per chunk # disk estimates: per-base wig text is ~10 bytes/ref_bp uncompressed; bgzip @@ -1586,7 +1587,7 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis disk=merge_disk, cores=merge_cores) wig_id = merge_job.rv() - merge_job.addFollowOnJobFn(export_file, wig_id, wig_export_path(options, track_subtree)) + merge_job.addFollowOnJobFn(export_file, wig_id, wig_export_path(options, track_subtree), walltime=cactus_fast_walltime()) if options.bigwig: # decompressed wig + .bw output + bbiFile scratch @@ -1602,7 +1603,7 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis memory=cactus_clamp_memory(bw_mem), cores=merge_cores) bw_job.addFollowOnJobFn(export_file, bw_job.rv(), - bigwig_export_path(options, track_subtree)) + bigwig_export_path(options, track_subtree), walltime=cactus_fast_walltime()) return wig_id diff --git a/src/cactus/paf/local_alignment.py b/src/cactus/paf/local_alignment.py index 193df4bd3..44b4a7736 100755 --- a/src/cactus/paf/local_alignment.py +++ b/src/cactus/paf/local_alignment.py @@ -20,7 +20,7 @@ import copy from Bio import SeqIO from cactus.paf.paf import get_event_pairs, get_leaves, get_node, get_distances -from cactus.shared.common import cactus_call, getOptionalAttrib, zip_gz +from cactus.shared.common import cactus_call, getOptionalAttrib, zip_gz, cactus_fast_walltime from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.preprocessor.unmasking import unmask_contigs_all from cactus.preprocessor.cactus_preprocessor import clean_if_different @@ -421,7 +421,7 @@ def invert_alignments(job, alignment_file): def make_ingroup_to_outgroup_alignments_0(job, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # Generate the alignments fle alignment_file = job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, - event_names_to_sequences, distances, params).rv() + event_names_to_sequences, distances, params, walltime=cactus_fast_walltime()).rv() # Invert the final alignment so that the query is the outgroup and the target is the ingroup return job.addFollowOnJobFn(invert_alignments, alignment_file).rv() @@ -493,7 +493,7 @@ def make_ingroup_to_outgroup_alignments_2(job, alignments, ingroup_event, outgro # recursively make alignments with the remaining outgroups alignments2 = root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, - event_names_to_sequences, distances, params).rv() + event_names_to_sequences, distances, params, walltime=cactus_fast_walltime()).rv() return root_job.addFollowOnJobFn(make_ingroup_to_outgroup_alignments_3, ingroup_event, event_names_to_sequences[ingroup_event.iD], alignments, alignments2).rv() @@ -654,7 +654,7 @@ def chain_alignments(job, alignment_files, alignment_names, reference_event_name memory=cactus_clamp_memory(4 * split_size)).rv() ) - return job.addFollowOnJobFn(merge_processed_alignments, processed_rvs).rv() + return job.addFollowOnJobFn(merge_processed_alignments, processed_rvs, walltime=cactus_fast_walltime()).rv() def chain_tile_trim_filter_one_contig(job, split_file_id, reference_event_name, params): @@ -738,9 +738,9 @@ def merge_processed_alignments(job, processed_file_ids): def sanitize_then_make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancestor_event_string, params, output_path=None): - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, event_names_to_sequences) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, event_names_to_sequences, walltime=cactus_fast_walltime()) paf_job = sanitize_job.addFollowOnJobFn(make_paf_alignments, event_tree_string, sanitize_job.rv(), - ancestor_event_string, params) + ancestor_event_string, params, walltime=cactus_fast_walltime()) # gzip the output if requested if output_path and output_path.endswith('.gz'): gzip_job = paf_job.addFollowOnJobFn(zip_gz, output_path, paf_job.rv()) @@ -792,7 +792,7 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest if getOptionalAttrib(lastz_params_node.find("unmask"), 'action', typeFn=str, default='none') != 'none': ingroups = [ingroup.iD for ingroup in ingroup_events] # Pass a copy of event_names_to_sequences to unmask_job to avoid circular reference - unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params) + unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params, walltime=cactus_fast_walltime()) for i,ingroup in enumerate(ingroups): event_names_to_sequences[ingroup] = unmask_job.rv(i) new_root_job = Job() @@ -822,7 +822,7 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest # for each ingroup make alignments to the outgroups if int(params.find("blast").attrib["trimIngroups"]): # Trim the ingroup sequences outgroup_alignments = [root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_0, ingroup, outgroup_events, - dict(event_names_to_sequences), distances, params).rv() + dict(event_names_to_sequences), distances, params, walltime=cactus_fast_walltime()).rv() for ingroup in ingroup_events] if len(outgroup_events) > 0 else [] else: outgroup_alignments = [root_job.addChildJobFn(make_chunked_alignments, @@ -844,11 +844,11 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest ingroup_alignments, ingroup_alignment_names, outgroup_alignments, outgroup_alignment_names, ancestor_event_string, params, - total_sequence_size=total_sequence_size).rv() + total_sequence_size=total_sequence_size, walltime=cactus_fast_walltime()).rv() # Delete the unmasked fastas (todo: should we do the unmasking somewhere further upstream?) for ingroup in ingroup_events: - root_job.addFollowOnJobFn(clean_if_different, event_names_to_sequences[ingroup.iD], input_sequence_map[ingroup.iD]) + root_job.addFollowOnJobFn(clean_if_different, event_names_to_sequences[ingroup.iD], input_sequence_map[ingroup.iD], walltime=cactus_fast_walltime()) return root_job.addFollowOnJobFn(chain_alignments, ingroup_alignments + outgroup_alignments, ingroup_alignment_names + outgroup_alignment_names, diff --git a/src/cactus/preprocessor/cactus_preprocessor.py b/src/cactus/preprocessor/cactus_preprocessor.py index b4cfa18a3..275ef5b32 100644 --- a/src/cactus/preprocessor/cactus_preprocessor.py +++ b/src/cactus/preprocessor/cactus_preprocessor.py @@ -19,7 +19,7 @@ from sonLib.bioio import getTempDirectory from toil.common import Toil from toil.job import Job -from cactus.shared.common import cactus_call +from cactus.shared.common import cactus_call, cactus_fast_walltime from cactus.shared.common import RoundedJob from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import runGetChunks @@ -36,7 +36,7 @@ from toil.statsAndLogging import set_logging_from_options from toil.realtimeLogger import RealtimeLogger -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.preprocessor.checkUniqueHeaders import checkUniqueHeaders from cactus.preprocessor.lastzRepeatMasking.cactus_lastzRepeatMask import LastzRepeatMaskJob from cactus.preprocessor.lastzRepeatMasking.cactus_lastzRepeatMask import RepeatMaskOptions @@ -305,7 +305,7 @@ def run(self, fileStore): ppJob = self.addChild(PreprocessSequence(prepOptions, self.inSequenceID)) outSeqID = ppJob.rv() - self.addFollowOnJobFn(clean_if_different, self.inSequenceID, outSeqID) + self.addFollowOnJobFn(clean_if_different, self.inSequenceID, outSeqID, walltime=cactus_fast_walltime()) else: logger.info("Skipping inactive preprocessor {}".format(prepNode.attrib["preprocessJob"])) outSeqID = self.inSequenceID @@ -419,7 +419,7 @@ def stageWorkflow(outputSequenceDir, configNode, inputSequences, toil, restart=F inputSequenceIDs.append(toil.importFile(makeURL(seq))) maskFileID = toil.importFile(makeURL(maskFile)) if maskFile else None unzip_job = Job.wrapJobFn(unzip_then_pp, configNode, inputSequences, inputSequenceIDs, inputEventNames, - maskFile, maskFileID, maskAction, minLength) + maskFile, maskFileID, maskAction, minLength, walltime=cactus_fast_walltime()) outputSequenceIDs = toil.start(unzip_job) else: outputSequenceIDs = toil.restart() @@ -435,13 +435,13 @@ def stageWorkflow(outputSequenceDir, configNode, inputSequences, toil, restart=F def unzip_then_pp(job, config_node, input_fa_paths, input_fa_ids, input_event_names, mask_file_path, mask_file_id, mask_file_action, min_length): """ unzip then preprocess """ - unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids) + unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids, walltime=cactus_fast_walltime()) if mask_file_id is not None: - mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id]) + mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id], walltime=cactus_fast_walltime()) config_node = mask_unzip_job.addFollowOnJobFn(maskJobOverride, config_node, mask_file_path, mask_unzip_job.rv(0), mask_file_action, min_length, disk=mask_file_id.size*20).rv() pp_job = unzip_job.addFollowOn(CactusPreprocessor([unzip_job.rv(i) for i in range(len(input_fa_ids))], config_node, eventNames=input_event_names)) - zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0]) + zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0], walltime=cactus_fast_walltime()) return zip_job.rv() def runCactusPreprocessor(outputSequenceDir, configFile, inputSequences, toilDir): @@ -453,6 +453,7 @@ def runCactusPreprocessor(outputSequenceDir, configFile, inputSequences, toilDir def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("inSeqFile", type=str, nargs='?', default=None, help = "Input Seq file") parser.add_argument("outSeqFile", type=str, nargs='?', default=None, help = "Output Seq file (ex generated with cactus-prepare)") parser.add_argument("--configFile", default=os.path.join(cactusRootPath(), "cactus_progressive_config.xml")) diff --git a/src/cactus/preprocessor/unmasking.py b/src/cactus/preprocessor/unmasking.py index e2e42d208..efdab86b7 100644 --- a/src/cactus/preprocessor/unmasking.py +++ b/src/cactus/preprocessor/unmasking.py @@ -5,7 +5,7 @@ from Bio.SeqRecord import SeqRecord import os from toil.job import Job -from cactus.shared.common import cactus_clamp_memory, catFiles, clean_jobstore_files +from cactus.shared.common import cactus_clamp_memory, catFiles, clean_jobstore_files, cactus_fast_walltime from cactus.shared.common import cactus_call, getOptionalAttrib from toil.realtimeLogger import RealtimeLogger from cactus.preprocessor.cactus_preprocessor import CactusPreprocessor @@ -98,7 +98,7 @@ def unmask_contigs_one(job, event, fasta_id, params): disk=fasta_id.size * 4) fasta_id = fa_merge_job.rv() # delete the pp_id - fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id]) + fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id], walltime=cactus_fast_walltime()) else: RealtimeLogger.warning('Remasking activated but no maskers active in preprocessor: doing nothing') diff --git a/src/cactus/progressive/cactus_prepare.py b/src/cactus/progressive/cactus_prepare.py index e3a1dbf59..313269541 100644 --- a/src/cactus/progressive/cactus_prepare.py +++ b/src/cactus/progressive/cactus_prepare.py @@ -32,7 +32,7 @@ from cactus.shared.common import findRequiredNode from cactus.shared.common import makeURL, cactus_call, RoundedJob from cactus.shared.common import write_s3, has_s3, get_aws_region -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_clamp_memory from toil.job import Job @@ -51,6 +51,7 @@ def main(toil_mode=False): parser = ArgumentParser() if toil_mode: parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("--latest", dest="latest", action="store_true", help="Use the latest version of the docker container " "rather than pulling one matching this version of cactus") @@ -1265,6 +1266,7 @@ def main_hal2fasta(): """ cli wrapper for hal2fasta so that cactus-perpare can send it to slurm consistently with other cactus commands """ parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help="input HAL file") parser.add_argument("genome", help="Genome to convert to FASTA") @@ -1332,6 +1334,7 @@ def hal2fasta(job, hal_id, hal_path, genome, fa_path): def main_hal_append_subtrees(): """ toil wrapper for halAppendSubtree command(s) """ parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("tgtFile", help="input HAL file to append to") parser.add_argument("subFiles", nargs='+', help="subtree HAL files to append") diff --git a/src/cactus/progressive/cactus_progressive.py b/src/cactus/progressive/cactus_progressive.py index 61ff4d963..17373a7af 100755 --- a/src/cactus/progressive/cactus_progressive.py +++ b/src/cactus/progressive/cactus_progressive.py @@ -18,7 +18,7 @@ from base64 import b64encode from toil.lib.bioio import getTempFile -from cactus.shared.common import cactus_cpu_count +from cactus.shared.common import cactus_cpu_count, cactus_fast_walltime from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options from toil.realtimeLogger import RealtimeLogger @@ -33,7 +33,7 @@ from cactus.shared.version import cactus_commit from cactus.shared.common import cactusRootPath from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import write_s3 from cactus.shared.common import cactus_clamp_memory from cactus.shared.common import clean_jobstore_files @@ -68,7 +68,7 @@ def preprocess_all(job, options, config_node, input_seq_id_map): pp_seq_ids[event] = preprocessor_job.rv(i) # do the logging and checkpointing - root_job.addFollowOnJobFn(save_preprocessed_files, options, config_node, pp_seq_ids) + root_job.addFollowOnJobFn(save_preprocessed_files, options, config_node, pp_seq_ids, walltime=cactus_fast_walltime()) return pp_seq_ids @@ -137,7 +137,7 @@ def progressive_schedule(job, options, config_node, seq_id_map, tree, og_map, ro # to be consistent with pre-refactor (and work with updating tests), we include the root when its id is input if event in seq_id_map and seq_id_map[event]: event_id_map[event] = seq_id_map[event] - event_job = Job.wrapJobFn(progressive_step, options, config_node, event_id_map, tree, og_map, event) + event_job = Job.wrapJobFn(progressive_step, options, config_node, event_id_map, tree, og_map, event, walltime=cactus_fast_walltime()) job_table[event] = event_job for dep in dep_table[event]: if dep in job_table: @@ -174,7 +174,7 @@ def progressive_step(job, options, config_node, seq_id_map, tree, og_map, event) # do the blast paf_job = job.addChildJobFn(make_paf_alignments, NXNewick().writeString(spanning_tree), - subtree_eventmap, event, config_node).encapsulate() + subtree_eventmap, event, config_node, walltime=cactus_fast_walltime()).encapsulate() outgroups = og_map[event] if event in og_map else [] # trim the outgroups @@ -182,14 +182,14 @@ def progressive_step(job, options, config_node, seq_id_map, tree, og_map, event) trim_sequences = paf_job.addChildJobFn(trim_unaligned_sequences, [subtree_eventmap[i] for i in outgroups], paf_job.rv(), config_node) cons_job = paf_job.addFollowOnJobFn(progressive_step_2, trim_sequences.rv(), options, config_node, subtree_eventmap, - spanning_tree, og_map, event) + spanning_tree, og_map, event, walltime=cactus_fast_walltime()) else: # Without outgroup trimming (or if there are no outgroups to trim) cons_job = paf_job.addChildJobFn(cactus_cons_with_resources, spanning_tree, event, config_node, subtree_eventmap, og_map, paf_job.rv(), cons_cores=options.consCores, cons_memory=options.consMemory, - intermediate_results_url=options.intermediateResultsUrl) + intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_fast_walltime()) # erase the paf since its now longer needed - cons_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[paf_job.rv()]) + cons_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[paf_job.rv()], walltime=cactus_fast_walltime()) return cons_job.rv() @@ -203,7 +203,7 @@ def progressive_step_2(job, trimmed_outgroups_and_alignments, options, config_no # now do consolidated return job.addChildJobFn(cactus_cons_with_resources, spanning_tree, event, config_node, subtree_eventmap, og_map, pafs, cons_cores=options.consCores, cons_memory=options.consMemory, - intermediate_results_url=options.intermediateResultsUrl).rv() + intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_fast_walltime()).rv() def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=None, cacheBytes=None, @@ -300,11 +300,11 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i ''' run the entire progressive workflow ''' # run the usual unzip / rename, even before preprocessing - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, walltime=cactus_fast_walltime()) # start with the preprocessor if not options.skipPreprocessor: - pp_job = sanitize_job.addFollowOnJobFn(preprocess_all, options, config_node, sanitize_job.rv()) + pp_job = sanitize_job.addFollowOnJobFn(preprocess_all, options, config_node, sanitize_job.rv(), walltime=cactus_fast_walltime()) seq_id_map = pp_job.rv() sanitize_job = pp_job else: @@ -322,7 +322,7 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i scaled_tree = get_ancestor_scaled_tree(mc_tree, root_event, max_div, branch_scale=options.branchScale, upweight_ancestors=upweight_ancestors) - progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event) + progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event, walltime=cactus_fast_walltime()) # then do the hal export hal_export_job = progressive_job.addFollowOnJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, @@ -332,6 +332,7 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file") parser.add_argument("outputHal", type=str, help = "Output HAL file") @@ -493,7 +494,7 @@ def main(): loadDnaBrnnModel(toil, config_node) # run the whole workflow - hal_id = toil.start(Job.wrapJobFn(progressive_workflow, options, config_node, mc_tree, og_map, input_seq_id_map)) + hal_id = toil.start(Job.wrapJobFn(progressive_workflow, options, config_node, mc_tree, og_map, input_seq_id_map, walltime=cactus_fast_walltime())) toil.exportFile(hal_id, makeURL(options.outputHal)) diff --git a/src/cactus/refmap/cactus_graphmap.py b/src/cactus/refmap/cactus_graphmap.py index 2e534bf7c..ad15b545a 100644 --- a/src/cactus/refmap/cactus_graphmap.py +++ b/src/cactus/refmap/cactus_graphmap.py @@ -17,12 +17,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import unzip_gz, zip_gz @@ -42,6 +42,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (will be modified if necessary to include graph Fasta sequence) (or chromfile with --batch)") parser.add_argument("minigraphGFA", nargs='?', default='', type=str, @@ -233,7 +234,7 @@ def graph_map(options): # run the workflow # output_dict is chrom -> paf_id, gfa_fa_id, gaf_id, unfiltered_paf_id, paf_filter_log, paf_was_filtered - output_dict = toil.start(Job.wrapJobFn(minigraph_batch_workflow, options, config_wrapper, input_dict, graph_event, True)) + output_dict = toil.start(Job.wrapJobFn(minigraph_batch_workflow, options, config_wrapper, input_dict, graph_event, True, walltime=cactus_fast_walltime())) export_graphmap_output(options, config_node, input_map, output_dict, toil) @@ -304,7 +305,7 @@ def minigraph_batch_workflow(job, options, config, input_dict, graph_event, sani else: chrom_options = options mgwf_job = job.addChildJobFn(minigraph_workflow, chrom_options, config, seq_id_map, gfa_id, graph_event, - sanitize, ref_collapse_paf_id, pansn_gfa_input) + sanitize, ref_collapse_paf_id, pansn_gfa_input, walltime=cactus_fast_walltime()) output_dict[chrom] = mgwf_job.rv() return output_dict @@ -325,7 +326,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa # enforce unique prefixes and unzip fastas if sanitize: - sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) seq_id_map = sanitize_job.rv() # add unique prefixes to the input PAF @@ -356,7 +357,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa gfa_id = gfa_unzip_job.rv() gfa_id_size *= 10 options.minigraphGFA = options.minigraphGFA[:-3] - paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event) + paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event, walltime=cactus_fast_walltime()) root_job.addFollowOn(paf_job) collapse_paf_id = ref_collapse_paf_id @@ -372,7 +373,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa paf_job.addFollowOn(gfa2paf_job) collapse_mode = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "collapse", typeFn=str, default="none") if collapse_mode in ['reference', 'all', 'nonref']: - collapse_job = paf_job.addChildJobFn(self_align_all, config, seq_id_map, options.reference, collapse_mode) + collapse_job = paf_job.addChildJobFn(self_align_all, config, seq_id_map, options.reference, collapse_mode, walltime=cactus_fast_walltime()) if ref_collapse_paf_id: collapse_paf_id = collapse_job.addFollowOnJobFn(merge_pafs, {"1":collapse_job.rv(), "2":ref_collapse_paf_id}, diff --git a/src/cactus/refmap/cactus_graphmap_join.py b/src/cactus/refmap/cactus_graphmap_join.py index 2d22e081e..37324784b 100644 --- a/src/cactus/refmap/cactus_graphmap_join.py +++ b/src/cactus/refmap/cactus_graphmap_join.py @@ -35,12 +35,12 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.multiCactusTree import MultiCactusTree -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import unzip_gz, write_s3 @@ -64,6 +64,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("--vg", required=False, nargs='+', default=None, help = "Input vg files (PackedGraph or HashGraph format)") parser.add_argument("--vgFull", nargs='+', default=None, @@ -583,7 +584,7 @@ def graphmap_join(options): vg_ids = bypass_full_ids or bypass_clip_ids or bypass_filter_ids wf_output = toil.start(Job.wrapJobFn(graphmap_join_workflow, options, config, vg_ids, hal_ids, sv_gfa_ids, - bypass_full_ids, bypass_clip_ids, bypass_filter_ids)) + bypass_full_ids, bypass_clip_ids, bypass_filter_ids, walltime=cactus_fast_walltime())) else: # load up the vgs vg_ids = [] @@ -591,7 +592,7 @@ def graphmap_join(options): vg_ids.append(toil.importFile(makeURL(vg_path))) # run the workflow - wf_output = toil.start(Job.wrapJobFn(graphmap_join_workflow, options, config, vg_ids, hal_ids, sv_gfa_ids)) + wf_output = toil.start(Job.wrapJobFn(graphmap_join_workflow, options, config, vg_ids, hal_ids, sv_gfa_ids, walltime=cactus_fast_walltime())) #export the split data export_join_data(toil, options, wf_output[0], wf_output[1], wf_output[2], wf_output[3], wf_output[4], wf_output[5]) @@ -750,7 +751,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, hal_id_dict = hal_merge_job.rv() out_dicts.append(hal_id_dict) # delete the chromosome hals - hal_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids) + hal_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_fast_walltime()) # optional minigraph gfa merge if sv_gfa_ids: @@ -759,7 +760,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, sv_gfa_id_dict = sv_gfa_merge_job.rv() out_dicts.append(sv_gfa_id_dict) # delete the chromosome gfas - sv_gfa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=sv_gfa_ids) + sv_gfa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=sv_gfa_ids, walltime=cactus_fast_walltime()) if options.indexMemory: index_mem = options.indexMemory @@ -858,7 +859,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vcf_ref in options.vcfReference: vcf_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, workflow_phase, index_mem, vcf_ref, phase_vg_ids, - ref_fasta_job.rv() if ref_fasta_job else None) + ref_fasta_job.rv() if ref_fasta_job else None, walltime=cactus_fast_walltime()) if ref_fasta_job: ref_fasta_job.addFollowOn(vcf_job) out_dicts.append(vcf_job.rv()) @@ -2102,7 +2103,7 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, vcf_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, tag.rstrip('.'), index_mem, vcf_ref, phase_vg_ids, ref_fasta_dict, vcftag=vcftag, - is_gref=is_gref, decon_L=decon_L) + is_gref=is_gref, decon_L=decon_L, walltime=cactus_fast_walltime()) out_dicts.append(vcf_job.rv()) # optional haplo index diff --git a/src/cactus/refmap/cactus_graphmap_split.py b/src/cactus/refmap/cactus_graphmap_split.py index dd413612f..3bd355c19 100644 --- a/src/cactus/refmap/cactus_graphmap_split.py +++ b/src/cactus/refmap/cactus_graphmap_split.py @@ -14,12 +14,12 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.multiCactusTree import MultiCactusTree -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import unzip_gz, write_s3 @@ -42,6 +42,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (gzipped fastas supported)") parser.add_argument("minigraphGFA", help = "Minigraph-compatible reference graph in GFA format (can be gzipped)") @@ -175,7 +176,7 @@ def cactus_graphmap_split(options): # run the workflow wf_output = toil.start(Job.wrapJobFn(graphmap_split_workflow, options, config, input_seq_id_map, input_name_map, gfa_id, options.minigraphGFA, - paf_id, options.graphmapPAF)) + paf_id, options.graphmapPAF, walltime=cactus_fast_walltime())) #export the split data export_split_data(toil, wf_output[0], wf_output[1], wf_output[2], wf_output[3], options.outDir, config) @@ -208,7 +209,7 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ # fix up the headers if sanitize: - sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) seq_id_map = sanitize_job.rv() else: sanitize_job = Job() @@ -250,7 +251,7 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ mask_bed_id = None if options.maskFilter: - mask_bed_id = sanitize_job.addFollowOnJobFn(get_mask_bed, seq_id_map, options.maskFilter).rv() + mask_bed_id = sanitize_job.addFollowOnJobFn(get_mask_bed, seq_id_map, options.maskFilter, walltime=cactus_fast_walltime()).rv() # use rgfa-split to split the gfa and paf up by contig split_gfa_job = root_job.addFollowOnJobFn(split_gfa, config, gfa_id, [paf_id], ref_contigs, @@ -259,14 +260,14 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ memory=cactus_clamp_memory((gfa_size + paf_size) * 3)) # use the output of the above splitting to do the fasta splitting - split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0)) + split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0), walltime=cactus_fast_walltime()) # gather everythign up into a table - gather_fas_job = split_fas_job.addFollowOnJobFn(gather_fas, split_gfa_job.rv(0), split_fas_job.rv(0), split_fas_job.rv(1)) + gather_fas_job = split_fas_job.addFollowOnJobFn(gather_fas, split_gfa_job.rv(0), split_fas_job.rv(0), split_fas_job.rv(1), walltime=cactus_fast_walltime()) # lump "other" contigs together into one file (to make fewer align jobs downstream) bin_other_job = gather_fas_job.addFollowOnJobFn(bin_other_contigs, config, ref_contigs, options.otherContig, gather_fas_job.rv(0), - disk=(gfa_size + paf_size) * 2) + disk=(gfa_size + paf_size) * 2, walltime=cactus_fast_walltime()) # return all the files, as well as the 2 split logs return (seq_name_map, bin_other_job.rv(), split_gfa_job.rv(1), gather_fas_job.rv(1)) diff --git a/src/cactus/refmap/cactus_minigraph.py b/src/cactus/refmap/cactus_minigraph.py index 5a238a0c2..fa8e5dee7 100644 --- a/src/cactus/refmap/cactus_minigraph.py +++ b/src/cactus/refmap/cactus_minigraph.py @@ -17,12 +17,12 @@ import gzip from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles, write_s3 from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import clean_jobstore_files @@ -43,6 +43,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (or chromfile with --batch)") parser.add_argument("outputGFA", help = "Output Minigraph GFA (or directory in --batch mode)") @@ -124,7 +125,7 @@ def main(): input_dict = minigraph_construct_import_sequences(options, config_wrapper, input_seqfiles, toil) # output_dict: chrom-> (gfa_id, pansn_gfa_id, train_id) - output_dict = toil.start(Job.wrapJobFn(minigraph_construct_batch_workflow, options, config_node, input_dict, options.outputGFA)) + output_dict = toil.start(Job.wrapJobFn(minigraph_construct_batch_workflow, options, config_node, input_dict, options.outputGFA, walltime=cactus_fast_walltime())) export_minigraph_construct_output(options, input_seqfiles, output_dict, toil) @@ -263,7 +264,7 @@ def minigraph_construct_batch_workflow(job, options, config_node, input_dict, gf gfa_path = os.path.join(options.outputGFA, '{}.gfa.gz'.format(chrom)) else: gfa_path = options.outputGFA - mgwf_job = job.addChildJobFn(minigraph_construct_workflow, options, config_node, seq_id_map, seq_order, gfa_path, sanitize) + mgwf_job = job.addChildJobFn(minigraph_construct_workflow, options, config_node, seq_id_map, seq_order, gfa_path, sanitize, walltime=cactus_fast_walltime()) output_dict[chrom] = mgwf_job.rv() return output_dict @@ -281,7 +282,7 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde seq_id_map, seq_order = refonly_seq_id_map, refonly_seq_order ref_size = seq_id_map[options.reference[0]].size if sanitize: - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) sanitized_seq_id_map = sanitize_job.rv() else: sanitized_seq_id_map = seq_id_map @@ -290,12 +291,12 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde xml_node = findRequiredNode(config_node, "graphmap") sort_type = getOptionalAttrib(xml_node, "minigraphSortInput", str, default=None) if sort_type == "mash" and len(seq_id_map) > 2: - sort_job = sanitize_job.addFollowOnJobFn(sort_minigraph_input_with_mash, options, config_node, sanitized_seq_id_map, seq_order) + sort_job = sanitize_job.addFollowOnJobFn(sort_minigraph_input_with_mash, options, config_node, sanitized_seq_id_map, seq_order, walltime=cactus_fast_walltime()) seq_order = sort_job.rv() prev_job = sort_job else: prev_job = sanitize_job - minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_in_batches, options, config_node, sanitized_seq_id_map, seq_order, gfa_path) + minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_in_batches, options, config_node, sanitized_seq_id_map, seq_order, gfa_path, walltime=cactus_fast_walltime()) train_id = None if options.lastTrain and len(seq_id_map) > 1: # note: somehow last training memory overruns don't seem to be detected by slurm so we @@ -344,7 +345,7 @@ def sort_minigraph_input_with_mash(job, options, config_node, seq_id_map, seq_or disk = 2 * sum(seq_id_map[x].size for x in names) + seq_id_map[seq_order[0]].size).rv() dist_maps.append(dist_map) - return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps).rv() + return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps, walltime=cactus_fast_walltime()).rv() def mash_sketch(job, ref_seq, seq_id_map): """ get the sketch """ @@ -495,7 +496,7 @@ def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_or if prev_job: prev_job.addFollowOn(minigraph_job) # delete the output of the previous batch from the job store - minigraph_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[prev_job.rv()]) + minigraph_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[prev_job.rv()], walltime=cactus_fast_walltime()) else: job.addChild(minigraph_job) prev_job = minigraph_job diff --git a/src/cactus/refmap/cactus_pangenome.py b/src/cactus/refmap/cactus_pangenome.py index 826c990f0..5cc96a354 100644 --- a/src/cactus/refmap/cactus_pangenome.py +++ b/src/cactus/refmap/cactus_pangenome.py @@ -21,12 +21,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import clean_jobstore_files @@ -217,6 +217,7 @@ def pangenome_config_overrides(options, config_node): def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (will be modified if necessary to include graph Fasta sequence)") parser.add_argument("--outDir", help = "Output directory", required=True) @@ -293,7 +294,7 @@ def main(): elif genome in input_seq_order: input_seq_order.remove(genome) - toil.start(Job.wrapJobFn(pangenome_end_to_end_workflow, options, config_wrapper, input_seq_id_map, input_path_map, input_seq_order, ref_collapse_paf_id, last_scores_id)) + toil.start(Job.wrapJobFn(pangenome_end_to_end_workflow, options, config_wrapper, input_seq_id_map, input_path_map, input_seq_order, ref_collapse_paf_id, last_scores_id, walltime=cactus_fast_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -370,7 +371,7 @@ def sanitize_fasta_headers_batch(job, chromfile_id_map): seq_id_map = value[0] else: seq_id_map = value - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) if type(value) is list: out_id_map[chrom] = [sanitize_job.rv()] + value[1:] else: @@ -489,7 +490,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ root_job = vcflib_checks(root_job, options, config_node) # sanitize headers (once here, skip in all workflows below) - sanitize_job = root_job.addFollowOnJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = root_job.addFollowOnJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) seq_id_map = sanitize_job.rv() assert type(options.reference) == list @@ -514,13 +515,13 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ else: split_config_node = config_node split_config_wrapper = config_wrapper - minigraph_job = sanitize_job.addFollowOnJobFn(minigraph_construct_workflow, mg_options, split_config_node, seq_id_map, seq_order, sv_gfa_path, sanitize=False) + minigraph_job = sanitize_job.addFollowOnJobFn(minigraph_construct_workflow, mg_options, split_config_node, seq_id_map, seq_order, sv_gfa_path, sanitize=False, walltime=cactus_fast_walltime()) sv_gfa_id = minigraph_job.rv(0) pansn_sv_gfa_id = minigraph_job.rv(1) if not last_scores_id: last_scores_id = minigraph_job.rv(2) # only build reference graph on first pass when doing minigraph-by-chrom pipeline - minigraph_wrapper_job = minigraph_job.addFollowOnJobFn(export_minigraph_wrapper, options, pansn_sv_gfa_id, sv_gfa_path, last_scores_id) + minigraph_wrapper_job = minigraph_job.addFollowOnJobFn(export_minigraph_wrapper, options, pansn_sv_gfa_id, sv_gfa_path, last_scores_id, walltime=cactus_fast_walltime()) # cactus_graphmap paf_path = os.path.join(options.outDir, options.outName + '.paf') @@ -531,31 +532,31 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ gm_options = copy.deepcopy(options) if options.mgSplit: gm_options.collapse = False - graphmap_job = minigraph_wrapper_job.addFollowOnJobFn(minigraph_workflow, gm_options, split_config_wrapper, seq_id_map, sv_gfa_id, graph_event, False, ref_collapse_paf_id, pansn_gfa_input=False) + graphmap_job = minigraph_wrapper_job.addFollowOnJobFn(minigraph_workflow, gm_options, split_config_wrapper, seq_id_map, sv_gfa_id, graph_event, False, ref_collapse_paf_id, pansn_gfa_input=False, walltime=cactus_fast_walltime()) paf_id, gfa_fa_id, gaf_id, unfiltered_paf_id, paf_filter_log = graphmap_job.rv(0), graphmap_job.rv(1), graphmap_job.rv(2), graphmap_job.rv(3), graphmap_job.rv(4) - graphmap_export_job = graphmap_job.addFollowOnJobFn(export_graphmap_wrapper, options, paf_id, paf_path, gaf_id, unfiltered_paf_id, paf_filter_log) + graphmap_export_job = graphmap_job.addFollowOnJobFn(export_graphmap_wrapper, options, paf_id, paf_path, gaf_id, unfiltered_paf_id, paf_filter_log, walltime=cactus_fast_walltime()) # we need to update the seqfile with the phonied in minigraph event - update_seqfile_job = graphmap_export_job.addFollowOnJobFn(update_seqfile, options, seq_id_map, seq_path_map, seq_order, gfa_fa_id, gfa_fa_path, graph_event) + update_seqfile_job = graphmap_export_job.addFollowOnJobFn(update_seqfile, options, seq_id_map, seq_path_map, seq_order, gfa_fa_id, gfa_fa_path, graph_event, walltime=cactus_fast_walltime()) seq_id_map, seq_path_map, seq_name_map = update_seqfile_job.rv(0), update_seqfile_job.rv(1), update_seqfile_job.rv(2) if options.noSplit: # we phony in the entire alignment as one chromsome called 'all' - phony_chromfile_job = update_seqfile_job.addFollowOnJobFn(phony_chromfile, options, paf_path) + phony_chromfile_job = update_seqfile_job.addFollowOnJobFn(phony_chromfile, options, paf_path, walltime=cactus_fast_walltime()) chromfile_path = phony_chromfile_job.rv() split_export_job = phony_chromfile_job else: # cactus_graphmap_split split_job = update_seqfile_job.addFollowOnJobFn(graphmap_split_workflow, options, split_config_wrapper, seq_id_map, seq_name_map, sv_gfa_id, - sv_gfa_path, paf_id, paf_path, sanitize=False, pansn_gfa_input=False) + sv_gfa_path, paf_id, paf_path, sanitize=False, pansn_gfa_input=False, walltime=cactus_fast_walltime()) wf_output = split_job.rv() split_out_path = os.path.join(options.outDir, 'chrom-subproblems') - split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper) + split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper, walltime=cactus_fast_walltime()) chromfile_path = os.path.join(split_out_path, 'chromfile.txt') # clean out some jobstore files we no longer need clean_jobstore_job = split_export_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[seq_id_map] if not options.noSplit else None, - file_ids=[sv_gfa_id, paf_id]) + file_ids=[sv_gfa_id, paf_id], walltime=cactus_fast_walltime()) options.batch = True minigraph_pansn_sv_gfa_ids = [] @@ -565,14 +566,14 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ chromfile_path) input_seqfiles = minigraph_batch_import_job.rv(0) raw_input_map = minigraph_batch_import_job.rv(1) - sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map) + sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map, walltime=cactus_fast_walltime()) input_map = sanitize_job.rv() options.outputGFA='' minigraph_batch_job = sanitize_job.addFollowOnJobFn(minigraph_construct_batch_workflow, options, config_node, - input_map, None, sanitize=False) + input_map, None, sanitize=False, walltime=cactus_fast_walltime()) minigraph_batch_results = minigraph_batch_job.rv() minigraph_batch_export_job = minigraph_batch_job.addFollowOnJobFn(export_minigraph_batch_wrapper, options, config_node, - input_seqfiles, input_map, minigraph_batch_results) + input_seqfiles, input_map, minigraph_batch_results, walltime=cactus_fast_walltime()) # now rerun cactus_graphmap but on a per-chromosome bassis graphmap_input_dict = minigraph_batch_export_job.rv(0) @@ -581,16 +582,16 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ minigraph_pansn_sv_gfa_ids = minigraph_batch_export_job.rv(3) graphmap_batch_job = minigraph_batch_export_job.addFollowOnJobFn(minigraph_batch_workflow, options, config_wrapper, graphmap_input_dict, graph_event, sanitize=False, - pansn_gfa_input=False) + pansn_gfa_input=False, walltime=cactus_fast_walltime()) graphmap_batch_results = graphmap_batch_job.rv() graphmap_batch_export_job = graphmap_batch_job.addFollowOnJobFn(export_graphmap_batch_wrapper, options, config_node, - graphmap_batch_results, input_seqfiles) + graphmap_batch_results, input_seqfiles, walltime=cactus_fast_walltime()) graphmap_file_ids = graphmap_batch_export_job.rv(0) chromfile_path = graphmap_batch_export_job.rv(1) # clean out the jobstore, as cactus_align reads everything from disk - clean_jobstore_job = graphmap_batch_export_job.addFollowOnJobFn(clean_jobstore_files, file_ids=graphmap_file_ids) + clean_jobstore_job = graphmap_batch_export_job.addFollowOnJobFn(clean_jobstore_files, file_ids=graphmap_file_ids, walltime=cactus_fast_walltime()) clean_jobstore_job = clean_jobstore_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=minigraph_output_maps, - file_ids=minigraph_output_ids, allow_none=True) + file_ids=minigraph_output_ids, allow_none=True, walltime=cactus_fast_walltime()) # cactus_align options.scoresFromChromfile = options.lastTrain and options.mgSplit @@ -598,7 +599,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ last_scores_id) align_jobs = align_jobs_make_job.rv() - align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs) + align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs, walltime=cactus_fast_walltime()) results_dict = align_job.rv() align_export_job = align_job.addFollowOnJobFn(export_align_wrapper, options, results_dict) join_options, vg_ids, hal_ids = align_export_job.rv(0), align_export_job.rv(1), align_export_job.rv(2) @@ -608,16 +609,16 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ # hal_ids, and it's also the only thing that frees the chromosome hals from the jobstore, so we # need to do that ourselves here or they'd leak for the rest of the run join_job = align_export_job.addFollowOnJobFn(graphmap_join_workflow, join_options, config_wrapper, vg_ids, - [] if options.noHal else hal_ids, minigraph_pansn_sv_gfa_ids) + [] if options.noHal else hal_ids, minigraph_pansn_sv_gfa_ids, walltime=cactus_fast_walltime()) join_wf_output = join_job.rv() if options.noHal: - join_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids) + join_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_fast_walltime()) # cactus-panpatch exports the chromosome vgs itself (they're the only thing export_join_data # would write, given it turns every other output off), so let it skip this to avoid writing # the biggest output twice if not options.noJoinExport: - join_job.addFollowOnJobFn(export_join_wrapper, join_options, join_wf_output) + join_job.addFollowOnJobFn(export_join_wrapper, join_options, join_wf_output, walltime=cactus_fast_walltime()) return join_options, join_wf_output, seq_id_map diff --git a/src/cactus/refmap/cactus_panpatch.py b/src/cactus/refmap/cactus_panpatch.py index e158edc47..d0ca51ce4 100644 --- a/src/cactus/refmap/cactus_panpatch.py +++ b/src/cactus/refmap/cactus_panpatch.py @@ -28,7 +28,7 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.cactus_prepare import human2bytesN -from cactus.shared.common import importSingularityImage +from cactus.shared.common import importSingularityImage, cactus_fast_walltime, add_cactus_toil_options from cactus.shared.common import makeURL from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory @@ -50,6 +50,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", nargs='?', default=None, help = "Seq file (as with cactus-pangenome), or, with --batch, a chromfile with one " @@ -222,7 +223,7 @@ def main(): ref_collapse_paf_id, last_scores_id, target_fasta_ids)) toil.start(Job.wrapJobFn(panpatch_batch_workflow, options, config_wrapper, run_inputs, - exclude_bed_id)) + exclude_bed_id, walltime=cactus_fast_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -477,18 +478,18 @@ def panpatch_batch_workflow(job, options, config_wrapper, run_inputs, exclude_be for run, pg_options, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, target_fasta_ids in run_inputs: job.addChildJobFn(panpatch_run_workflow, options, pg_options, config_wrapper, run, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, exclude_bed_id, - target_fasta_ids) + target_fasta_ids, walltime=cactus_fast_walltime()) def panpatch_run_workflow(job, options, pg_options, config_wrapper, run, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, exclude_bed_id, target_fasta_ids): """ cactus-pangenome, then panpatch on the chromosome graphs it made """ pangenome_job = job.addChildJobFn(pangenome_end_to_end_workflow, pg_options, config_wrapper, seq_id_map, - seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id) + seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, walltime=cactus_fast_walltime()) # a follow-on of the job hosting the pangenome workflow only runs once that job's entire child # subtree is done, which is what makes this safe pangenome_job.addFollowOnJobFn(panpatch_workflow, options, run, pangenome_job.rv(0), pangenome_job.rv(1), - pangenome_job.rv(2), exclude_bed_id, target_fasta_ids) + pangenome_job.rv(2), exclude_bed_id, target_fasta_ids, walltime=cactus_fast_walltime()) def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_map, exclude_bed_id, target_fasta_ids): """ the pangenome's promises have resolved by now. run one single-threaded panpatch job per @@ -526,7 +527,7 @@ def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_ma target_fasta_ids, memory=cactus_clamp_memory(max(2**32, ref_size * 2)), disk=gather_disk) gather_job.addFollowOnJobFn(export_panpatch_wrapper, options, run, gather_job.rv(), full_vg_ids, vg_names, - disk=sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30) + disk=sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30, walltime=cactus_fast_walltime()) def run_panpatch_chrom(job, options, run, vg_id, vg_name, exclude_bed_id): """ run single-threaded panpatch on one chromosome graph. returns the per-haplotype fastas, the @@ -720,7 +721,7 @@ def export_panpatch_wrapper(job, options, run, output_id_map, full_vg_ids, vg_na job.fileStore.exportFile(vg_id, makeURL(os.path.join(chrom_dir, vg_name))) if not options.keepPangenome: - job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run) + job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run, walltime=cactus_fast_walltime()) def cleanup_pangenome_wrapper(job, options, run): """ the cactus-pangenome output is only ever scratch for us. everything the user asked for has diff --git a/src/cactus/refmap/cactus_refmap.py b/src/cactus/refmap/cactus_refmap.py index 4c2f2c867..9f9ccf904 100644 --- a/src/cactus/refmap/cactus_refmap.py +++ b/src/cactus/refmap/cactus_refmap.py @@ -36,12 +36,12 @@ from cactus.refmap import fasta_preprocessing from cactus.refmap import apply_dipcall_bed_filter -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import makeURL from cactus.shared.common import cactus_call from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import cactusRootPath -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers @@ -139,8 +139,8 @@ def run_cactus_reference_align(job, assembly_files, reference, debug_export=Fals """ Preprocesses assemblies, then runs mappings. """ - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, assembly_files) - mappings = sanitize_job.addFollowOnJobFn(map_all_to_ref, sanitize_job.rv(), reference, debug_export, dipcall_bed_filter, dipcall_vcf_filter).rv() + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, assembly_files, walltime=cactus_fast_walltime()) + mappings = sanitize_job.addFollowOnJobFn(map_all_to_ref, sanitize_job.rv(), reference, debug_export, dipcall_bed_filter, dipcall_vcf_filter, walltime=cactus_fast_walltime()).rv() return mappings def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_bed_filter=False, dipcall_vcf_filter=False): @@ -162,7 +162,7 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b * Filters out all mappings below min_var_len=50k and min_mapq=5 from a lastz file Defaults to False. """ - lead_job = job.addChildJobFn(empty) + lead_job = job.addChildJobFn(empty, walltime=cactus_fast_walltime()) # map all assemblies to the reference. Don't map reference to reference, though. ref_mappings = dict() @@ -228,6 +228,7 @@ def map_a_to_b(job, a, b, dipcall_filter): def get_options(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) # addCactusWorkflowOptions(parser) # ### For quick debugging of apply_dipcall_bed_filter: @@ -327,7 +328,7 @@ def main(): ## Perform alignments: if not toil.options.restart: - alignments = toil.start(Job.wrapJobFn(run_cactus_reference_align, input_seq_id_map, options.reference, options.debug_export, options.dipcall_bed_filter, options.dipcall_vcf_filter)) + alignments = toil.start(Job.wrapJobFn(run_cactus_reference_align, input_seq_id_map, options.reference, options.debug_export, options.dipcall_bed_filter, options.dipcall_vcf_filter, walltime=cactus_fast_walltime())) else: alignments = toil.restart() diff --git a/src/cactus/setup/cactus_align.py b/src/cactus/setup/cactus_align.py index 3bd92d14c..a1b1094c6 100644 --- a/src/cactus/setup/cactus_align.py +++ b/src/cactus/setup/cactus_align.py @@ -15,13 +15,13 @@ import multiprocessing from operator import itemgetter -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.pipeline.cactus_workflow import cactus_cons_with_resources from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree from cactus.progressive.cactus_progressive import export_hal from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import findRequiredNode from cactus.shared.common import getOptionalAttrib from cactus.shared.common import cactus_call @@ -48,6 +48,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (or chromfile with --batch)") parser.add_argument("pafFile", nargs='?', default='', type=str, help = "Pairiwse aliginments (from cactus-blast, cactus-refmap or cactus-graphmap)") @@ -190,7 +191,7 @@ def main(): results_dict = toil.restart() else: align_jobs = make_batch_align_jobs(options, toil) - results_dict = toil.start(Job.wrapJobFn(batch_align_jobs, align_jobs)) + results_dict = toil.start(Job.wrapJobFn(batch_align_jobs, align_jobs, walltime=cactus_fast_walltime())) # when using s3 output urls, things get checkpointed as they're made so no reason to export # todo: make a more unified interface throughout cactus for this @@ -398,7 +399,7 @@ def make_align_job(options, toil, config_wrapper=None, chrom_name=None): do_filter_paf=options.pangenome, chrom_name=chrom_name, scores_id=scores_id, - branch_scale=options.branchScale) + branch_scale=options.branchScale, walltime=cactus_fast_walltime()) return align_job def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, paf_id, paf_path, root_name, og_map, checkpointInfo, doVG, doGFA, delay=0, @@ -422,7 +423,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, apply_scores_to_config(score_dict, config_wrapper.xmlRoot) # unzip the input sequences and enforce unique header prefixes - sanitize_job = head_job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, pangenome=doVG or doGFA or do_filter_paf) + sanitize_job = head_job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, pangenome=doVG or doGFA or do_filter_paf, walltime=cactus_fast_walltime()) new_seq_id_map = sanitize_job.rv() # run pangenome-specific paf filter @@ -446,7 +447,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # run consolidated cons_job = head_job.addFollowOnJobFn(cactus_cons_with_resources, spanning_tree, root_name, config_wrapper.xmlRoot, new_seq_id_map, og_map, paf_id, - cons_cores = cons_cores, cons_memory=cons_memory, chrom_name=chrom_name) + cons_cores = cons_cores, cons_memory=cons_memory, chrom_name=chrom_name, walltime=cactus_fast_walltime()) results = {root_name : (cons_job.rv(1), cons_job.rv(2))} # get the immediate subtree (which is all export_hal can use) @@ -459,7 +460,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, memory_override=cons_memory) # clean out some of the intermediate jobstore files - hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id]) + hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id], walltime=cactus_fast_walltime()) # optionally create the VG if doVG or doGFA: @@ -546,6 +547,7 @@ def main_batch(): cons: less efficient use of resources """ parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("chromFile", help = "chroms file") parser.add_argument("outHal", type=str, help = "Output directory (can be s3://)") @@ -630,7 +632,7 @@ def main_batch(): chrom_dict[chrom] = toil.importFile(makeURL(seqfile)), toil.importFile(makeURL(alnFile)) if chrom in options.configOverrides: options.configOverrides[chrom][1] = toil.importFile(makeURL(options.configOverrides[chrom][0])) - results_dict = toil.start(Job.wrapJobFn(align_toil_batch, chrom_dict, config_id, options)) + results_dict = toil.start(Job.wrapJobFn(align_toil_batch, chrom_dict, config_id, options, walltime=cactus_fast_walltime())) # when using s3 output urls, things get checkpointed as they're made so no reason to export # todo: make a more unified interface throughout cactus for this diff --git a/src/cactus/shared/common.py b/src/cactus/shared/common.py index bf3d83f8d..bd6acabe3 100644 --- a/src/cactus/shared/common.py +++ b/src/cactus/shared/common.py @@ -107,6 +107,13 @@ def cactus_override_toil_options(options): os.environ['CACTUS_MAX_MEMORY'] = str(max_mem) os.environ['CACTUS_DEFAULT_MEMORY'] = str(human2bytes(str(options.defaultMemory)) if options.defaultMemory else 2**31) + # store the "fast" walltime (in seconds) for small coordination jobs here so we can get + # at it without carrying options around, and so it propagates to workers (see + # cactus_fast_walltime). An explicit --fastWalltime wins over any inherited env var. + fast_walltime = getattr(options, 'fastWalltime', None) + if fast_walltime: + os.environ['CACTUS_FAST_WALLTIME'] = str(int(fast_walltime)) + # auto-set cactus_log_memory try: subprocess.check_call(['/usr/bin/time', '-v', 'ls'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -118,6 +125,29 @@ def cactus_clamp_memory(memory_bytes): """ use the environment variables from --maxMemory and --defaultMemory to clamp a given memory value """ return max(min(int(os.environ['CACTUS_MAX_MEMORY']), int(memory_bytes)), int(os.environ['CACTUS_DEFAULT_MEMORY'])) +def cactus_fast_walltime(): + """ Walltime, in seconds, to assign to small "coordination" jobs (ones that just schedule + other jobs or do trivial work) so that Slurm can route them to a fast partition. Set via + --fastWalltime, which stores CACTUS_FAST_WALLTIME so the value reaches workers just like + cactus_clamp_memory (or export CACTUS_FAST_WALLTIME directly). Returns None (ie no walltime + override, so the job falls back to Toil's --defaultWalltime) when it isn't set. """ + val = os.environ.get('CACTUS_FAST_WALLTIME') + if not val: + return None + seconds = int(val) + # treat 0 (or negative) as "disabled" so it stays a no-op rather than requesting a + # 0-second job (which Slurm would map to its shortest partition) + return seconds if seconds > 0 else None + +def add_cactus_toil_options(parser): + """ Add cactus-specific options on top of Toil's default argument parser + (Job.Runner.getDefaultArgumentParser). Call this right after creating the parser. """ + parser.add_argument("--fastWalltime", type=int, default=1800, + help="Walltime, in seconds, to assign to small \"coordination\" jobs so that " + "(on Slurm) Toil can route them to a fast partition [default: 1800 (30 min), " + "which is very conservative for these jobs]. Set to 0 to disable. Pair with " + "Toil's --defaultWalltime, which sets the walltime for every other job.") + def makeURL(path_or_url): if urlparse(path_or_url).scheme == '': return "file://" + os.path.abspath(path_or_url) @@ -945,7 +975,7 @@ class RoundedJob(Job): # Default rounding amount: 100 MiB roundingAmount = 100*1024*1024 def __init__(self, memory=None, cores=None, disk=None, preemptable=None, - unitName=None, checkpoint=False, accelerators=None): + unitName=None, checkpoint=False, accelerators=None, walltime=None): if memory is not None: memory = self.roundUp(memory) if disk is not None: @@ -955,7 +985,8 @@ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, disk = 1500*1024*1024 + self.roundUp(disk) super(RoundedJob, self).__init__(memory=memory, cores=cores, disk=disk, preemptable=preemptable, unitName=unitName, - checkpoint=checkpoint, accelerators=accelerators) + checkpoint=checkpoint, accelerators=accelerators, + walltime=walltime) def roundUp(self, bytesRequirement): """ @@ -1005,12 +1036,12 @@ class ChildTreeJob(RoundedJob): slightly, but reducing the wall-clock time taken dramatically. """ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, - unitName=None, checkpoint=False, maxChildrenPerJob=20): + unitName=None, checkpoint=False, maxChildrenPerJob=20, walltime=None): self.queuedChildJobs = [] self.maxChildrenPerJob = maxChildrenPerJob super(ChildTreeJob, self).__init__(memory=memory, cores=cores, disk=disk, preemptable=preemptable, unitName=unitName, - checkpoint=checkpoint) + checkpoint=checkpoint, walltime=walltime) def addChild(self, job): self.queuedChildJobs.append(job) diff --git a/src/cactus/update/cactus_hal2seqfile.py b/src/cactus/update/cactus_hal2seqfile.py index f443a4175..0eab5633f 100644 --- a/src/cactus/update/cactus_hal2seqfile.py +++ b/src/cactus/update/cactus_hal2seqfile.py @@ -30,9 +30,9 @@ from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import makeURL, cactus_call, cactus_clamp_memory from cactus.shared.version import cactus_commit @@ -117,6 +117,7 @@ def hal2fasta_gz(job, hal_id, hal_name, genome): def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help="input HAL alignment") parser.add_argument("outDir", help="output directory for the gzipped per-genome FASTAs (created if needed)") @@ -170,7 +171,7 @@ def main(): # defaults to symlink=True in Toil, but we set it explicitly so this holds within the tool hal_id = toil.importFile(makeURL(options.halFile), symlink=True) fa_ids = toil.start(Job.wrapJobFn(export_subtree_fastas, hal_id, - os.path.basename(options.halFile), genomes, lengths)) + os.path.basename(options.halFile), genomes, lengths, walltime=cactus_fast_walltime())) # export each gzipped fasta to /.fa.gz for genome, fa_id in fa_ids.items(): diff --git a/toil-requirement.txt b/toil-requirement.txt index 30248feae..f6fd9eecb 100644 --- a/toil-requirement.txt +++ b/toil-requirement.txt @@ -1,2 +1,4 @@ backports.zoneinfo[tzdata];python_version<"3.9" -toil[aws]==9.5.0 +# Pinned to Toil master for per-job walltime support (not yet in a release: 9.5.0 lacks it). +# TODO: revert to a released version (toil[aws]==) once walltime lands in a Toil release. +toil[aws] @ git+https://github.com/DataBiosphere/toil.git@091ed8861390c06230495819ba0fbef4cc4db5a0 From b209200908d1057c8fc53e555451c2081406e51f Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 10 Sep 2026 07:50:58 -0400 Subject: [PATCH 03/18] Pin Toil to a9681fe0 for --doubleTime Bumps the Toil master pin from 091ed886 to a9681fe0 ("Add `--doubleTime` argument", #5576). --doubleTime is the walltime analogue of --doubleMem: a job Slurm kills for exceeding its walltime is retried with twice the walltime. That is what makes tight per-job walltime estimates safe -- an occasional underestimate costs a retry, not the run -- so it is a prerequisite for setting real times on the rest of the jobs. Also documents it alongside --doubleMem in the cluster options. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- doc/progressive.md | 3 ++- toil-requirement.txt | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/progressive.md b/doc/progressive.md index d548e6ed5..2e462b127 100644 --- a/doc/progressive.md +++ b/doc/progressive.md @@ -422,7 +422,8 @@ On a cluster with partitions and/or time limits, make sure to use * `--defaultWalltime` to specify the time, **in seconds**, for each job, ex `--defaultWalltime 720000` (200 hours). Slurm uses each job's walltime to choose a partition, so give the big jobs plenty of time here. * `--fastWalltime` sets the walltime, **in seconds**, that Cactus gives its many small "coordination" jobs; it defaults to `1800` (30 minutes), which is very conservative for these jobs. On a cluster with a fast/short partition, this lets those little jobs get routed there instead of tying up a long partition; everything else falls back to `--defaultWalltime`. Pass `--fastWalltime 0` to disable it. * `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU. But this option will allow you to override that. -* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job `--defaultWalltime` / `--fastWalltime` values above (so it also disables the fast-partition routing). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. +* `--doubleTime true` (highly recommended): the time analogue of `--doubleMem`. If slurm kills a job for running past its walltime, retry it with double the walltime. This is what makes the per-job estimates above safe to keep tight: an occasional underestimate costs one retry rather than the run. (A job with no walltime at all -- `--defaultWalltime 0` -- has nothing to double, so this only helps once the times above are set.) +* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job `--defaultWalltime` / `--fastWalltime` values above (so it also disables the fast-partition routing, and `--doubleTime` along with it). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. You can also use diff --git a/toil-requirement.txt b/toil-requirement.txt index f6fd9eecb..e4deff228 100644 --- a/toil-requirement.txt +++ b/toil-requirement.txt @@ -1,4 +1,6 @@ backports.zoneinfo[tzdata];python_version<"3.9" -# Pinned to Toil master for per-job walltime support (not yet in a release: 9.5.0 lacks it). -# TODO: revert to a released version (toil[aws]==) once walltime lands in a Toil release. -toil[aws] @ git+https://github.com/DataBiosphere/toil.git@091ed8861390c06230495819ba0fbef4cc4db5a0 +# Pinned to Toil master for the per-job walltime requirement and --doubleTime, neither of which +# is in a release yet (9.5.0 lacks both). +# a9681fe0 = "Add `--doubleTime` argument (#5576)". +# TODO: revert to a released version (toil[aws]==) once these land in a Toil release. +toil[aws] @ git+https://github.com/DataBiosphere/toil.git@a9681fe077c15fff10f583478a8cb7ecf444c278 From cf25b3ba3ebb67bb776124cf7958bacc46c638cc Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 10 Sep 2026 08:36:17 -0400 Subject: [PATCH 04/18] Estimate a per-job walltime instead of giving every job the same one Slurm picks a job's partition from its time limit, so the standing advice of --slurmTime 100:00:00 drops all of Cactus's jobs -- including the many thousands that finish in seconds -- into the slowest queue. This estimates a walltime per job from its inputs, the way memory is already estimated, so the short jobs can be routed to a short partition. Machinery (src/cactus/shared/common.py): - cactus_walltime(seconds, io_bytes) replaces cactus_fast_walltime(). Call sites pass their best estimate of the job's real runtime; the safety margin (--walltimeFactor, default 2.5), the floor (--minWalltime, 600s) and the ceiling (--maxWalltime) are applied centrally. io_bytes covers jobstore staging, which is what makes an otherwise trivial job slow. Mirrors cactus_clamp_memory, including reaching workers through the environment. - --doubleTime now defaults to true, the way --retryCount is defaulted to 5. It is what makes tight estimates safe: a job Slurm kills for running over is retried with twice the time rather than failing the run. - cactus_slurm_max_walltime() caps every estimate at the longest partition the jobs could land on. Toil raises, rather than falling back to a default, when it cannot find a partition that fits a walltime, so an over-estimate would otherwise kill the run at submission. - --fastWalltime is gone; there is one estimator now rather than a separate knob for the small jobs. Estimates, each fitted to real cluster logs (1.8M timed commands from the VGP 577-way, 291k from four HPRC pangenome runs): - cactus_consolidated: 800 * disk_gb**0.95, scaled up below 24 cores. Fitted to 576 alignments; every one of them lands at 65-75% of its request. - lastz: a per-divergence-class table in the config (), scaled by the actual chunk sizes. 827k runs: the closest class is p99 331s against 4324s for the most diverged, which is exactly the spread that a single global time throws away. - Red: 933 s/Gb of fasta (625 runs, p99 2799 s/Gb, divided by the 3x speedup). - hal2chains batches: the existing chain_pair_cost, calibrated at 1.78e-5 s per unit. Per pair it is noisy, but a batch sums many pairs -- replaying the LPT assignment, predicted batch time is within 1.02-1.55x of the truth. - hal2maf batches: per chunk per genome, plus one slow chunk for the tail that dividing by the core count ignores. Covers 98% of the 824 VGP batches. - taffy index/coverage/view: 3000s + 200 s/GB of MAF, tapered below 4.5 GB. Cost per GB is 3x higher on a small MAF than a large one, so a slope-only model asks far too much of the big ones. Also fixes the first pass's tagging: jobs that export, merge or import a large file are not the coordination tier, whatever they compute. hal2maf_ranges and the hal2maf batches import the whole HAL; the merged-MAF, pangenome and panpatch exports write hundreds of GB; save_preprocessed_files copies every preprocessed sequence. These now carry io_bytes from a real size in scope at the call site. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- doc/pangenome.md | 2 +- doc/progressive.md | 20 ++- src/cactus/blast/cactus_blast.py | 4 +- src/cactus/cactus_progressive_config.xml | 18 ++ src/cactus/maf/cactus_hal2chains.py | 43 ++++- src/cactus/maf/cactus_hal2maf.py | 73 ++++++-- src/cactus/maf/cactus_maf2bigmaf.py | 4 +- src/cactus/maf/cactus_phast.py | 38 ++-- src/cactus/maf/maf_chunk.py | 27 +++ src/cactus/paf/local_alignment.py | 70 ++++++-- src/cactus/pipeline/cactus_workflow.py | 22 ++- .../preprocessor/cactus_preprocessor.py | 12 +- src/cactus/preprocessor/redMasking.py | 13 +- src/cactus/preprocessor/unmasking.py | 4 +- src/cactus/progressive/cactus_progressive.py | 30 ++-- src/cactus/refmap/cactus_graphmap.py | 12 +- src/cactus/refmap/cactus_graphmap_join.py | 14 +- src/cactus/refmap/cactus_graphmap_split.py | 15 +- src/cactus/refmap/cactus_minigraph.py | 16 +- src/cactus/refmap/cactus_pangenome.py | 61 ++++--- src/cactus/refmap/cactus_panpatch.py | 21 ++- src/cactus/refmap/cactus_refmap.py | 14 +- src/cactus/setup/cactus_align.py | 14 +- src/cactus/shared/common.py | 163 +++++++++++++++--- src/cactus/update/cactus_hal2seqfile.py | 4 +- 25 files changed, 528 insertions(+), 186 deletions(-) diff --git a/doc/pangenome.md b/doc/pangenome.md index 871f40255..e3080a517 100644 --- a/doc/pangenome.md +++ b/doc/pangenome.md @@ -554,7 +554,7 @@ One thing to watch out for: **error intervals at contig tips interact with telom ``` cactus-panpatch ./js chromfile.txt --outDir patched --batch --requireTelomeres \ - --batchSystem slurm --defaultWalltime 36000 --doubleMem true --maxMemory 1.5T --retryCount 10 \ + --batchSystem slurm --doubleMem true --maxMemory 1.5T --retryCount 10 \ --consCores 16 --mgCores 32 --mapCores 16 ``` diff --git a/doc/progressive.md b/doc/progressive.md index 2e462b127..3c1dbf23f 100644 --- a/doc/progressive.md +++ b/doc/progressive.md @@ -232,7 +232,7 @@ cactus-hal2maf ./js evolverMammals.hal evolverMammals.maf.gz --refGenome simHuma Exporting a MAF for each reference in an 8-way [ape alignment](https://cglgenomics.ucsc.edu/february-2024-t2t-apes/) on UCSC Slurm cluster: ``` -for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16 --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --defaultWalltime 720000 --slurmPartition long;done +for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16 --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --slurmPartition long;done ``` Note that this invocation creates two MAFs per reference (as dictated by the `--outType` option). Notably, the `.single.maf.gz` files will be filtered so that each genome appears at most once per block, which is often required by browsers and other MAF-reading tools. @@ -343,7 +343,7 @@ cactus-phast ./js-vgp \ --geneAnnotation https://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/ncbiRefSeq.txt.gz \ --bigwig --batchSystem slurm --chunkCores 32 --phyloFitCores 32 \ --substMod REV --modFreqs --precision HIGH \ - --slurmPartition medium --defaultWalltime 36000 \ + --slurmPartition medium \ --doubleMem true ``` @@ -419,11 +419,15 @@ These are the most relevant options for running on a cluster On a cluster with partitions and/or time limits, make sure to use -* `--defaultWalltime` to specify the time, **in seconds**, for each job, ex `--defaultWalltime 720000` (200 hours). Slurm uses each job's walltime to choose a partition, so give the big jobs plenty of time here. -* `--fastWalltime` sets the walltime, **in seconds**, that Cactus gives its many small "coordination" jobs; it defaults to `1800` (30 minutes), which is very conservative for these jobs. On a cluster with a fast/short partition, this lets those little jobs get routed there instead of tying up a long partition; everything else falls back to `--defaultWalltime`. Pass `--fastWalltime 0` to disable it. +Slurm chooses a job's partition from its time limit, so giving every job one big time (the old `--slurmTime 200:00:00` advice) puts even the thousands of jobs that finish in seconds into the slowest queue. **Cactus now estimates a walltime for each job itself**, from the size of that job's inputs, the same way it estimates memory. There is nothing you need to set for this: the defaults below are the knobs for when the estimates do not suit your cluster. + +* `--doubleTime true` is **on by default** (Cactus turns it on the way it raises `--retryCount` to 5). It is the time analogue of `--doubleMem`: if slurm kills a job for running past its walltime, it is retried with double the walltime. This is what makes tight per-job estimates safe -- an occasional underestimate costs one retry, not the run. Pass `--doubleTime false` to turn it off. +* `--walltimeFactor` is the safety multiplier applied to every estimate (default `2.5`). Raise it if your cluster is slower than the ones the estimates were measured on and jobs are being killed for running over. Pass `--walltimeFactor 0` to switch per-job walltimes off entirely, which puts every job back on `--defaultWalltime`. +* `--minWalltime` is the floor for any request, in seconds (default `600`). Cactus schedules a great many jobs that finish in well under a second; this keeps worker startup and jobstore I/O from pushing them over their limit. +* `--maxWalltime` is the ceiling, in seconds. By default, on Slurm, Cactus queries `sinfo` and caps every request at the longest time limit of any partition your jobs could land on. This matters: when Toil cannot find a partition that fits a job's walltime it raises rather than falling back to a default, which would kill the run at submission time. +* `--defaultWalltime` is Toil's fallback for jobs Cactus did not estimate a time for. With the estimates on there is very little left for it to cover, so you can usually leave it alone. * `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU. But this option will allow you to override that. -* `--doubleTime true` (highly recommended): the time analogue of `--doubleMem`. If slurm kills a job for running past its walltime, retry it with double the walltime. This is what makes the per-job estimates above safe to keep tight: an occasional underestimate costs one retry rather than the run. (A job with no walltime at all -- `--defaultWalltime 0` -- has nothing to double, so this only helps once the times above are set.) -* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job `--defaultWalltime` / `--fastWalltime` values above (so it also disables the fast-partition routing, and `--doubleTime` along with it). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. +* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job estimates above (so it also disables the partition routing they exist for, and `--doubleTime` along with it). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. You can also use @@ -442,13 +446,13 @@ source /private/groups/cgl/cactus/venv-cactus-latest/bin/activate Some recommended options: ``` -cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true --defaultWalltime 720000 +cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true ``` To run the same command step by step, ``` -cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --defaultWalltime 720000 --batchSystem slurm" --alignCores 64 --script > mammals.sh +cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --batchSystem slurm" --alignCores 64 --script > mammals.sh chmod+x mammals.sh ./mammals.sh ``` diff --git a/src/cactus/blast/cactus_blast.py b/src/cactus/blast/cactus_blast.py index d52f40302..0c1abfd13 100644 --- a/src/cactus/blast/cactus_blast.py +++ b/src/cactus/blast/cactus_blast.py @@ -12,7 +12,7 @@ import timeit from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -157,7 +157,7 @@ def runCactusBlastOnly(options): input_seq_id_map[genome] = toil.importFile(seq) paf_id = toil.start(Job.wrapJobFn(sanitize_then_make_paf_alignments, NXNewick().writeString(spanning_tree), - input_seq_id_map, options.root, config_node, options.outputFile, walltime=cactus_fast_walltime())) + input_seq_id_map, options.root, config_node, options.outputFile, walltime=cactus_walltime())) # export the alignments toil.exportFile(paf_id, makeURL(options.outputFile)) diff --git a/src/cactus/cactus_progressive_config.xml b/src/cactus/cactus_progressive_config.xml index 830220e39..441d7800f 100644 --- a/src/cactus/cactus_progressive_config.xml +++ b/src/cactus/cactus_progressive_config.xml @@ -135,6 +135,20 @@ five="--step=2 --ambiguous=iupac,100,100 --ydrop=4000 --hspthresh=2400 --gappedthresh=2600 --queryhspbest=100000" default="--step=1 --ambiguous=iupac,100,100 --ydrop=4000 --hspthresh=2200 --gappedthresh=2400 --queryhspbest=100000" /> + + diff --git a/src/cactus/maf/cactus_hal2chains.py b/src/cactus/maf/cactus_hal2chains.py index b0cbdce3d..692efee7a 100644 --- a/src/cactus/maf/cactus_hal2chains.py +++ b/src/cactus/maf/cactus_hal2chains.py @@ -14,7 +14,7 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -155,7 +155,7 @@ def main(): config.substituteAllPredefinedConstantsWithLiterals(options) hal_id = toil.importFile(options.halFile) - chains_id_dict = toil.start(Job.wrapJobFn(hal2chains_workflow, config, options, hal_id, walltime=cactus_fast_walltime())) + chains_id_dict = toil.start(Job.wrapJobFn(hal2chains_workflow, config, options, hal_id, walltime=cactus_walltime())) #export the chains for query_genome in chains_id_dict.keys(): @@ -184,8 +184,8 @@ def hal2chains_workflow(job, config, options, hal_id): disk=int(hal_id.size * 1.2)) leaf_genomes = get_genomes_job.rv(0) distance_matrix = get_genomes_job.rv(1) - chrom_info_job = get_genomes_job.addFollowOnJobFn(hal2chains_chrom_info_all, config, options, hal_id, leaf_genomes, walltime=cactus_fast_walltime()) - hal2chains_all_job = chrom_info_job.addFollowOnJobFn(hal2chains_all, config, options, hal_id, chrom_info_job.rv(), distance_matrix, walltime=cactus_fast_walltime()) + chrom_info_job = get_genomes_job.addFollowOnJobFn(hal2chains_chrom_info_all, config, options, hal_id, leaf_genomes, walltime=cactus_walltime()) + hal2chains_all_job = chrom_info_job.addFollowOnJobFn(hal2chains_all, config, options, hal_id, chrom_info_job.rv(), distance_matrix, walltime=cactus_walltime()) return hal2chains_all_job.rv() def hal2chains_check_tools(job, options): @@ -283,6 +283,22 @@ def chain_pair_cost(q, t, chrom_info_dict, distance_matrix, epsilon=0.05): d = distance_matrix[q][t] return min(q_size, t_size) / (d + epsilon) +# Seconds of chain pipeline per unit of chain_pair_cost. Calibrated against the 1719 +# halLiftover|pslPosTarget|axtChain runs of the VGP 577-way: total time / total cost. The +# proxy is noisy per pair (log-log r=0.56, and the worst single pair is 26x the p50 rate), but +# a batch sums many pairs and the noise averages out -- replaying that LPT assignment, the +# predicted batch time is within 1.02x of the truth at 3 batches and 1.55x at 64. +CHAIN_SECS_PER_COST = 1.783e-05 + + +def estimate_batch_walltime(options, batch_pairs, chrom_info_dict, distance_matrix): + """ estimated seconds for one hal2chains batch: the batch's total pair cost spread over + the GNU parallel slots it will actually use """ + total_cost = sum(chain_pair_cost(q, t, chrom_info_dict, distance_matrix) for q, t in batch_pairs) + slots = max(1, min(options.batchParallelHal2chains or 1, len(batch_pairs))) + return CHAIN_SECS_PER_COST * total_cost / slots + + def estimate_batch_memory(options, hal_id, pair_2bit_max=0): """ pick a memory request for a hal2chains batch job. @@ -344,10 +360,16 @@ def hal2chains_chrom_info_all(job, config, options, hal_id, genomes): continue # disk: hal copy + headroom for all the 2bits/beds we'll generate (2bits ~ fasta size ~ hal/ngenomes) batch_disk = int(hal_id.size * 1.2) + int(hal_id.size * 1.5 * len(batch_genomes) / max(1, len(all_genomes))) + # hal2fasta|faToTwoBit was p50 20s / p99 82s per genome across the 577-way, run + # batchParallelHal2chains at a time -- but on a HAL that size the job spends most of + # its life copying the HAL, which is what io_bytes accounts for + batch_slots = max(1, min(options.batchParallelHal2chains or 1, len(batch_genomes))) + chrom_info_secs = 100.0 * len(batch_genomes) / batch_slots batch_job = job.addChildJobFn(hal2chains_chrom_info_batch, config, options, hal_id, batch_genomes, disk=batch_disk, cores=options.batchCores, - memory=batch_memory) + memory=batch_memory, + walltime=cactus_walltime(chrom_info_secs, io_bytes=hal_id.size)) for g in batch_genomes: chrom_info_dict[g] = batch_job.rv(g) return chrom_info_dict @@ -460,7 +482,10 @@ def hal2chains_all(job, config, options, hal_id, chrom_info_dict, distance_matri batch_pairs, batch_chrom_info, batch_distances, disk=batch_disk, cores=options.batchCores, - memory=batch_memory) + memory=batch_memory, + walltime=cactus_walltime( + estimate_batch_walltime(options, batch_pairs, chrom_info_dict, distance_matrix), + io_bytes=hal_id.size + total_2bit)) for q, t in batch_pairs: if q not in output_dict: @@ -471,10 +496,14 @@ def hal2chains_all(job, config, options, hal_id, chrom_info_dict, distance_matri # disk: chain file + intermediates (~10x target 2bit is generous). # memory: hgLoadChain peak is ~2x uncompressed chain size ≈ a few GiB for big mammals. t_2bit_size = chrom_info_dict[t]['2bit'].size + # every command here was under two minutes across the 577-way's 1719 bigChains + # (bedToBigBed p99 63s, hgLoadChain p99 15s); the chain it reads is the only + # thing that grows, and it is small next to the 2bit bigchains_job = batch_job.addFollowOnJobFn(chain2bigchain, options, q, t, chrom_info_dict[t], batch_job.rv(q, t), disk=max(20 * t_2bit_size, 1024**3), - memory=cactus_clamp_memory(max(5 * t_2bit_size, 2 * 1024**3))) + memory=cactus_clamp_memory(max(5 * t_2bit_size, 2 * 1024**3)), + walltime=cactus_walltime(300, io_bytes=2 * t_2bit_size)) output_dict[q][t]['bigChain'] = bigchains_job.rv(0) output_dict[q][t]['bigLink'] = bigchains_job.rv(1) diff --git a/src/cactus/maf/cactus_hal2maf.py b/src/cactus/maf/cactus_hal2maf.py index 6c37e0d83..f9836b8a9 100644 --- a/src/cactus/maf/cactus_hal2maf.py +++ b/src/cactus/maf/cactus_hal2maf.py @@ -16,7 +16,7 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -30,6 +30,7 @@ from cactus.progressive.cactus_prepare import human2bytesN from cactus.progressive.multiCactusTree import MultiCactusTree from cactus.maf.maf_chunk import parse_bed_ranges +from cactus.maf.maf_chunk import taffy_walltime_secs, TAFFY_SIDE_OUTPUT_FRACTION from toil.job import Job from toil.common import Toil @@ -222,12 +223,37 @@ def main(): bed_id = toil.importFile(options.bedRanges) if options.bedRanges else None hal_id = toil.importFile(options.halFile) - toil.start(Job.wrapJobFn(hal2maf_workflow, hal_id, bed_id, options, config, walltime=cactus_fast_walltime())) + toil.start(Job.wrapJobFn(hal2maf_workflow, hal_id, bed_id, options, config, walltime=cactus_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time logger.info("cactus-hal2maf has finished after {} seconds".format(run_time)) +# Core-seconds of hal2maf-plus-taffy per chunk per genome in the alignment. Fitted to the 824 +# batch jobs of the VGP 577-way MAF export (31 references, 209k chunks, 22153 CPU-hours): the +# work in a chunk is set by how many genomes align there, not by how many bases the chunk spans +# -- a fit against chunk length comes out with a negative slope. +HAL2MAF_CORE_SECS_PER_CHUNK_PER_GENOME = 1.2 + +# A batch cannot finish faster than its slowest single chunk, and chunk cost varies enormously +# (per-chunk hal2maf was p50 71s against p99 1570s on that alignment). Dividing the total work +# by the core count assumes a balance that does not exist when a batch holds only a few chunks +# per core, which is where every badly-underestimated batch in the fit came from. This term is +# one slow chunk's worth, again per genome. +HAL2MAF_SLOW_CHUNK_CORE_SECS_PER_GENOME = 2.0 + + +def hal2maf_batch_walltime(batch_chunks, genome_list, options): + """ estimated seconds for one hal2maf batch: its total work spread over the cores it has, + plus the one slow chunk that sets the tail. Covers 98% of the 824 VGP batches once + cactus_walltime()'s factor is applied; the rest are what --doubleTime is for. """ + n_chunks = len(batch_chunks) + n_genomes = max(1, len(genome_list)) + slots = max(1, min(options.batchCores or 1, n_chunks)) + return (HAL2MAF_CORE_SECS_PER_CHUNK_PER_GENOME * n_genomes * n_chunks / slots + + HAL2MAF_SLOW_CHUNK_CORE_SECS_PER_GENOME * n_genomes) + + def export_file(job, file_id, out_path): """ run toil export in its own job in order to a) do it right away but b) in a separate job (in case it fails) """ job.fileStore.exportFile(file_id, makeURL(out_path)) @@ -235,11 +261,13 @@ def export_file(job, file_id, out_path): def hal2maf_workflow(job, hal_id, bed_id, options, config): - hal2maf_ranges_job = job.addChildJobFn(hal2maf_ranges, hal_id, bed_id, options, cores=1, disk=hal_id.size) + # halStats itself is seconds; on a HAL of any size this job is a HAL copy and little else + hal2maf_ranges_job = job.addChildJobFn(hal2maf_ranges, hal_id, bed_id, options, cores=1, disk=hal_id.size, + walltime=cactus_walltime(300, io_bytes=hal_id.size)) chunks, genome_list = hal2maf_ranges_job.rv(0), hal2maf_ranges_job.rv(1) - hal2maf_all_job = hal2maf_ranges_job.addFollowOnJobFn(hal2maf_all, hal_id, chunks, genome_list, options, config, walltime=cactus_fast_walltime()) + hal2maf_all_job = hal2maf_ranges_job.addFollowOnJobFn(hal2maf_all, hal_id, chunks, genome_list, options, config, walltime=cactus_walltime()) hal2maf_merge_job = hal2maf_all_job.addFollowOnJobFn(hal2maf_merge_all, hal2maf_all_job.rv(), options, genome_list, - disk=hal_id.size, walltime=cactus_fast_walltime()) + disk=hal_id.size, walltime=cactus_walltime()) hal2maf_ranges_job.addFollowOn(hal2maf_merge_job) # note: merge job also handles exporting (and some cleanup), indexing and coverage @@ -366,10 +394,14 @@ def hal2maf_all(job, hal_id, chunks, genome_list, options, config): cur_chunk = i * batch_size cur_batch_size = min(chunks_left, batch_size) if cur_batch_size: - batch_results.append(job.addChildJobFn(hal2maf_batch, hal_id, chunks[cur_chunk:cur_chunk+cur_batch_size], + batch_chunks = chunks[cur_chunk:cur_chunk+cur_batch_size] + batch_results.append(job.addChildJobFn(hal2maf_batch, hal_id, batch_chunks, genome_list, options, config, disk=math.ceil((1 + 1.5 / num_batches)*hal_id.size), cores=options.batchCores, - memory=batch_memory).rv()) + memory=batch_memory, + walltime=cactus_walltime( + hal2maf_batch_walltime(batch_chunks, genome_list, options), + io_bytes=hal_id.size)).rv()) chunks_left -= cur_batch_size assert chunks_left == 0 @@ -674,8 +706,12 @@ def hal2maf_merge_all(job, output_dicts, options, genome_list): for out_type in options.outType: maf_ids = [out_dict[out_type] for out_dict in output_dicts] maf_size = sum([maf_id.size for maf_id in maf_ids]) + # a cat of every chunk MAF, so it is pure I/O -- unless the output is TAF, which adds a + # taffy view pass over the merged file + merge_secs = taffy_walltime_secs(maf_size) if options.outputMAF.endswith(('.taf', '.taf.gz')) else 0 merge_job = job.addChildJobFn(hal2maf_merge, maf_ids, options, - disk=int(3 * maf_size)) + disk=int(3 * maf_size), + walltime=cactus_walltime(merge_secs, io_bytes=2 * maf_size)) # we export ASAP output_name, output_ext = os.path.splitext(options.outputMAF) if output_ext == '.gz': @@ -683,21 +719,30 @@ def hal2maf_merge_all(job, output_dicts, options, genome_list): output_ext += '.gz' if out_type != 'norm' and len(options.outType) > 1: output_name += '.{}'.format(out_type) - export_job = merge_job.addFollowOnJobFn(export_file, merge_job.rv(), output_name + output_ext, walltime=cactus_fast_walltime()) - merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=maf_ids, walltime=cactus_fast_walltime()) + # the merged MAF is a promise, so it has no size here; it is a concatenation of the + # chunk MAFs, so their total is the right stand-in (an upper bound once it is gzipped) + export_job = merge_job.addFollowOnJobFn(export_file, merge_job.rv(), output_name + output_ext, + walltime=cactus_walltime(0, io_bytes=2 * maf_size)) + merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=maf_ids, walltime=cactus_walltime()) if options.index: index_job = merge_job.addFollowOnJobFn(taffy_index, merge_job.rv(), output_name + output_ext, disk=int(1.1 * maf_size), - memory=cactus_clamp_memory(maf_size / 10)) - index_job.addFollowOnJobFn(export_file, index_job.rv(), output_name + output_ext + '.tai', walltime=cactus_fast_walltime()) + memory=cactus_clamp_memory(maf_size / 10), + walltime=cactus_walltime(taffy_walltime_secs(maf_size), + io_bytes=maf_size)) + index_job.addFollowOnJobFn(export_file, index_job.rv(), output_name + output_ext + '.tai', + walltime=cactus_walltime(0, io_bytes=2 * TAFFY_SIDE_OUTPUT_FRACTION * maf_size)) if options.coverage: coverage_job = merge_job.addFollowOnJobFn(taffy_coverage, merge_job.rv(), output_name + output_ext, genome_list, options, disk=int(1.1 * maf_size), - memory=cactus_clamp_memory(maf_size / 10)) - coverage_job.addFollowOnJobFn(export_file, coverage_job.rv(), output_name + output_ext + '.cov.tsv', walltime=cactus_fast_walltime()) + memory=cactus_clamp_memory(maf_size / 10), + walltime=cactus_walltime(taffy_walltime_secs(maf_size), + io_bytes=maf_size)) + coverage_job.addFollowOnJobFn(export_file, coverage_job.rv(), output_name + output_ext + '.cov.tsv', + walltime=cactus_walltime(0, io_bytes=2 * TAFFY_SIDE_OUTPUT_FRACTION * maf_size)) return export_job.rv() diff --git a/src/cactus/maf/cactus_maf2bigmaf.py b/src/cactus/maf/cactus_maf2bigmaf.py index ec5aeb710..21b1f6bd8 100644 --- a/src/cactus/maf/cactus_maf2bigmaf.py +++ b/src/cactus/maf/cactus_maf2bigmaf.py @@ -14,7 +14,7 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -94,7 +94,7 @@ def main(): if options.halFile: hal_id = toil.importFile(options.halFile) - bigmaf_id_dict = toil.start(Job.wrapJobFn(maf2bigmaf_workflow, config, options, maf_id, hal_id, walltime=cactus_fast_walltime())) + bigmaf_id_dict = toil.start(Job.wrapJobFn(maf2bigmaf_workflow, config, options, maf_id, hal_id, walltime=cactus_walltime())) #export the big maf out_bm_path = makeURL(options.outFile) diff --git a/src/cactus/maf/cactus_phast.py b/src/cactus/maf/cactus_phast.py index 99a891dad..79542495a 100644 --- a/src/cactus/maf/cactus_phast.py +++ b/src/cactus/maf/cactus_phast.py @@ -23,7 +23,7 @@ import timeit import xml.etree.ElementTree as ET -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -35,7 +35,8 @@ from cactus.shared.version import cactus_commit from cactus.progressive.cactus_prepare import human2bytesN -from cactus.maf.maf_chunk import (taffy_index_job, get_ref_sequence_lengths, +from cactus.maf.maf_chunk import (taffy_walltime_secs, TAFFY_SIDE_OUTPUT_FRACTION, + taffy_index_job, get_ref_sequence_lengths, get_aligned_ref_contigs, plan_chunks, plan_chunks_in_regions, parse_bed_ranges, filter_chunks_to_indexed, @@ -416,7 +417,7 @@ def main(): toil.start(Job.wrapJobFn(phast_workflow, config, options, maf_id, tai_id, tai_built, hal_id, ann_id, model_id, - bed_id, walltime=cactus_fast_walltime())) + bed_id, walltime=cactus_walltime())) end_time = timeit.default_timer() logger.info("cactus-phast finished in {} seconds".format(end_time - start_time)) @@ -815,20 +816,23 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # Convenience sed script: maps phast canonical names back to HAL genome # names. Only relevant when at least one HAL genome name contains '.'; # the job no-ops otherwise. - setup_job.addFollowOnJobFn(export_name_map_job, options, species_list, walltime=cactus_fast_walltime()) + setup_job.addFollowOnJobFn(export_name_map_job, options, species_list, walltime=cactus_walltime()) # build .tai if missing, else just chain through the existing one if tai_built: idx_job = setup_job.addFollowOnJobFn(taffy_index_job, maf_id, os.path.basename(options.inMaf), - disk=int(maf_id.size * 1.1)) + disk=int(maf_id.size * 1.1), + walltime=cactus_walltime(taffy_walltime_secs(maf_id.size), + io_bytes=maf_id.size)) tai_id = idx_job.rv() - idx_job.addFollowOnJobFn(export_file, tai_id, tai_export_path(options), walltime=cactus_fast_walltime()) + idx_job.addFollowOnJobFn(export_file, tai_id, tai_export_path(options), + walltime=cactus_walltime(0, io_bytes=2 * TAFFY_SIDE_OUTPUT_FRACTION * maf_id.size)) plan_parent = idx_job else: plan_parent = setup_job plan_job = plan_parent.addFollowOnJobFn(plan_chunks_job, options, ref_seq_lengths, tai_id, - bed_id, aligned_contigs, walltime=cactus_fast_walltime()) + bed_id, aligned_contigs, walltime=cactus_walltime()) chunk_specs = plan_job.rv() # The single multi-core chunker job: localizes the source MAF once, @@ -868,7 +872,7 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # ----- phyloFit branch (also runs in phyloP mode if no model was given) ----- if need_train: train_job = chunk_job.addFollowOnJobFn(train_workflow, options, chunks, - species_list, tree_str, ann_id, walltime=cactus_fast_walltime()) + species_list, tree_str, ann_id, walltime=cactus_walltime()) # train_workflow returns the model file id trained_model_id = train_job.rv() if options.mode == 'phyloFit': @@ -891,7 +895,7 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ for sub in track_inputs: score_job = score_parent.addFollowOnJobFn(phyloP_workflow, options, chunks, model_id, ref_seq_lengths, species_list, - sub, effective_root_name, walltime=cactus_fast_walltime()) + sub, effective_root_name, walltime=cactus_walltime()) track_rvs.append(score_job.rv()) return track_rvs @@ -1302,7 +1306,7 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): # projection); with --root, it's restricted to clade leaves. leaves_csv = ','.join(phast_name(g) for g in newick_leaves(tree_str)) extract_job = job.addChildJobFn(extract_4d_all, options, chunks, ann_id, - species_list, leaves_csv, walltime=cactus_fast_walltime()) + species_list, leaves_csv, walltime=cactus_walltime()) ss_results = extract_job.rv() # flat list of ss file ids (one per chunk-group) # 4d-site SS files are tiny even at 447-way (~300 MB aggregated). 4 GiB @@ -1311,7 +1315,7 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): species_list, tree_str, disk=4 * 1024**3) aggregate_id = aggregate_job.rv() - aggregate_job.addFollowOnJobFn(export_file, aggregate_id, ss_export_path(options), walltime=cactus_fast_walltime()) + aggregate_job.addFollowOnJobFn(export_file, aggregate_id, ss_export_path(options), walltime=cactus_walltime()) # phyloFit on 577-way × 1.4 GB SS peaked at ~300 MiB; default memory is # fine unless the user overrides via --phyloFitMemory. @@ -1330,7 +1334,7 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): else: export_parent = fit_job - export_parent.addFollowOnJobFn(export_file, model_id, model_export_path(options), walltime=cactus_fast_walltime()) + export_parent.addFollowOnJobFn(export_file, model_id, model_export_path(options), walltime=cactus_walltime()) return model_id @@ -1572,7 +1576,7 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis else: RealtimeLogger.info('phyloP track: global conservation (no --subtree)') score_job = job.addChildJobFn(phyloP_all, options, chunks, model_id, species_list, - track_subtree, walltime=cactus_fast_walltime()) + track_subtree, walltime=cactus_walltime()) per_chunk_wigs = score_job.rv() # flat list of (contig, start, wig_id_or_None) per chunk # disk estimates: per-base wig text is ~10 bytes/ref_bp uncompressed; bgzip @@ -1587,7 +1591,10 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis disk=merge_disk, cores=merge_cores) wig_id = merge_job.rv() - merge_job.addFollowOnJobFn(export_file, wig_id, wig_export_path(options, track_subtree), walltime=cactus_fast_walltime()) + # the merged wig is a promise, but its size follows the reference: ~10 bytes/bp of wig text, + # which bgzip takes down to roughly a byte a base (see the disk estimate above) + merge_job.addFollowOnJobFn(export_file, wig_id, wig_export_path(options, track_subtree), + walltime=cactus_walltime(0, io_bytes=2 * total_ref_bp)) if options.bigwig: # decompressed wig + .bw output + bbiFile scratch @@ -1603,7 +1610,8 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis memory=cactus_clamp_memory(bw_mem), cores=merge_cores) bw_job.addFollowOnJobFn(export_file, bw_job.rv(), - bigwig_export_path(options, track_subtree), walltime=cactus_fast_walltime()) + bigwig_export_path(options, track_subtree), + walltime=cactus_walltime(0, io_bytes=2 * total_ref_bp)) return wig_id diff --git a/src/cactus/maf/maf_chunk.py b/src/cactus/maf/maf_chunk.py index 301fcc85a..1366f0fe7 100644 --- a/src/cactus/maf/maf_chunk.py +++ b/src/cactus/maf/maf_chunk.py @@ -32,6 +32,33 @@ from cactus.shared.common import cactus_call +# One sequential taffy pass over a MAF, as seconds plus seconds per GB. Fitted to the 94 +# taffy index and taffy coverage jobs of the VGP 577-way MAF export, whose MAFs ran from 4.5 to +# 351 GB and whose runtimes ran from 16 minutes to 19 hours. The intercept is what makes this +# work: cost per GB is far higher on a small MAF (169 s/GB at the median against 608 at the +# worst), so a slope-only model tuned to cover the small ones asks three times too much of the +# big ones. These values leave every one of the 94 inside its request once cactus_walltime()'s +# factor is applied. taffy view gets the same treatment -- it is the same single pass. +TAFFY_SECS_BASE = 3000 +TAFFY_SECS_PER_GB = 200 + +# Smallest MAF in that fit. Below it the intercept is pure extrapolation and would hand an +# evolver-sized test MAF the better part of an hour, so it is tapered to zero. +TAFFY_RAMP_GB = 4.5 + + +def taffy_walltime_secs(maf_bytes): + """ estimated seconds for one sequential taffy pass (index, coverage, view) over a MAF """ + maf_gb = maf_bytes / 1e9 + ramp = min(1.0, maf_gb / TAFFY_RAMP_GB) if TAFFY_RAMP_GB > 0 else 1.0 + return TAFFY_SECS_BASE * ramp + TAFFY_SECS_PER_GB * maf_gb + +# The .tai index and .cov.tsv that taffy writes beside a MAF are small next to the MAF itself: +# the largest coverage table of that export was 560 MB against a 386 GB MAF. Jobs that only +# move one of those get a size in this proportion rather than the MAF's own. +TAFFY_SIDE_OUTPUT_FRACTION = 0.01 + + def taffy_index_job(job, maf_id, maf_basename): """ Build a .tai for the given source MAF. Single sequential pass. """ work_dir = job.fileStore.getLocalTempDir() diff --git a/src/cactus/paf/local_alignment.py b/src/cactus/paf/local_alignment.py index fe80b7a7b..1dcc5ca7e 100755 --- a/src/cactus/paf/local_alignment.py +++ b/src/cactus/paf/local_alignment.py @@ -20,12 +20,53 @@ import copy from Bio import SeqIO from cactus.paf.paf import get_event_pairs, get_leaves, get_node, get_distances -from cactus.shared.common import cactus_call, getOptionalAttrib, zip_gz, cactus_fast_walltime +from cactus.shared.common import cactus_call, getOptionalAttrib, zip_gz, cactus_walltime from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.preprocessor.unmasking import unmask_contigs_all from cactus.preprocessor.cactus_preprocessor import clean_if_different from cactus.shared.common import cactus_clamp_memory +def get_divergence_class(distance, params): + """ The bucket a pair at the given distance falls in ("one" through + "five", or "default" for anything more diverged). It selects the lastz parameters, and with + them most of the runtime, so the walltime estimate keys off the same bucket. """ + divergences = params.find("constants").find("divergences") + if getOptionalAttrib(divergences, 'useDefault', typeFn=bool, default=False): + return "default" + for i in "one", "two", "three", "four", "five": + if distance <= float(divergences.attrib[i]): + return i + return "default" + + +def get_lastz_walltime(distance, params, chunk_a, chunk_b): + """ Estimated seconds for one lastz/kegalign job on this pair of chunks. + + holds the per-divergence-class time for a pair of full-sized chunks; + we scale it by how big these two chunks actually are, so a small genome (whose whole + sequence is one short chunk) gets a short time rather than the 577-way figure. + + The table was measured on lastz, and is used for whichever mapper is configured: minimap2 + and FastGA are both faster, so it is conservative for them rather than wrong. """ + lastz_params_node = params.find("blast") + walltime_node = lastz_params_node.find("lastzWalltime") + if walltime_node is None: + return None + lastz_class = get_divergence_class(distance, params) + base = getOptionalAttrib(walltime_node, lastz_class, typeFn=float, default=None) + if base is None: + return None + gpu = getOptionalAttrib(lastz_params_node, 'gpu', typeFn=int, default=0) + fastga = getOptionalAttrib(lastz_params_node, 'mapper', typeFn=str) == 'fastga' + chunk_attr = 'bigChunkSize' if gpu or fastga else 'chunkSize' + chunk_size = getOptionalAttrib(lastz_params_node, chunk_attr, typeFn=float, default=None) + if chunk_size and chunk_size > 0: + # the table was measured on a pair of full chunks; lastz time is roughly linear in the + # sequence it is handed, so a pair that is half that size gets half the time + base *= (chunk_a.size + chunk_b.size) / (2.0 * chunk_size) + return base + + def run_lastz(job, name_A, genome_A, name_B, genome_B, distance, params): # Create a local temporary file to put the alignments in. work_dir = job.fileStore.getLocalTempDir() @@ -43,12 +84,8 @@ def run_lastz(job, name_A, genome_A, name_B, genome_B, distance, params): cpu = getOptionalAttrib(lastz_params_node, 'cpu', typeFn=int, default=None) lastz_divergence_node = lastz_params_node.find("kegalignArguments" if gpu else "lastzArguments") divergences = params.find("constants").find("divergences") - lastz_params = lastz_divergence_node.attrib["default"] + lastz_params = lastz_divergence_node.attrib[get_divergence_class(distance, params)] if not getOptionalAttrib(divergences, 'useDefault', typeFn=bool, default=False): - for i in "one", "two", "three", "four", "five": - if distance <= float(divergences.attrib[i]): - lastz_params = lastz_divergence_node.attrib[i] - break logger.info("For distance {} for genomes {}, {} using {} lastz parameters".format(distance, genome_A, genome_B, lastz_params)) if gpu: @@ -432,7 +469,8 @@ def make_chunks(genome): cores=lastz_cores, disk=max(4*(chunk_a.size+chunk_b.size), memory), memory=cactus_clamp_memory(memory), - accelerators=accelerators).rv()) + accelerators=accelerators, + walltime=cactus_walltime(get_lastz_walltime(distance, params, chunk_a, chunk_b))).rv()) dechunk_batch_size = getOptionalAttrib(lastz_params_node, 'dechunkBatchSize', typeFn=int, default=1e9) return job.addFollowOnJobFn(combine_chunks, chunked_alignment_files, dechunk_batch_size).rv() # Combine the chunked alignment files @@ -451,7 +489,7 @@ def invert_alignments(job, alignment_file): def make_ingroup_to_outgroup_alignments_0(job, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # Generate the alignments fle alignment_file = job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, - event_names_to_sequences, distances, params, walltime=cactus_fast_walltime()).rv() + event_names_to_sequences, distances, params, walltime=cactus_walltime()).rv() # Invert the final alignment so that the query is the outgroup and the target is the ingroup. # paffy invert holds the alignment and its inverse on local disk at once. The alignment's size @@ -524,7 +562,7 @@ def make_ingroup_to_outgroup_alignments_2(job, alignments, ingroup_event, outgro # recursively make alignments with the remaining outgroups alignments2 = root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, - event_names_to_sequences, distances, params, walltime=cactus_fast_walltime()).rv() + event_names_to_sequences, distances, params, walltime=cactus_walltime()).rv() return root_job.addFollowOnJobFn(make_ingroup_to_outgroup_alignments_3, ingroup_event, event_names_to_sequences[ingroup_event.iD], alignments, alignments2).rv() @@ -686,7 +724,7 @@ def chain_alignments(job, alignment_files, alignment_names, reference_event_name ) return job.addFollowOnJobFn(merge_processed_alignments, processed_rvs, disk=2 * merged_size, - walltime=cactus_fast_walltime()).rv() + walltime=cactus_walltime(0, io_bytes=2 * merged_size)).rv() def chain_tile_trim_filter_one_contig(job, split_file_id, reference_event_name, params): @@ -770,9 +808,9 @@ def merge_processed_alignments(job, processed_file_ids): def sanitize_then_make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancestor_event_string, params, output_path=None): - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, event_names_to_sequences, walltime=cactus_fast_walltime()) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, event_names_to_sequences, walltime=cactus_walltime()) paf_job = sanitize_job.addFollowOnJobFn(make_paf_alignments, event_tree_string, sanitize_job.rv(), - ancestor_event_string, params, walltime=cactus_fast_walltime()) + ancestor_event_string, params, walltime=cactus_walltime()) # gzip the output if requested if output_path and output_path.endswith('.gz'): gzip_job = paf_job.addFollowOnJobFn(zip_gz, output_path, paf_job.rv()) @@ -820,7 +858,7 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest if getOptionalAttrib(lastz_params_node.find("unmask"), 'action', typeFn=str, default='none') != 'none': ingroups = [ingroup.iD for ingroup in ingroup_events] # Pass a copy of event_names_to_sequences to unmask_job to avoid circular reference - unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params, walltime=cactus_fast_walltime()) + unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params, walltime=cactus_walltime()) for i,ingroup in enumerate(ingroups): event_names_to_sequences[ingroup] = unmask_job.rv(i) new_root_job = Job() @@ -855,7 +893,7 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest # for each ingroup make alignments to the outgroups if int(params.find("blast").attrib["trimIngroups"]): # Trim the ingroup sequences outgroup_alignments = [root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_0, ingroup, outgroup_events, - dict(event_names_to_sequences), distances, params, walltime=cactus_fast_walltime()).rv() + dict(event_names_to_sequences), distances, params, walltime=cactus_walltime()).rv() for ingroup in ingroup_events] if len(outgroup_events) > 0 else [] else: outgroup_alignments = [root_job.addChildJobFn(make_chunked_alignments, @@ -878,11 +916,11 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest ingroup_alignments, ingroup_alignment_names, outgroup_alignments, outgroup_alignment_names, ancestor_event_string, params, - total_sequence_size=total_sequence_size, walltime=cactus_fast_walltime()).rv() + total_sequence_size=total_sequence_size, walltime=cactus_walltime()).rv() # Delete the unmasked fastas (todo: should we do the unmasking somewhere further upstream?) for ingroup in ingroup_events: - root_job.addFollowOnJobFn(clean_if_different, event_names_to_sequences[ingroup.iD], input_sequence_map[ingroup.iD], walltime=cactus_fast_walltime()) + root_job.addFollowOnJobFn(clean_if_different, event_names_to_sequences[ingroup.iD], input_sequence_map[ingroup.iD], walltime=cactus_walltime()) return root_job.addFollowOnJobFn(chain_alignments, ingroup_alignments + outgroup_alignments, ingroup_alignment_names + outgroup_alignment_names, diff --git a/src/cactus/pipeline/cactus_workflow.py b/src/cactus/pipeline/cactus_workflow.py index 3e4827c22..63f7e4e05 100644 --- a/src/cactus/pipeline/cactus_workflow.py +++ b/src/cactus/pipeline/cactus_workflow.py @@ -20,6 +20,7 @@ from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import findRequiredNode, getOptionalAttrib from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime ############################################################ ############################################################ @@ -154,9 +155,28 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma name, bytes2human(mem), bytes2human(max_system_memory))) mem = max_system_memory + # Runtime, unlike peak memory, tracks input volume closely: fitted to the same 576 VGP + # alignments, secs = 800 * (disk/1e9)**0.95 lands within 65-75% of the walltime every one + # of them actually needed, once cactus_walltime()'s safety factor is applied on top and the + # 2x cactus_consolidated speedup since those logs is taken out. `disk` is used as the size + # term because it already combines the sequence and paf sizes in the proportions that drive + # the work (5:2), so there is one number to key off rather than two. + wt_coef = getOptionalAttrib(cons_node, 'walltime_coefficient_secs', typeFn=float, default=800.0) + wt_exp = getOptionalAttrib(cons_node, 'walltime_input_exponent', typeFn=float, default=0.95) + walltime_secs = wt_coef * ((disk / 1e9) ** wt_exp) if disk > 0 else 0 + # bar is 63.5% of consolidated's time across those 576 alignments, caf 20.3%, reference + # 15.4%; the parallel part of that stops improving somewhere around 24 cores (which is why + # --consCores above ~24 buys memory, not speed), so scale up only when a job is given fewer + # than that. The fit's own jobs all ran at 64 cores, i.e. already on the plateau. + wt_core_baseline = getOptionalAttrib(cons_node, 'walltime_core_scale_baseline', typeFn=int, default=24) + wt_parallel = getOptionalAttrib(cons_node, 'walltime_parallel_fraction', typeFn=float, default=0.64) + if cons_cores and 0 < cons_cores < wt_core_baseline: + walltime_secs *= (1.0 - wt_parallel) + wt_parallel * (float(wt_core_baseline) / cons_cores) + cons_job = job.addChildJobFn(cactus_cons, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, intermediate_results_url=intermediate_results_url, chrom_name=chrom_name, cores = cons_cores, - memory=cactus_clamp_memory(mem), disk=disk, retain_pages=retain_pages) + memory=cactus_clamp_memory(mem), disk=disk, retain_pages=retain_pages, + walltime=cactus_walltime(walltime_secs)) return cons_job.rv() def cactus_cons(job, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, diff --git a/src/cactus/preprocessor/cactus_preprocessor.py b/src/cactus/preprocessor/cactus_preprocessor.py index 336d9291b..afafb37cc 100644 --- a/src/cactus/preprocessor/cactus_preprocessor.py +++ b/src/cactus/preprocessor/cactus_preprocessor.py @@ -19,7 +19,7 @@ from sonLib.bioio import getTempDirectory from toil.common import Toil from toil.job import Job -from cactus.shared.common import cactus_call, cactus_fast_walltime +from cactus.shared.common import cactus_call, cactus_walltime from cactus.shared.common import RoundedJob from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import runGetChunks @@ -325,7 +325,7 @@ def run(self, fileStore): prepOptions.dnabrnnAction, disk=3*inSize if inSize else None) checkJob.addFollowOnJobFn(clean_if_different, self.inSequenceID, outSeqID, - walltime=cactus_fast_walltime()) + walltime=cactus_walltime()) else: logger.info("Skipping inactive preprocessor {}".format(prepNode.attrib["preprocessJob"])) outSeqID = self.inSequenceID @@ -462,7 +462,7 @@ def stageWorkflow(outputSequenceDir, configNode, inputSequences, toil, restart=F inputSequenceIDs.append(toil.importFile(makeURL(seq))) maskFileID = toil.importFile(makeURL(maskFile)) if maskFile else None unzip_job = Job.wrapJobFn(unzip_then_pp, configNode, inputSequences, inputSequenceIDs, inputEventNames, - maskFile, maskFileID, maskAction, minLength, walltime=cactus_fast_walltime()) + maskFile, maskFileID, maskAction, minLength, walltime=cactus_walltime()) outputSequenceIDs = toil.start(unzip_job) else: outputSequenceIDs = toil.restart() @@ -478,13 +478,13 @@ def stageWorkflow(outputSequenceDir, configNode, inputSequences, toil, restart=F def unzip_then_pp(job, config_node, input_fa_paths, input_fa_ids, input_event_names, mask_file_path, mask_file_id, mask_file_action, min_length): """ unzip then preprocess """ - unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids, walltime=cactus_fast_walltime()) + unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids, walltime=cactus_walltime()) if mask_file_id is not None: - mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id], walltime=cactus_fast_walltime()) + mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id], walltime=cactus_walltime()) config_node = mask_unzip_job.addFollowOnJobFn(maskJobOverride, config_node, mask_file_path, mask_unzip_job.rv(0), mask_file_action, min_length, disk=mask_file_id.size*20).rv() pp_job = unzip_job.addFollowOn(CactusPreprocessor([unzip_job.rv(i) for i in range(len(input_fa_ids))], config_node, eventNames=input_event_names)) - zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0], walltime=cactus_fast_walltime()) + zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0], walltime=cactus_walltime()) return zip_job.rv() def runCactusPreprocessor(outputSequenceDir, configFile, inputSequences, toilDir): diff --git a/src/cactus/preprocessor/redMasking.py b/src/cactus/preprocessor/redMasking.py index 8942cb52c..2aa5209c7 100644 --- a/src/cactus/preprocessor/redMasking.py +++ b/src/cactus/preprocessor/redMasking.py @@ -19,6 +19,7 @@ from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime from cactus.preprocessor.checkPreprocessedSequence import check_sequence_preserved from cactus.preprocessor.maskingCommon import prefilter_cmd, masked_base_count from cactus.preprocessor.maskingCommon import extract_masking_bed, soft_mask_intervals @@ -65,6 +66,14 @@ def red_memory_estimate(fasta_size, longest_record_bytes): return int(1.25 * (table_bytes + 8 * longest_record_bytes)) +# Seconds of Red per GB of input fasta. Across the 625 Red runs of the VGP 577-way (0.13 to +# 10 Gb of genome) the p99 was 2799 s/Gb and the worst 5147 s/Gb; Red is about 3x faster than +# it was for those, hence the /3. Everything else this job runs -- the prefilter, the +# softmask/hardmask conversions, extracting and applying the intervals -- came to well under +# 100 s each even on the largest genome, and is covered by cactus_walltime()'s safety factor. +RED_SECS_PER_GB = 2799 / 3.0 + + class RedMaskJob(RoundedJob): def __init__(self, fastaID, redOpts, redPrefilterOpts, eventName=None, unmask=False, longestRecordSize=None): @@ -74,7 +83,9 @@ def __init__(self, fastaID, redOpts, redPrefilterOpts, eventName=None, unmask=Fa longestRecordSize = fastaID.size memory = cactus_clamp_memory(red_memory_estimate(fastaID.size, longestRecordSize)) disk = 5*(fastaID.size) - RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True, + walltime=cactus_walltime(RED_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=2 * fastaID.size)) self.fastaID = fastaID self.redOpts = redOpts self.redPrefilterOpts = redPrefilterOpts diff --git a/src/cactus/preprocessor/unmasking.py b/src/cactus/preprocessor/unmasking.py index efdab86b7..78e19c761 100644 --- a/src/cactus/preprocessor/unmasking.py +++ b/src/cactus/preprocessor/unmasking.py @@ -5,7 +5,7 @@ from Bio.SeqRecord import SeqRecord import os from toil.job import Job -from cactus.shared.common import cactus_clamp_memory, catFiles, clean_jobstore_files, cactus_fast_walltime +from cactus.shared.common import cactus_clamp_memory, catFiles, clean_jobstore_files, cactus_walltime from cactus.shared.common import cactus_call, getOptionalAttrib from toil.realtimeLogger import RealtimeLogger from cactus.preprocessor.cactus_preprocessor import CactusPreprocessor @@ -98,7 +98,7 @@ def unmask_contigs_one(job, event, fasta_id, params): disk=fasta_id.size * 4) fasta_id = fa_merge_job.rv() # delete the pp_id - fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id], walltime=cactus_fast_walltime()) + fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id], walltime=cactus_walltime()) else: RealtimeLogger.warning('Remasking activated but no maskers active in preprocessor: doing nothing') diff --git a/src/cactus/progressive/cactus_progressive.py b/src/cactus/progressive/cactus_progressive.py index 1b6d91387..dd489534b 100755 --- a/src/cactus/progressive/cactus_progressive.py +++ b/src/cactus/progressive/cactus_progressive.py @@ -18,7 +18,7 @@ from base64 import b64encode from toil.lib.bioio import getTempFile -from cactus.shared.common import cactus_cpu_count, cactus_fast_walltime +from cactus.shared.common import cactus_cpu_count, cactus_walltime from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options from toil.realtimeLogger import RealtimeLogger @@ -67,8 +67,12 @@ def preprocess_all(job, options, config_node, input_seq_id_map): for i, event in enumerate(events): pp_seq_ids[event] = preprocessor_job.rv(i) - # do the logging and checkpointing - root_job.addFollowOnJobFn(save_preprocessed_files, options, config_node, pp_seq_ids, walltime=cactus_fast_walltime()) + # do the logging and checkpointing. with --intermediateResultsUrl this copies every + # preprocessed sequence out of the jobstore, so it is an I/O job, not a coordination one; + # masking does not change a sequence's length, so the inputs are the right size for it + input_seq_bytes = sum(seq_id.size for seq_id in input_seq_id_map.values()) + root_job.addFollowOnJobFn(save_preprocessed_files, options, config_node, pp_seq_ids, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) return pp_seq_ids @@ -137,7 +141,7 @@ def progressive_schedule(job, options, config_node, seq_id_map, tree, og_map, ro # to be consistent with pre-refactor (and work with updating tests), we include the root when its id is input if event in seq_id_map and seq_id_map[event]: event_id_map[event] = seq_id_map[event] - event_job = Job.wrapJobFn(progressive_step, options, config_node, event_id_map, tree, og_map, event, walltime=cactus_fast_walltime()) + event_job = Job.wrapJobFn(progressive_step, options, config_node, event_id_map, tree, og_map, event, walltime=cactus_walltime()) job_table[event] = event_job for dep in dep_table[event]: if dep in job_table: @@ -174,7 +178,7 @@ def progressive_step(job, options, config_node, seq_id_map, tree, og_map, event) # do the blast paf_job = job.addChildJobFn(make_paf_alignments, NXNewick().writeString(spanning_tree), - subtree_eventmap, event, config_node, walltime=cactus_fast_walltime()).encapsulate() + subtree_eventmap, event, config_node, walltime=cactus_walltime()).encapsulate() outgroups = og_map[event] if event in og_map else [] # trim the outgroups @@ -182,14 +186,14 @@ def progressive_step(job, options, config_node, seq_id_map, tree, og_map, event) trim_sequences = paf_job.addChildJobFn(trim_unaligned_sequences, [subtree_eventmap[i] for i in outgroups], paf_job.rv(), config_node) cons_job = paf_job.addFollowOnJobFn(progressive_step_2, trim_sequences.rv(), options, config_node, subtree_eventmap, - spanning_tree, og_map, event, walltime=cactus_fast_walltime()) + spanning_tree, og_map, event, walltime=cactus_walltime()) else: # Without outgroup trimming (or if there are no outgroups to trim) cons_job = paf_job.addChildJobFn(cactus_cons_with_resources, spanning_tree, event, config_node, subtree_eventmap, og_map, paf_job.rv(), cons_cores=options.consCores, cons_memory=options.consMemory, cons_retain_pages=getattr(options, 'consRetainPages', None), - intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_fast_walltime()) + intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_walltime()) # erase the paf since its now longer needed - cons_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[paf_job.rv()], walltime=cactus_fast_walltime()) + cons_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[paf_job.rv()], walltime=cactus_walltime()) return cons_job.rv() @@ -203,7 +207,7 @@ def progressive_step_2(job, trimmed_outgroups_and_alignments, options, config_no # now do consolidated return job.addChildJobFn(cactus_cons_with_resources, spanning_tree, event, config_node, subtree_eventmap, og_map, pafs, cons_cores=options.consCores, cons_memory=options.consMemory, cons_retain_pages=getattr(options, 'consRetainPages', None), - intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_fast_walltime()).rv() + intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_walltime()).rv() def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=None, cacheBytes=None, @@ -300,11 +304,11 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i ''' run the entire progressive workflow ''' # run the usual unzip / rename, even before preprocessing - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, walltime=cactus_fast_walltime()) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, walltime=cactus_walltime()) # start with the preprocessor if not options.skipPreprocessor: - pp_job = sanitize_job.addFollowOnJobFn(preprocess_all, options, config_node, sanitize_job.rv(), walltime=cactus_fast_walltime()) + pp_job = sanitize_job.addFollowOnJobFn(preprocess_all, options, config_node, sanitize_job.rv(), walltime=cactus_walltime()) seq_id_map = pp_job.rv() sanitize_job = pp_job else: @@ -322,7 +326,7 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i scaled_tree = get_ancestor_scaled_tree(mc_tree, root_event, max_div, branch_scale=options.branchScale, upweight_ancestors=upweight_ancestors) - progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event, walltime=cactus_fast_walltime()) + progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event, walltime=cactus_walltime()) # then do the hal export hal_export_job = progressive_job.addFollowOnJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, @@ -497,7 +501,7 @@ def main(): loadDnaBrnnModel(toil, config_node) # run the whole workflow - hal_id = toil.start(Job.wrapJobFn(progressive_workflow, options, config_node, mc_tree, og_map, input_seq_id_map, walltime=cactus_fast_walltime())) + hal_id = toil.start(Job.wrapJobFn(progressive_workflow, options, config_node, mc_tree, og_map, input_seq_id_map, walltime=cactus_walltime())) toil.exportFile(hal_id, makeURL(options.outputHal)) diff --git a/src/cactus/refmap/cactus_graphmap.py b/src/cactus/refmap/cactus_graphmap.py index 2579cbc59..1fba7cecd 100644 --- a/src/cactus/refmap/cactus_graphmap.py +++ b/src/cactus/refmap/cactus_graphmap.py @@ -17,7 +17,7 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -233,7 +233,7 @@ def graph_map(options): # run the workflow # output_dict is chrom -> paf_id, gfa_fa_id, gaf_id, unfiltered_paf_id, paf_filter_log, paf_was_filtered - output_dict = toil.start(Job.wrapJobFn(minigraph_batch_separate_workflow, options, config_wrapper, input_dict, graph_event, True, walltime=cactus_fast_walltime())) + output_dict = toil.start(Job.wrapJobFn(minigraph_batch_separate_workflow, options, config_wrapper, input_dict, graph_event, True, walltime=cactus_walltime())) export_graphmap_output(options, config_node, input_map, output_dict, toil) @@ -309,7 +309,7 @@ def minigraph_batch_workflow(job, options, config, input_dict, graph_event, sani else: chrom_options = options mgwf_job = job.addChildJobFn(minigraph_workflow, chrom_options, config, seq_id_map, gfa_id, graph_event, - sanitize, ref_collapse_paf_id, pansn_gfa_input, walltime=cactus_fast_walltime()) + sanitize, ref_collapse_paf_id, pansn_gfa_input, walltime=cactus_walltime()) output_dict[chrom] = mgwf_job.rv() return output_dict @@ -348,7 +348,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa # enforce unique prefixes and unzip fastas if sanitize: - sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) + sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() # add unique prefixes to the input PAF @@ -381,7 +381,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa gfa_id = gfa_unzip_job.rv() gfa_id_size *= 10 options.minigraphGFA = options.minigraphGFA[:-3] - paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event, walltime=cactus_fast_walltime()) + paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event, walltime=cactus_walltime()) root_job.addFollowOn(paf_job) collapse_paf_id = ref_collapse_paf_id @@ -397,7 +397,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa paf_job.addFollowOn(gfa2paf_job) collapse_mode = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "collapse", typeFn=str, default="none") if collapse_mode in ['reference', 'all', 'nonref']: - collapse_job = paf_job.addChildJobFn(self_align_all, config, seq_id_map, options.reference, collapse_mode, walltime=cactus_fast_walltime()) + collapse_job = paf_job.addChildJobFn(self_align_all, config, seq_id_map, options.reference, collapse_mode, walltime=cactus_walltime()) if ref_collapse_paf_id: collapse_paf_id = collapse_job.addFollowOnJobFn(merge_pafs, {"1":collapse_job.rv(), "2":ref_collapse_paf_id}, diff --git a/src/cactus/refmap/cactus_graphmap_join.py b/src/cactus/refmap/cactus_graphmap_join.py index 0010a3cae..598c9ef28 100644 --- a/src/cactus/refmap/cactus_graphmap_join.py +++ b/src/cactus/refmap/cactus_graphmap_join.py @@ -35,7 +35,7 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.multiCactusTree import MultiCactusTree -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -677,7 +677,7 @@ def graphmap_join(options): vg_ids, hal_ids, sv_gfa_ids, bypass_full_ids, bypass_clip_ids, bypass_filter_ids, contig_sizes_id=contig_sizes_id, - walltime=cactus_fast_walltime())) + walltime=cactus_walltime())) else: # load up the vgs (from their real paths: options.vg has had the .raw tag stripped) vg_ids = [] @@ -688,7 +688,7 @@ def graphmap_join(options): wf_output = toil.start(Job.wrapJobFn(graphmap_join_workflow, options, config, vg_ids, hal_ids, sv_gfa_ids, contig_sizes_id=contig_sizes_id, - walltime=cactus_fast_walltime())) + walltime=cactus_walltime())) #export the split data @@ -854,7 +854,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, hal_id_dict = hal_merge_job.rv() out_dicts.append(hal_id_dict) # delete the chromosome hals - hal_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_fast_walltime()) + hal_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_walltime()) # optional minigraph gfa merge if sv_gfa_ids: @@ -863,7 +863,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, sv_gfa_id_dict = sv_gfa_merge_job.rv() out_dicts.append(sv_gfa_id_dict) # delete the chromosome gfas - sv_gfa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=sv_gfa_ids, walltime=cactus_fast_walltime()) + sv_gfa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=sv_gfa_ids, walltime=cactus_walltime()) if options.indexMemory: index_mem = options.indexMemory @@ -978,7 +978,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vcf_ref in options.vcfReference: vcf_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, workflow_phase, index_mem, vcf_ref, phase_vg_ids, - ref_fasta_job.rv() if ref_fasta_job else None, walltime=cactus_fast_walltime()) + ref_fasta_job.rv() if ref_fasta_job else None, walltime=cactus_walltime()) if ref_fasta_job: ref_fasta_job.addFollowOn(vcf_job) out_dicts.append(vcf_job.rv()) @@ -2547,7 +2547,7 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, if vcf_ref and options.vcf: vcf_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, tag.rstrip('.'), index_mem, vcf_ref, phase_vg_ids, - ref_fasta_dict, vcftag=vcftag, walltime=cactus_fast_walltime()) + ref_fasta_dict, vcftag=vcftag, walltime=cactus_walltime()) out_dicts.append(vcf_job.rv()) if decon_L is not None: diff --git a/src/cactus/refmap/cactus_graphmap_split.py b/src/cactus/refmap/cactus_graphmap_split.py index 09fed4fc1..61a4238d5 100644 --- a/src/cactus/refmap/cactus_graphmap_split.py +++ b/src/cactus/refmap/cactus_graphmap_split.py @@ -14,7 +14,7 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.multiCactusTree import MultiCactusTree -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -176,7 +176,7 @@ def cactus_graphmap_split(options): # run the workflow wf_output = toil.start(Job.wrapJobFn(graphmap_split_workflow, options, config, input_seq_id_map, input_name_map, gfa_id, options.minigraphGFA, - paf_id, options.graphmapPAF, walltime=cactus_fast_walltime())) + paf_id, options.graphmapPAF, walltime=cactus_walltime())) #export the split data export_split_data(toil, wf_output[0], wf_output[1], wf_output[2], wf_output[3], options.outDir, config) @@ -204,7 +204,7 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ # fix up the headers if sanitize: - sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) + sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() else: sanitize_job = Job() @@ -248,7 +248,7 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ mask_bed_id = None if options.maskFilter: - mask_bed_id = sanitize_job.addFollowOnJobFn(get_mask_bed, seq_id_map, options.maskFilter, walltime=cactus_fast_walltime()).rv() + mask_bed_id = sanitize_job.addFollowOnJobFn(get_mask_bed, seq_id_map, options.maskFilter, walltime=cactus_walltime()).rv() # use rgfa-split to split the gfa and paf up by contig split_gfa_job = root_job.addFollowOnJobFn(split_gfa, config, gfa_id, [paf_id], ref_contigs, @@ -257,14 +257,15 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ memory=cactus_clamp_memory((gfa_size + paf_size) * 3)) # use the output of the above splitting to do the fasta splitting - split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0), walltime=cactus_fast_walltime()) + split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0), walltime=cactus_walltime()) # gather everythign up into a table - gather_fas_job = split_fas_job.addFollowOnJobFn(gather_fas, split_gfa_job.rv(0), split_fas_job.rv(0), split_fas_job.rv(1), walltime=cactus_fast_walltime()) + gather_fas_job = split_fas_job.addFollowOnJobFn(gather_fas, split_gfa_job.rv(0), split_fas_job.rv(0), split_fas_job.rv(1), walltime=cactus_walltime()) # lump "other" contigs together into one file (to make fewer align jobs downstream) bin_other_job = gather_fas_job.addFollowOnJobFn(bin_other_contigs, config, ref_contigs, options.otherContig, gather_fas_job.rv(0), - disk=(gfa_size + paf_size) * 2, walltime=cactus_fast_walltime()) + disk=(gfa_size + paf_size) * 2, + walltime=cactus_walltime(0, io_bytes=(gfa_size + paf_size) * 2)) # return all the files, as well as the 2 split logs return (seq_name_map, bin_other_job.rv(), split_gfa_job.rv(1), gather_fas_job.rv(1)) diff --git a/src/cactus/refmap/cactus_minigraph.py b/src/cactus/refmap/cactus_minigraph.py index 415a355d1..ff7aed0cf 100644 --- a/src/cactus/refmap/cactus_minigraph.py +++ b/src/cactus/refmap/cactus_minigraph.py @@ -17,7 +17,7 @@ import gzip from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.refmap.pangenome_exclusions import event_to_pansn_prefix from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper @@ -126,7 +126,7 @@ def main(): input_dict = minigraph_construct_import_sequences(options, config_wrapper, input_seqfiles, toil) # output_dict: chrom-> (gfa_id, pansn_gfa_id, train_id) - output_dict = toil.start(Job.wrapJobFn(minigraph_construct_batch_workflow, options, config_node, input_dict, options.outputGFA, walltime=cactus_fast_walltime())) + output_dict = toil.start(Job.wrapJobFn(minigraph_construct_batch_workflow, options, config_node, input_dict, options.outputGFA, walltime=cactus_walltime())) export_minigraph_construct_output(options, input_seqfiles, output_dict, toil) @@ -265,7 +265,7 @@ def minigraph_construct_batch_workflow(job, options, config_node, input_dict, gf gfa_path = os.path.join(options.outputGFA, '{}.gfa.gz'.format(chrom)) else: gfa_path = options.outputGFA - mgwf_job = job.addChildJobFn(minigraph_construct_workflow, options, config_node, seq_id_map, seq_order, gfa_path, sanitize, walltime=cactus_fast_walltime()) + mgwf_job = job.addChildJobFn(minigraph_construct_workflow, options, config_node, seq_id_map, seq_order, gfa_path, sanitize, walltime=cactus_walltime()) output_dict[chrom] = mgwf_job.rv() return output_dict @@ -283,7 +283,7 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde seq_id_map, seq_order = refonly_seq_id_map, refonly_seq_order ref_size = seq_id_map[options.reference[0]].size if sanitize: - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) sanitized_seq_id_map = sanitize_job.rv() else: sanitized_seq_id_map = seq_id_map @@ -292,12 +292,12 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde xml_node = findRequiredNode(config_node, "graphmap") sort_type = getOptionalAttrib(xml_node, "minigraphSortInput", str, default=None) if sort_type == "mash" and len(seq_id_map) > 2: - sort_job = sanitize_job.addFollowOnJobFn(sort_minigraph_input_with_mash, options, config_node, sanitized_seq_id_map, seq_order, walltime=cactus_fast_walltime()) + sort_job = sanitize_job.addFollowOnJobFn(sort_minigraph_input_with_mash, options, config_node, sanitized_seq_id_map, seq_order, walltime=cactus_walltime()) seq_order = sort_job.rv() prev_job = sort_job else: prev_job = sanitize_job - minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_in_batches, options, config_node, sanitized_seq_id_map, seq_order, gfa_path, walltime=cactus_fast_walltime()) + minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_in_batches, options, config_node, sanitized_seq_id_map, seq_order, gfa_path, walltime=cactus_walltime()) train_id = None if options.lastTrain and len(seq_id_map) > 1: # note: somehow last training memory overruns don't seem to be detected by slurm so we @@ -346,7 +346,7 @@ def sort_minigraph_input_with_mash(job, options, config_node, seq_id_map, seq_or disk = 2 * sum(seq_id_map[x].size for x in names) + seq_id_map[seq_order[0]].size).rv() dist_maps.append(dist_map) - return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps, walltime=cactus_fast_walltime()).rv() + return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps, walltime=cactus_walltime()).rv() def mash_sketch(job, ref_seq, seq_id_map): """ get the sketch """ @@ -497,7 +497,7 @@ def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_or if prev_job: prev_job.addFollowOn(minigraph_job) # delete the output of the previous batch from the job store - minigraph_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[prev_job.rv()], walltime=cactus_fast_walltime()) + minigraph_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[prev_job.rv()], walltime=cactus_walltime()) else: job.addChild(minigraph_job) prev_job = minigraph_job diff --git a/src/cactus/refmap/cactus_pangenome.py b/src/cactus/refmap/cactus_pangenome.py index b8b9bc610..6090bb222 100644 --- a/src/cactus/refmap/cactus_pangenome.py +++ b/src/cactus/refmap/cactus_pangenome.py @@ -21,7 +21,7 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -307,7 +307,7 @@ def main(): elif genome in input_seq_order: input_seq_order.remove(genome) - toil.start(Job.wrapJobFn(pangenome_end_to_end_workflow, options, config_wrapper, input_seq_id_map, input_path_map, input_seq_order, ref_collapse_paf_id, last_scores_id, walltime=cactus_fast_walltime())) + toil.start(Job.wrapJobFn(pangenome_end_to_end_workflow, options, config_wrapper, input_seq_id_map, input_path_map, input_seq_order, ref_collapse_paf_id, last_scores_id, walltime=cactus_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -384,7 +384,7 @@ def sanitize_fasta_headers_batch(job, chromfile_id_map): seq_id_map = value[0] else: seq_id_map = value - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) if type(value) is list: out_id_map[chrom] = [sanitize_job.rv()] + value[1:] else: @@ -509,11 +509,19 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ job.addChild(root_job) config_node = config_wrapper.xmlRoot + # Every file this pipeline exports -- the GFA, the PAF, the seqfile, the whole join output -- + # is produced by a job further down and reaches its export site as a promise, which has no + # size. The assemblies that went in are the one real size we have here, and since the + # pangenome collapses the sequence they share, every one of those outputs is smaller than + # their total. So it is used as the bound for the export jobs' I/O below: generous for the + # small ones, but these run once per workflow, and the alternative is guessing. + input_seq_bytes = sum(seq_id.size for seq_id in seq_id_map.values()) + # make sure this is done up front root_job = vcflib_checks(root_job, options, config_node) # sanitize headers (once here, skip in all workflows below) - sanitize_job = root_job.addFollowOnJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_fast_walltime()) + sanitize_job = root_job.addFollowOnJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() # snapshot the input contig sizes while the sanitized fastas still exist: they are the baseline @@ -553,13 +561,14 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ else: split_config_node = config_node split_config_wrapper = config_wrapper - minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_workflow, mg_options, split_config_node, seq_id_map, seq_order, sv_gfa_path, sanitize=False, walltime=cactus_fast_walltime()) + minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_workflow, mg_options, split_config_node, seq_id_map, seq_order, sv_gfa_path, sanitize=False, walltime=cactus_walltime()) sv_gfa_id = minigraph_job.rv(0) pansn_sv_gfa_id = minigraph_job.rv(1) if not last_scores_id: last_scores_id = minigraph_job.rv(2) # only build reference graph on first pass when doing minigraph-by-chrom pipeline - minigraph_wrapper_job = minigraph_job.addFollowOnJobFn(export_minigraph_wrapper, options, pansn_sv_gfa_id, sv_gfa_path, last_scores_id, walltime=cactus_fast_walltime()) + minigraph_wrapper_job = minigraph_job.addFollowOnJobFn(export_minigraph_wrapper, options, pansn_sv_gfa_id, sv_gfa_path, last_scores_id, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) # cactus_graphmap paf_path = os.path.join(options.outDir, options.outName + '.paf') @@ -570,17 +579,19 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ gm_options = copy.deepcopy(options) if options.mgSplit: gm_options.collapse = False - graphmap_job = minigraph_wrapper_job.addFollowOnJobFn(minigraph_workflow, gm_options, split_config_wrapper, seq_id_map, sv_gfa_id, graph_event, False, ref_collapse_paf_id, pansn_gfa_input=False, walltime=cactus_fast_walltime()) + graphmap_job = minigraph_wrapper_job.addFollowOnJobFn(minigraph_workflow, gm_options, split_config_wrapper, seq_id_map, sv_gfa_id, graph_event, False, ref_collapse_paf_id, pansn_gfa_input=False, walltime=cactus_walltime()) paf_id, gfa_fa_id, gaf_id, unfiltered_paf_id, paf_filter_log = graphmap_job.rv(0), graphmap_job.rv(1), graphmap_job.rv(2), graphmap_job.rv(3), graphmap_job.rv(4) - graphmap_export_job = graphmap_job.addFollowOnJobFn(export_graphmap_wrapper, options, paf_id, paf_path, gaf_id, unfiltered_paf_id, paf_filter_log, walltime=cactus_fast_walltime()) + graphmap_export_job = graphmap_job.addFollowOnJobFn(export_graphmap_wrapper, options, paf_id, paf_path, gaf_id, unfiltered_paf_id, paf_filter_log, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) # we need to update the seqfile with the phonied in minigraph event - update_seqfile_job = graphmap_export_job.addFollowOnJobFn(update_seqfile, options, seq_id_map, seq_path_map, seq_order, gfa_fa_id, gfa_fa_path, graph_event, walltime=cactus_fast_walltime()) + update_seqfile_job = graphmap_export_job.addFollowOnJobFn(update_seqfile, options, seq_id_map, seq_path_map, seq_order, gfa_fa_id, gfa_fa_path, graph_event, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) seq_id_map, seq_path_map, seq_name_map = update_seqfile_job.rv(0), update_seqfile_job.rv(1), update_seqfile_job.rv(2) if options.noSplit: # we phony in the entire alignment as one chromsome called 'all' - phony_chromfile_job = update_seqfile_job.addFollowOnJobFn(phony_chromfile, options, paf_path, walltime=cactus_fast_walltime()) + phony_chromfile_job = update_seqfile_job.addFollowOnJobFn(phony_chromfile, options, paf_path, walltime=cactus_walltime()) chromfile_path = phony_chromfile_job.rv() split_export_job = phony_chromfile_job # nothing is binned or dropped without a split, so the exclusion report has no split log @@ -589,16 +600,16 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ else: # cactus_graphmap_split split_job = update_seqfile_job.addFollowOnJobFn(graphmap_split_workflow, options, split_config_wrapper, seq_id_map, seq_name_map, sv_gfa_id, - sv_gfa_path, paf_id, paf_path, sanitize=False, pansn_gfa_input=False, walltime=cactus_fast_walltime()) + sv_gfa_path, paf_id, paf_path, sanitize=False, pansn_gfa_input=False, walltime=cactus_walltime()) wf_output = split_job.rv() split_log_id = split_job.rv(2) split_out_path = os.path.join(options.outDir, 'chrom-subproblems') - split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper, walltime=cactus_fast_walltime()) + split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper, walltime=cactus_walltime()) chromfile_path = os.path.join(split_out_path, 'chromfile.txt') # clean out some jobstore files we no longer need clean_jobstore_job = split_export_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[seq_id_map] if not options.noSplit else None, - file_ids=[sv_gfa_id, paf_id], walltime=cactus_fast_walltime()) + file_ids=[sv_gfa_id, paf_id], walltime=cactus_walltime()) options.batch = True minigraph_pansn_sv_gfa_ids = [] @@ -608,14 +619,14 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ chromfile_path) input_seqfiles = minigraph_batch_import_job.rv(0) raw_input_map = minigraph_batch_import_job.rv(1) - sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map, walltime=cactus_fast_walltime()) + sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map, walltime=cactus_walltime()) input_map = sanitize_job.rv() options.outputGFA='' minigraph_batch_job = sanitize_job.addFollowOnJobFn(minigraph_construct_batch_workflow, options, config_node, - input_map, None, sanitize=False, walltime=cactus_fast_walltime()) + input_map, None, sanitize=False, walltime=cactus_walltime()) minigraph_batch_results = minigraph_batch_job.rv() minigraph_batch_export_job = minigraph_batch_job.addFollowOnJobFn(export_minigraph_batch_wrapper, options, config_node, - input_seqfiles, input_map, minigraph_batch_results, walltime=cactus_fast_walltime()) + input_seqfiles, input_map, minigraph_batch_results, walltime=cactus_walltime()) # now rerun cactus_graphmap but on a per-chromosome bassis graphmap_input_dict = minigraph_batch_export_job.rv(0) @@ -624,7 +635,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ minigraph_pansn_sv_gfa_ids = minigraph_batch_export_job.rv(3) graphmap_batch_job = minigraph_batch_export_job.addFollowOnJobFn(minigraph_batch_workflow, options, config_wrapper, graphmap_input_dict, graph_event, sanitize=False, - pansn_gfa_input=False, walltime=cactus_fast_walltime()) + pansn_gfa_input=False, walltime=cactus_walltime()) # hold the contigs of any multi-reference-contig bin apart. the export has to be chained # onto this job, not onto graphmap_batch_job alongside it, or it runs while the separation # pass's children are still going and reads their promises unresolved @@ -632,13 +643,13 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ graphmap_batch_results = separate_job.rv() graphmap_batch_export_job = separate_job.addFollowOnJobFn(export_graphmap_batch_wrapper, options, config_node, graphmap_batch_results, input_seqfiles, - walltime=cactus_fast_walltime()) + walltime=cactus_walltime()) graphmap_file_ids = graphmap_batch_export_job.rv(0) chromfile_path = graphmap_batch_export_job.rv(1) # clean out the jobstore, as cactus_align reads everything from disk - clean_jobstore_job = graphmap_batch_export_job.addFollowOnJobFn(clean_jobstore_files, file_ids=graphmap_file_ids, walltime=cactus_fast_walltime()) + clean_jobstore_job = graphmap_batch_export_job.addFollowOnJobFn(clean_jobstore_files, file_ids=graphmap_file_ids, walltime=cactus_walltime()) clean_jobstore_job = clean_jobstore_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=minigraph_output_maps, - file_ids=minigraph_output_ids, allow_none=True, walltime=cactus_fast_walltime()) + file_ids=minigraph_output_ids, allow_none=True, walltime=cactus_walltime()) # cactus_align options.scoresFromChromfile = options.lastTrain and options.mgSplit @@ -646,7 +657,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ last_scores_id) align_jobs = align_jobs_make_job.rv() - align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs, walltime=cactus_fast_walltime()) + align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs, walltime=cactus_walltime()) results_dict = align_job.rv() align_export_job = align_job.addFollowOnJobFn(export_align_wrapper, options, results_dict) join_options, vg_ids, hal_ids = align_export_job.rv(0), align_export_job.rv(1), align_export_job.rv(2) @@ -658,17 +669,19 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ join_job = align_export_job.addFollowOnJobFn(graphmap_join_workflow, join_options, config_wrapper, vg_ids, [] if options.noHal else hal_ids, minigraph_pansn_sv_gfa_ids, contig_sizes_id=contig_sizes_id, - split_log_id=split_log_id, walltime=cactus_fast_walltime()) + split_log_id=split_log_id, walltime=cactus_walltime()) join_wf_output = join_job.rv() if options.noHal: - join_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_fast_walltime()) + join_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_walltime()) # cactus-panpatch exports the chromosome vgs itself (they're the only thing export_join_data # would write, given it turns every other output off), so let it skip this to avoid writing # the biggest output twice if not options.noJoinExport: + # the biggest export of the run: every chromosome graph, index and VCF the join made join_job.addFollowOnJobFn(export_join_wrapper, join_options, join_wf_output, - contig_sizes_id=contig_sizes_id, walltime=cactus_fast_walltime()) + contig_sizes_id=contig_sizes_id, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) return join_options, join_wf_output, seq_id_map diff --git a/src/cactus/refmap/cactus_panpatch.py b/src/cactus/refmap/cactus_panpatch.py index 9ef69e8cc..5c0449c94 100644 --- a/src/cactus/refmap/cactus_panpatch.py +++ b/src/cactus/refmap/cactus_panpatch.py @@ -28,7 +28,7 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.cactus_prepare import human2bytesN -from cactus.shared.common import importSingularityImage, cactus_fast_walltime, add_cactus_toil_options +from cactus.shared.common import importSingularityImage, cactus_walltime, add_cactus_toil_options from cactus.shared.common import makeURL from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory @@ -245,7 +245,7 @@ def main(): mask_bed_ids, target_error_bed_ids)) toil.start(Job.wrapJobFn(panpatch_batch_workflow, options, config_wrapper, run_inputs, - exclude_bed_id, walltime=cactus_fast_walltime())) + exclude_bed_id, walltime=cactus_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -597,10 +597,10 @@ def panpatch_batch_workflow(job, options, config_wrapper, run_inputs, exclude_be run_job = job.addChildJobFn(panpatch_run_workflow, options, pg_options, config_wrapper, run, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, exclude_bed_id, target_fasta_ids, mask_bed_ids, target_error_bed_ids, - walltime=cactus_fast_walltime()) + walltime=cactus_walltime()) summaries.append((run['name'], run_job.rv())) # once every sample is done, roll the per-sample reports up into one cross-sample summary - job.addFollowOnJobFn(write_batch_summary, options, summaries, walltime=cactus_fast_walltime()) + job.addFollowOnJobFn(write_batch_summary, options, summaries, walltime=cactus_walltime()) def mask_assembly_errors(job, fasta_id, bed_id): """ return a copy of an input assembly fasta with each error-BED interval replaced by an equal-length @@ -676,11 +676,11 @@ def panpatch_run_workflow(job, options, pg_options, config_wrapper, run, seq_id_ # done, which is what makes panpatch_workflow safe pangenome_job = job.addFollowOnJobFn(pangenome_end_to_end_workflow, pg_options, config_wrapper, masked, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, - walltime=cactus_fast_walltime()) + walltime=cactus_walltime()) # return the run's output-file map (its report id feeds the cross-sample batch summary) return pangenome_job.addFollowOnJobFn(panpatch_workflow, options, run, pangenome_job.rv(0), pangenome_job.rv(1), pangenome_job.rv(2), exclude_bed_id, target_fasta_ids, - target_error_bed_ids, walltime=cactus_fast_walltime()).rv() + target_error_bed_ids, walltime=cactus_walltime()).rv() def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_map, exclude_bed_id, target_fasta_ids, target_error_bed_ids): @@ -735,9 +735,12 @@ def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_ma target_fasta_ids, target_error_bed_ids, memory=cactus_clamp_memory(max(2**32, gather_mem)), disk=gather_disk) + # this copies the patched assemblies out, and with --keepGraphs every chromosome graph too, + # so it is bounded by the same bytes its disk request is + export_bytes = sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30 gather_job.addFollowOnJobFn(export_panpatch_wrapper, options, run, gather_job.rv(), full_vg_ids, vg_names, - disk=sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30, - walltime=cactus_fast_walltime()) + disk=export_bytes, + walltime=cactus_walltime(0, io_bytes=export_bytes)) # surface this run's output-file map (incl. its report id) up to the cross-sample batch summary return gather_job.rv() @@ -1142,7 +1145,7 @@ def export_panpatch_wrapper(job, options, run, output_id_map, full_vg_ids, vg_na job.fileStore.exportFile(vg_id, makeURL(os.path.join(chrom_dir, vg_name))) if not options.keepPangenome: - job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run, walltime=cactus_fast_walltime()) + job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run, walltime=cactus_walltime()) def concat_reports(chrom_report_paths, cap_lines, out_path): """ concatenate the per-chromosome panpatch reports (in the order given) into one sample report: diff --git a/src/cactus/refmap/cactus_refmap.py b/src/cactus/refmap/cactus_refmap.py index 0a7ece2f5..06f5a15dc 100644 --- a/src/cactus/refmap/cactus_refmap.py +++ b/src/cactus/refmap/cactus_refmap.py @@ -36,7 +36,7 @@ from cactus.refmap import fasta_preprocessing from cactus.refmap import apply_dipcall_bed_filter -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import makeURL from cactus.shared.common import cactus_call from cactus.shared.configWrapper import ConfigWrapper @@ -139,8 +139,12 @@ def run_cactus_reference_align(job, assembly_files, reference, debug_export=Fals """ Preprocesses assemblies, then runs mappings. """ - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, assembly_files, walltime=cactus_fast_walltime()) - mappings = sanitize_job.addFollowOnJobFn(map_all_to_ref, sanitize_job.rv(), reference, debug_export, dipcall_bed_filter, dipcall_vcf_filter, walltime=cactus_fast_walltime()).rv() + # map_all_to_ref reads every assembly out of the jobstore before it schedules anything, so + # it is bounded by the inputs; sanitizing does not change their size + input_seq_bytes = sum(seq_id.size for seq_id in assembly_files.values()) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, assembly_files, walltime=cactus_walltime()) + mappings = sanitize_job.addFollowOnJobFn(map_all_to_ref, sanitize_job.rv(), reference, debug_export, dipcall_bed_filter, dipcall_vcf_filter, + walltime=cactus_walltime(0, io_bytes=input_seq_bytes)).rv() return mappings def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_bed_filter=False, dipcall_vcf_filter=False): @@ -162,7 +166,7 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b * Filters out all mappings below min_var_len=50k and min_mapq=5 from a lastz file Defaults to False. """ - lead_job = job.addChildJobFn(empty, walltime=cactus_fast_walltime()) + lead_job = job.addChildJobFn(empty, walltime=cactus_walltime()) # map all assemblies to the reference. Don't map reference to reference, though. ref_mappings = dict() @@ -328,7 +332,7 @@ def main(): ## Perform alignments: if not toil.options.restart: - alignments = toil.start(Job.wrapJobFn(run_cactus_reference_align, input_seq_id_map, options.reference, options.debug_export, options.dipcall_bed_filter, options.dipcall_vcf_filter, walltime=cactus_fast_walltime())) + alignments = toil.start(Job.wrapJobFn(run_cactus_reference_align, input_seq_id_map, options.reference, options.debug_export, options.dipcall_bed_filter, options.dipcall_vcf_filter, walltime=cactus_walltime())) else: alignments = toil.restart() diff --git a/src/cactus/setup/cactus_align.py b/src/cactus/setup/cactus_align.py index de1f1f5f3..1ae3799e1 100644 --- a/src/cactus/setup/cactus_align.py +++ b/src/cactus/setup/cactus_align.py @@ -15,7 +15,7 @@ import multiprocessing from operator import itemgetter -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.pipeline.cactus_workflow import cactus_cons_with_resources from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree from cactus.progressive.cactus_progressive import export_hal @@ -196,7 +196,7 @@ def main(): results_dict = toil.restart() else: align_jobs = make_batch_align_jobs(options, toil) - results_dict = toil.start(Job.wrapJobFn(batch_align_jobs, align_jobs, walltime=cactus_fast_walltime())) + results_dict = toil.start(Job.wrapJobFn(batch_align_jobs, align_jobs, walltime=cactus_walltime())) # when using s3 output urls, things get checkpointed as they're made so no reason to export # todo: make a more unified interface throughout cactus for this @@ -407,7 +407,7 @@ def make_align_job(options, toil, config_wrapper=None, chrom_name=None): do_filter_paf=options.pangenome, chrom_name=chrom_name, scores_id=scores_id, - branch_scale=options.branchScale, walltime=cactus_fast_walltime()) + branch_scale=options.branchScale, walltime=cactus_walltime()) return align_job def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, paf_id, paf_path, root_name, og_map, checkpointInfo, doVG, doGFA, delay=0, @@ -431,7 +431,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, apply_scores_to_config(score_dict, config_wrapper.xmlRoot) # unzip the input sequences and enforce unique header prefixes - sanitize_job = head_job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, pangenome=doVG or doGFA or do_filter_paf, walltime=cactus_fast_walltime()) + sanitize_job = head_job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, pangenome=doVG or doGFA or do_filter_paf, walltime=cactus_walltime()) new_seq_id_map = sanitize_job.rv() # run pangenome-specific paf filter. this also runs in cactus-graphmap-split, but that stage is @@ -469,7 +469,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # run consolidated cons_job = head_job.addFollowOnJobFn(cactus_cons_with_resources, spanning_tree, root_name, config_wrapper.xmlRoot, new_seq_id_map, og_map, paf_id, cons_cores = cons_cores, cons_memory=cons_memory, chrom_name=chrom_name, - cons_retain_pages=cons_retain_pages, walltime=cactus_fast_walltime()) + cons_retain_pages=cons_retain_pages, walltime=cactus_walltime()) results = {root_name : (cons_job.rv(1), cons_job.rv(2))} # get the immediate subtree (which is all export_hal can use) @@ -482,7 +482,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, memory_override=cons_memory) # clean out some of the intermediate jobstore files - hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id], walltime=cactus_fast_walltime()) + hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id], walltime=cactus_walltime()) # optionally create the VG if doVG or doGFA: @@ -656,7 +656,7 @@ def main_batch(): chrom_dict[chrom] = toil.importFile(makeURL(seqfile)), toil.importFile(makeURL(alnFile)) if chrom in options.configOverrides: options.configOverrides[chrom][1] = toil.importFile(makeURL(options.configOverrides[chrom][0])) - results_dict = toil.start(Job.wrapJobFn(align_toil_batch, chrom_dict, config_id, options, walltime=cactus_fast_walltime())) + results_dict = toil.start(Job.wrapJobFn(align_toil_batch, chrom_dict, config_id, options, walltime=cactus_walltime())) # when using s3 output urls, things get checkpointed as they're made so no reason to export # todo: make a more unified interface throughout cactus for this diff --git a/src/cactus/shared/common.py b/src/cactus/shared/common.py index 54dbb4149..d81cfc25a 100644 --- a/src/cactus/shared/common.py +++ b/src/cactus/shared/common.py @@ -125,6 +125,42 @@ def cactus_slurm_max_memory(options): # node's full RealMemory can otherwise sit pending forever. return int(max_mb * 1024 * 1024 * 0.95) +def cactus_slurm_max_walltime(options): + """ Query the Slurm cluster for the longest walltime any reachable partition will accept, + in seconds, or None if it can't be determined (or if some partition takes unlimited jobs). + + This is the walltime analogue of cactus_slurm_max_memory, and it matters more than it looks: + when Toil cannot find a partition that fits a job's walltime it does not fall back to a + default, it raises, which kills the workflow at submission time. So we clamp every walltime + estimate to what the cluster can actually run, exactly as we clamp memory to what the biggest + node can actually provide. + + As with memory, we narrow to the partition jobs will actually land on when --slurmPartition + pins one. """ + try: + from toil.batchSystems.slurm import SlurmBatchSystem + partition_set = SlurmBatchSystem.PartitionSet() + partitions = partition_set.all_partitions + except Exception as e: + logger.warning('Unable to query Slurm for partition time limits ({}); not clamping walltime'.format(e)) + return None + if not partitions: + return None + + target_partition = getattr(options, 'slurm_partition', None) + if target_partition: + limits = [p.time_limit for p in partitions if p.partition_name == target_partition] + else: + limits = [p.time_limit for p in partitions] + limits = [t for t in limits if t and t > 0] + if not limits: + return None + max_time = max(limits) + if math.isinf(max_time): + # some partition takes unlimited jobs, so there is nothing to clamp to + return None + return int(max_time) + def cactus_override_toil_options(options): """ Mess with some toil options to create useful defaults. """ if options.retryCount is None and options.batchSystem.lower() not in ['single_machine', 'singleMachine']: @@ -177,12 +213,20 @@ def cactus_override_toil_options(options): os.environ['CACTUS_MAX_MEMORY'] = str(max_mem) os.environ['CACTUS_DEFAULT_MEMORY'] = str(human2bytes(str(options.defaultMemory)) if options.defaultMemory else 2**31) - # store the "fast" walltime (in seconds) for small coordination jobs here so we can get - # at it without carrying options around, and so it propagates to workers (see - # cactus_fast_walltime). An explicit --fastWalltime wins over any inherited env var. - fast_walltime = getattr(options, 'fastWalltime', None) - if fast_walltime: - os.environ['CACTUS_FAST_WALLTIME'] = str(int(fast_walltime)) + # store the walltime knobs here so cactus_walltime() can get at them without carrying + # options around, and so they propagate to workers just like the memory limits above + os.environ['CACTUS_WALLTIME_FACTOR'] = str(getattr(options, 'walltimeFactor', WALLTIME_FACTOR)) + os.environ['CACTUS_MIN_WALLTIME'] = str(getattr(options, 'minWalltime', WALLTIME_MIN)) + max_walltime = getattr(options, 'maxWalltime', 0) or 0 + if not max_walltime and options.batchSystem.lower() == 'slurm': + # no partition can run a job for longer than its time limit, and Toil raises (rather + # than falling back to a default) when it can't find one that fits, so cap our + # estimates at what the cluster will actually accept + slurm_max_walltime = cactus_slurm_max_walltime(options) + if slurm_max_walltime: + logger.info('Clamping maximum job walltime to {} seconds (the longest Slurm partition time limit)'.format(slurm_max_walltime)) + max_walltime = slurm_max_walltime + os.environ['CACTUS_MAX_WALLTIME'] = str(int(max_walltime)) # auto-set cactus_log_memory try: @@ -195,28 +239,101 @@ def cactus_clamp_memory(memory_bytes): """ use the environment variables from --maxMemory and --defaultMemory to clamp a given memory value """ return max(min(int(os.environ['CACTUS_MAX_MEMORY']), int(memory_bytes)), int(os.environ['CACTUS_DEFAULT_MEMORY'])) -def cactus_fast_walltime(): - """ Walltime, in seconds, to assign to small "coordination" jobs (ones that just schedule - other jobs or do trivial work) so that Slurm can route them to a fast partition. Set via - --fastWalltime, which stores CACTUS_FAST_WALLTIME so the value reaches workers just like - cactus_clamp_memory (or export CACTUS_FAST_WALLTIME directly). Returns None (ie no walltime - override, so the job falls back to Toil's --defaultWalltime) when it isn't set. """ - val = os.environ.get('CACTUS_FAST_WALLTIME') - if not val: +# --------------------------------------------------------------------------- +# Per-job walltime +# --------------------------------------------------------------------------- +# Slurm picks each job's partition from its time limit, so one global walltime +# (--slurmTime 100:00:00) drops every job -- including the many thousands that finish in +# seconds -- into the slowest queue. So Cactus estimates a walltime per job, the same way +# it estimates memory, and turns on Toil's --doubleTime so a job that overruns is retried +# with twice the time instead of failing the run. +# +# Call sites pass their best estimate of how long the job actually takes. The safety +# margin (--walltimeFactor) and the [--minWalltime, --maxWalltime] bounds are applied here, +# centrally, so there is one place to tune and one place for a cluster-specific override. + +# Estimated runtime, in seconds, of a job that only schedules other jobs and stages no +# large file. These finish in well under a second; the number is worker startup overhead +# (and, with --binariesMode singularity, cooking the image). This is the default, so a +# bare cactus_walltime() is the right call for a coordination job. +WALLTIME_COORDINATION = 120 + +# Assumed jobstore throughput, in bytes/second, for the io_bytes term. Staging is what +# makes an otherwise trivial job slow, and a shared cluster filesystem with a few hundred +# concurrent Cactus jobs on it goes nowhere near its headline number. WALLTIME_FACTOR is +# applied on top of this, so the effective worst case is a good deal slower again. +WALLTIME_IO_RATE = 100 * 1024**2 + +# Multiplier applied to every estimate, overridable with --walltimeFactor. The estimates +# at the call sites aim at roughly the p99 of what we have measured on the largest runs we +# have logs for, so this is headroom on top of that. +WALLTIME_FACTOR = 2.5 + +# Floor for any walltime request, overridable with --minWalltime. Nothing is gained by +# asking Slurm for less: worker startup, jobstore round-trips and Slurm's own granularity +# swamp it, and a too-short request just buys a --doubleTime retry. +WALLTIME_MIN = 600 + +def cactus_walltime(seconds=WALLTIME_COORDINATION, io_bytes=0): + """ Turn an estimate of how long a job takes into the walltime to request for it. + + seconds: estimated compute time, in seconds. Defaults to the coordination tier, so a + bare cactus_walltime() is what a job that only schedules other jobs wants. + Pass None to say the estimate is *unknown* -- that returns None, leaving the + job on Toil's --defaultWalltime rather than guessing. A job whose cost is all + I/O should pass 0 and use io_bytes. + io_bytes: bytes the job stages in and out of the jobstore, which on a busy shared + filesystem routinely dwarfs the compute. Pass real sizes here (a Toil + FileID has .size); a promise has no size at scheduling time, so where the + inputs are promises fold the I/O into a constant `seconds` instead. + + Returns None -- meaning no per-job walltime, so the job falls back to Toil's + --defaultWalltime -- when estimation is turned off with --walltimeFactor 0. + + Mirrors cactus_clamp_memory: the value travels to workers through the environment (see + cactus_override_toil_options), so this works on the leader and in a worker alike. """ + if seconds is None: + return None + factor = float(os.environ.get('CACTUS_WALLTIME_FACTOR', WALLTIME_FACTOR)) + if factor <= 0: return None - seconds = int(val) - # treat 0 (or negative) as "disabled" so it stays a no-op rather than requesting a - # 0-second job (which Slurm would map to its shortest partition) - return seconds if seconds > 0 else None + estimate = float(seconds) + if io_bytes: + estimate += float(io_bytes) / WALLTIME_IO_RATE + estimate *= factor + estimate = max(estimate, float(os.environ.get('CACTUS_MIN_WALLTIME', WALLTIME_MIN))) + max_walltime = float(os.environ.get('CACTUS_MAX_WALLTIME', 0)) + if max_walltime > 0: + estimate = min(estimate, max_walltime) + return int(math.ceil(estimate)) def add_cactus_toil_options(parser): """ Add cactus-specific options on top of Toil's default argument parser (Job.Runner.getDefaultArgumentParser). Call this right after creating the parser. """ - parser.add_argument("--fastWalltime", type=int, default=1800, - help="Walltime, in seconds, to assign to small \"coordination\" jobs so that " - "(on Slurm) Toil can route them to a fast partition [default: 1800 (30 min), " - "which is very conservative for these jobs]. Set to 0 to disable. Pair with " - "Toil's --defaultWalltime, which sets the walltime for every other job.") + parser.add_argument("--walltimeFactor", type=float, default=WALLTIME_FACTOR, + help="Safety multiplier applied to every per-job walltime Cactus estimates " + "[default: {}]. Raise it if jobs are being killed for running over " + "(a slower cluster than the estimates were tuned on); pass 0 to switch " + "per-job walltimes off entirely, leaving every job on Toil's " + "--defaultWalltime.".format(WALLTIME_FACTOR)) + parser.add_argument("--minWalltime", type=int, default=WALLTIME_MIN, + help="Floor, in seconds, for any walltime Cactus requests [default: {}]. " + "Cactus schedules many jobs that finish in under a second; this keeps " + "worker startup and jobstore I/O from pushing them over their " + "limit.".format(WALLTIME_MIN)) + parser.add_argument("--maxWalltime", type=int, default=0, + help="Ceiling, in seconds, for any walltime Cactus requests [default: on " + "Slurm, the longest time limit of any partition the jobs could land " + "on; otherwise none]. This matters because Toil raises, rather than " + "falling back to a default, when no partition can fit a job's " + "walltime.") + + # --doubleTime is what makes tight per-job walltimes safe: a job Slurm kills for running + # over is retried with twice the time rather than failing the run. Default it on the way + # retryCount is defaulted up, while still letting "--doubleTime false" win. Guarded + # because it only exists in Toil newer than 9.5.0 (see toil-requirement.txt). + if any('--doubleTime' in (action.option_strings or []) for action in parser._actions): + parser.set_defaults(doubleTime=True) def makeURL(path_or_url): if urlparse(path_or_url).scheme == '': diff --git a/src/cactus/update/cactus_hal2seqfile.py b/src/cactus/update/cactus_hal2seqfile.py index 0eab5633f..e83fece66 100644 --- a/src/cactus/update/cactus_hal2seqfile.py +++ b/src/cactus/update/cactus_hal2seqfile.py @@ -30,7 +30,7 @@ from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options -from cactus.shared.common import setupBinaries, importSingularityImage, cactus_fast_walltime +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import enableDumpStack from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import makeURL, cactus_call, cactus_clamp_memory @@ -171,7 +171,7 @@ def main(): # defaults to symlink=True in Toil, but we set it explicitly so this holds within the tool hal_id = toil.importFile(makeURL(options.halFile), symlink=True) fa_ids = toil.start(Job.wrapJobFn(export_subtree_fastas, hal_id, - os.path.basename(options.halFile), genomes, lengths, walltime=cactus_fast_walltime())) + os.path.basename(options.halFile), genomes, lengths, walltime=cactus_walltime())) # export each gzipped fasta to /.fa.gz for genome, fa_id in fa_ids.items(): From 2a543249d1d0460c0d53d3de2d8a8f0854bbcf3e Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 10 Sep 2026 08:41:09 -0400 Subject: [PATCH 05/18] Keep half the longest Slurm partition in reserve for a --doubleTime retry Toil raises, rather than falling back, when no partition can fit a job's walltime -- and it does so on the batch system's own thread, so the whole workflow goes down, not just the job. Cactus now turns --doubleTime on by default, which retries a job Slurm killed for running over with twice the walltime. Together those meant an estimate clamped to the longest partition's limit would turn its own first retry into exactly that crash. So the automatic ceiling is now half the longest partition rather than all of it, leaving room for one doubling. A job that genuinely needs more than half still ends up in the long partition, it just spends one attempt getting there. The reserve is skipped with --doubleTime false, where there is nothing to retry into. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- doc/progressive.md | 2 +- src/cactus/shared/common.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/progressive.md b/doc/progressive.md index 3c1dbf23f..5afb8f209 100644 --- a/doc/progressive.md +++ b/doc/progressive.md @@ -424,7 +424,7 @@ Slurm chooses a job's partition from its time limit, so giving every job one big * `--doubleTime true` is **on by default** (Cactus turns it on the way it raises `--retryCount` to 5). It is the time analogue of `--doubleMem`: if slurm kills a job for running past its walltime, it is retried with double the walltime. This is what makes tight per-job estimates safe -- an occasional underestimate costs one retry, not the run. Pass `--doubleTime false` to turn it off. * `--walltimeFactor` is the safety multiplier applied to every estimate (default `2.5`). Raise it if your cluster is slower than the ones the estimates were measured on and jobs are being killed for running over. Pass `--walltimeFactor 0` to switch per-job walltimes off entirely, which puts every job back on `--defaultWalltime`. * `--minWalltime` is the floor for any request, in seconds (default `600`). Cactus schedules a great many jobs that finish in well under a second; this keeps worker startup and jobstore I/O from pushing them over their limit. -* `--maxWalltime` is the ceiling, in seconds. By default, on Slurm, Cactus queries `sinfo` and caps every request at the longest time limit of any partition your jobs could land on. This matters: when Toil cannot find a partition that fits a job's walltime it raises rather than falling back to a default, which would kill the run at submission time. +* `--maxWalltime` is the ceiling, in seconds. By default, on Slurm, Cactus queries `sinfo` and caps every request at *half* the longest time limit of any partition your jobs could land on. This matters more than it looks: when Toil cannot find a partition that fits a job's walltime it raises rather than falling back to a default, and it does so on the batch system's own thread, taking the whole workflow down rather than just the job. Since `--doubleTime` retries a killed job with twice the walltime, an estimate sitting at the longest partition's limit would turn its own first retry into exactly that crash -- hence the half. A job needing more than half your longest partition still gets there, it just spends one attempt on the way. With `--doubleTime false` there is nothing to retry into and the full limit is used. * `--defaultWalltime` is Toil's fallback for jobs Cactus did not estimate a time for. With the estimates on there is very little left for it to cover, so you can usually leave it alone. * `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU. But this option will allow you to override that. * `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job estimates above (so it also disables the partition routing they exist for, and `--doubleTime` along with it). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. diff --git a/src/cactus/shared/common.py b/src/cactus/shared/common.py index d81cfc25a..a486aab9c 100644 --- a/src/cactus/shared/common.py +++ b/src/cactus/shared/common.py @@ -224,8 +224,18 @@ def cactus_override_toil_options(options): # estimates at what the cluster will actually accept slurm_max_walltime = cactus_slurm_max_walltime(options) if slurm_max_walltime: - logger.info('Clamping maximum job walltime to {} seconds (the longest Slurm partition time limit)'.format(slurm_max_walltime)) max_walltime = slurm_max_walltime + if getattr(options, 'doubleTime', False): + # ...and that raise happens on the batch system's own thread, which takes the + # whole workflow down with it, not just the job. --doubleTime retries a job + # Slurm killed for running over with twice the walltime, so an estimate sitting + # at the longest partition's limit turns its own first retry into exactly that + # crash. Keep half the ceiling in reserve so the retry still has somewhere to + # go; a job that needs more than half the longest partition was going to end up + # there regardless, and only pays for one attempt on the way. + max_walltime = int(max_walltime / WALLTIME_DOUBLETIME_RESERVE) + logger.info('Clamping maximum job walltime to {} seconds (from the longest Slurm partition time limit of {})'.format( + max_walltime, slurm_max_walltime)) os.environ['CACTUS_MAX_WALLTIME'] = str(int(max_walltime)) # auto-set cactus_log_memory @@ -274,6 +284,10 @@ def cactus_clamp_memory(memory_bytes): # swamp it, and a too-short request just buys a --doubleTime retry. WALLTIME_MIN = 600 +# How much of the longest Slurm partition to keep in reserve for --doubleTime to retry into. +# 2 leaves room for exactly one doubling; see cactus_override_toil_options. +WALLTIME_DOUBLETIME_RESERVE = 2 + def cactus_walltime(seconds=WALLTIME_COORDINATION, io_bytes=0): """ Turn an estimate of how long a job takes into the walltime to request for it. From b46b85f96f71c185b97c3bb3faa0e56054f500a3 Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 10 Sep 2026 09:23:12 -0400 Subject: [PATCH 06/18] Give a walltime to every remaining Toil job Extends the per-job estimates to the ~160 scheduling sites the first pass left on --defaultWalltime. Coverage is now 283 of 284 job-function sites and all 13 RoundedJob subclasses, against 126 and 1 before. Each rate is fitted to the mined cluster logs and carries its measurement in the comment beside it, in the style of the memory estimates already in these files. Where a job's cost has no size signal it says so and uses a constant. Two things this turned up that are worth naming: - cactus_consolidated has two regimes, selected by (what --maxLen sets), and they differ in shape rather than scale: progressive is 214 * disk_gb**0.906 (r=0.64), pangenome is 2522 * disk_gb**0.251 (r=0.26). Banding to 10 kb bounds bar's work per column, so cost follows reference columns rather than sequence volume -- adding haplotypes moves `disk` a great deal and the runtime hardly at all. Applying the progressive fit to a pangenome chromosome overshoots 13-17x at the median. Both coefficients are divided by 4: the 2x speedup already landed, and a further 2x expected but not yet measured. If that second 2x underdelivers, 5 of 576 progressive and 3 of 50 pangenome alignments run over, none by more than 1.5x, so one --doubleTime retry rescues each. - filter_paf read .size off a promise on the gzipped-PAF path, for disk and memory, both predating this branch -- a gzipped PAF with --pangenome raised AttributeError before it scheduled anything. All three now size from a paf_size captured before the rebinding. An adversarial pass over each module's diff found, and this fixes: a 577-genome CactusPreprocessor fan-out asking 46 h to do nothing but addChild (Toil chains a *lone* successor, so only a one-genome fan-out covers its child's work); align_toil's additive floor giving an evolver-sized test PAF 10 h; FasTAN borrowing Red's rate along with the 3x speedup divisor it has no claim to; hal2vg fitted to HAL sizes recovered from a clamped memory request, which manufactures a superlinear trend the data does not have; maf2bigmaf reusing a peak-disk multiplier as a compression ratio; hal2seqfile substituting the whole HAL for one genome of unknown length; and several call sites stacking their own 3-4x margin on top of --walltimeFactor, which is that factor's job. The gzip rate and the consolidated core scaling are now shared rather than copied into each module that needs them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- src/cactus/cactus_progressive_config.xml | 7 +- src/cactus/maf/cactus_hal2chains.py | 30 +- src/cactus/maf/cactus_maf2bigmaf.py | 45 +- src/cactus/maf/cactus_phast.py | 101 ++++- src/cactus/paf/local_alignment.py | 178 ++++++-- src/cactus/pipeline/cactus_workflow.py | 66 ++- .../preprocessor/cactus_preprocessor.py | 108 ++++- src/cactus/preprocessor/checkUniqueHeaders.py | 16 +- src/cactus/preprocessor/cutHeaders.py | 13 +- src/cactus/preprocessor/dnabrnnMasking.py | 15 +- src/cactus/preprocessor/fastanMasking.py | 15 +- src/cactus/preprocessor/fileMasking.py | 15 +- .../cactus_lastzRepeatMask.py | 18 +- src/cactus/preprocessor/redMasking.py | 14 +- src/cactus/preprocessor/unmasking.py | 31 +- src/cactus/progressive/cactus_prepare.py | 70 +++- src/cactus/progressive/cactus_progressive.py | 43 +- src/cactus/refmap/cactus_graphmap.py | 155 ++++++- src/cactus/refmap/cactus_graphmap_join.py | 383 +++++++++++++++--- src/cactus/refmap/cactus_graphmap_split.py | 68 +++- src/cactus/refmap/cactus_minigraph.py | 58 ++- src/cactus/refmap/cactus_pangenome.py | 26 +- src/cactus/refmap/cactus_panpatch.py | 42 +- src/cactus/refmap/cactus_refmap.py | 38 +- src/cactus/refmap/pangenome_exclusions.py | 10 +- src/cactus/setup/cactus_align.py | 78 +++- src/cactus/shared/common.py | 28 +- src/cactus/update/cactus_hal2seqfile.py | 20 +- 28 files changed, 1442 insertions(+), 249 deletions(-) diff --git a/src/cactus/cactus_progressive_config.xml b/src/cactus/cactus_progressive_config.xml index 441d7800f..f4581c0dd 100644 --- a/src/cactus/cactus_progressive_config.xml +++ b/src/cactus/cactus_progressive_config.xml @@ -138,7 +138,7 @@ = 2 * batch_size: # run combine_chunks in batches - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) batch_results = [] for chunk_idx in range(math.ceil(len(chunked_alignment_files) / batch_size)): batch = chunked_alignment_files[chunk_idx * batch_size : chunk_idx * batch_size + batch_size] + batch_bytes = sum([f.size for f in batch]) batch_results.append(root_job.addChildJobFn(combine_chunks, batch, batch_size, - disk=sum([f.size for f in batch])).rv()) + disk=batch_bytes, + walltime=cactus_walltime(DECHUNK_SECS_PER_CHUNK * len(batch), + io_bytes=2 * batch_bytes)).rv()) + # merge_combined_chunks runs no command at all -- it is a copyfileobj of each batch result -- + # so its time is the staging, which was 23 GB at p90 in the 577-way logs + total_bytes = sum([f.size for f in chunked_alignment_files]) return root_job.addFollowOnJobFn(merge_combined_chunks, batch_results, - disk=2*sum([f.size for f in chunked_alignment_files])).rv() + disk=2*total_bytes, + walltime=cactus_walltime(30, io_bytes=2 * total_bytes)).rv() else: # Make combined alignments file alignment_file = job.fileStore.getLocalTempFile() @@ -430,6 +459,21 @@ def chunked_alignment_memory(genome_a, genome_b, params): genome_a.size, genome_b.size)) +# Seconds of faffy chunk per GB of input fasta. The 577-way blast logs have 8040 chunk calls +# (two per make_chunked_alignments) topping out at 174s, against genomes of up to 10 Gb -- about +# 17 s/Gb at the worst. +CHUNK_SECS_PER_GB = 20 + + +def chunked_alignment_walltime(genome_a, genome_b): + """ Walltime for a make_chunked_alignments job: one faffy chunk pass over each genome, then a + writeGlobalFile per chunk. Both terms scale with the two genomes, which are real FileIDs at + every call site -- where unmasking has replaced them with promises, pass the pre-unmask sizes, + which are the same bytes since unmasking only changes case. """ + seq_size = genome_a.size + genome_b.size + return cactus_walltime(CHUNK_SECS_PER_GB * seq_size / 1e9, io_bytes=2 * seq_size) + + def make_chunked_alignments(job, event_a, genome_a, event_b, genome_b, distance, params): lastz_params_node = params.find("blast") gpu = getOptionalAttrib(lastz_params_node, 'gpu', typeFn=int, default=0) @@ -473,7 +517,13 @@ def make_chunks(genome): walltime=cactus_walltime(get_lastz_walltime(distance, params, chunk_a, chunk_b))).rv()) dechunk_batch_size = getOptionalAttrib(lastz_params_node, 'dechunkBatchSize', typeFn=int, default=1e9) - return job.addFollowOnJobFn(combine_chunks, chunked_alignment_files, dechunk_batch_size).rv() # Combine the chunked alignment files + # combine_chunks either dechunks the chunk pafs itself or fans them out in batches, so it never + # runs more than a batch's worth of dechunks. The chunk pafs are promises here, so the bytes it + # stages are proxied by the sequence they came from (the paf is about 2x the sequence at p90). + dechunks = min(len(chunked_alignment_files), 2 * dechunk_batch_size) + return job.addFollowOnJobFn(combine_chunks, chunked_alignment_files, dechunk_batch_size, + walltime=cactus_walltime(DECHUNK_SECS_PER_CHUNK * dechunks, + io_bytes=8 * (genome_a.size + genome_b.size))).rv() # Combine the chunked alignment files def invert_alignments(job, alignment_file): @@ -496,14 +546,20 @@ def make_ingroup_to_outgroup_alignments_0(job, ingroup_event, outgroup_events, e # is a promise here, so estimate it from the sequence that went into it: on a repeat-rich # genome the ingroup-to-outgroups paf runs to several times the sequence, and this job # otherwise falls back on the 2 Gi default and overruns it by orders of magnitude. - alignment_disk = 10 * (event_names_to_sequences[ingroup_event.iD].size + - sum(event_names_to_sequences[outgroup.iD].size for outgroup in outgroup_events)) - return job.addFollowOnJobFn(invert_alignments, alignment_file, disk=alignment_disk).rv() + alignment_seq_size = (event_names_to_sequences[ingroup_event.iD].size + + sum(event_names_to_sequences[outgroup.iD].size for outgroup in outgroup_events)) + alignment_disk = 10 * alignment_seq_size + # The walltime uses the same proxy for the same reason: paffy invert ran at 65 s/GB of paf at + # p90 across the 1726 inverts in the 577-way blast logs, and the paf is about 2x its sequence + # at p90, so ~150 s/GB of sequence. + return job.addFollowOnJobFn(invert_alignments, alignment_file, disk=alignment_disk, + walltime=cactus_walltime(150 * alignment_seq_size / 1e9, + io_bytes=4 * alignment_seq_size)).rv() def make_ingroup_to_outgroup_alignments_1(job, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # a job should never set its own follow-on, so we hang everything off root_job here to encapsulate - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # align ingroup to first outgroup to produce paf alignments @@ -514,7 +570,9 @@ def make_ingroup_to_outgroup_alignments_1(job, ingroup_event, outgroup_events, e ingroup_event.iD, event_names_to_sequences[ingroup_event.iD], distances[ingroup_event, outgroup], params, memory=chunked_alignment_memory(event_names_to_sequences[outgroup.iD], event_names_to_sequences[ingroup_event.iD], params), - disk=4*(event_names_to_sequences[ingroup_event.iD].size+event_names_to_sequences[outgroup.iD].size)).rv() + disk=4*(event_names_to_sequences[ingroup_event.iD].size+event_names_to_sequences[outgroup.iD].size), + walltime=chunked_alignment_walltime(event_names_to_sequences[outgroup.iD], + event_names_to_sequences[ingroup_event.iD])).rv() # post process the alignments and recursively generate alignments to remaining outgroups # Memory: paffy to_bed creates SequenceCountArray (2 bytes per base) for query (ingroup) sequences @@ -523,14 +581,20 @@ def make_ingroup_to_outgroup_alignments_1(job, ingroup_event, outgroup_events, e return root_job.addFollowOnJobFn(make_ingroup_to_outgroup_alignments_2, alignment, ingroup_event, outgroup_events[1:], event_names_to_sequences, distances, params, disk=4*(ingroup_size + outgroup_size), - memory=cactus_clamp_memory(4*ingroup_size + 2*outgroup_size)).rv() if len(outgroup_events) > 1 else alignment + memory=cactus_clamp_memory(4*ingroup_size + 2*outgroup_size), + # _2 runs paffy to_bed over the ingroup-to-outgroup paf and then + # faffy extract over the ingroup fasta: 43 s/GB of (ingroup + + # outgroup) sequence at p99 in the 577-way blast logs, on top of + # staging the paf and the fasta. The paf is a promise here. + walltime=cactus_walltime(100 * (ingroup_size + outgroup_size) / 1e9, + io_bytes=3 * (ingroup_size + outgroup_size))).rv() if len(outgroup_events) > 1 else alignment def make_ingroup_to_outgroup_alignments_2(job, alignments, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # a job should never set its own follow-on, so we hang everything off root_job here to encapsulate - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # identify all ingroup sub-sequences that remain unaligned longer than a threshold as follows: @@ -564,8 +628,10 @@ def make_ingroup_to_outgroup_alignments_2(job, alignments, ingroup_event, outgro alignments2 = root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, event_names_to_sequences, distances, params, walltime=cactus_walltime()).rv() + # _3 with has_resources unset only re-schedules itself once the two pafs have resolved, so it + # is coordination; the instance that does the merging is sized below return root_job.addFollowOnJobFn(make_ingroup_to_outgroup_alignments_3, ingroup_event, event_names_to_sequences[ingroup_event.iD], - alignments, alignments2).rv() + alignments, alignments2, walltime=cactus_walltime()).rv() def make_ingroup_to_outgroup_alignments_3(job, ingroup_event, ingroup_seq_file, alignments, alignments2, has_resources=False): @@ -573,8 +639,12 @@ def make_ingroup_to_outgroup_alignments_3(job, ingroup_event, ingroup_seq_file, if not has_resources: # unpack promises for disk requirement + paf_size = alignments.size + alignments2.size + # a paffy dechunk of alignments2 then a cat of both. cat measured 10 s/GB of paf at p99 in + # the 577-way blast logs, so staging the two pafs in and the merged one out is the bigger half return job.addChildJobFn(make_ingroup_to_outgroup_alignments_3, ingroup_event, ingroup_seq_file, alignments, - alignments2, has_resources=True, disk=3*(alignments.size + alignments2.size)).rv() + alignments2, has_resources=True, disk=3*paf_size, + walltime=cactus_walltime(30 * paf_size / 1e9, io_bytes=3 * paf_size)).rv() alignments = job.fileStore.readGlobalFile(alignments) # Copy the global alignment files locally alignments2 = job.fileStore.readGlobalFile(alignments2) @@ -599,8 +669,12 @@ def merge_alignments(job, alignment_file1, alignment_file2, has_resources=False) """" Merge together two alignment files """ if not has_resources: # unpack promises for disk requirement + paf_size = alignment_file1.size + alignment_file2.size + # one cat, so this is almost all staging: cat itself was 10 s/GB of paf at p99 in the + # 577-way blast logs against 30 s/GB for reading both pafs and writing the merge return job.addChildJobFn(merge_alignments, alignment_file1, alignment_file2, has_resources=True, - disk = 2 * (alignment_file1.size + alignment_file2.size)).rv() + disk = 2 * paf_size, + walltime=cactus_walltime(20 * paf_size / 1e9, io_bytes=3 * paf_size)).rv() # Get a temporary directory to work in work_dir = job.fileStore.getLocalTempDir() @@ -644,7 +718,14 @@ def chain_alignments_splitting_ingroups_and_outgroups(job, ingroup_alignment_fil total_sequence_size=total_sequence_size, disk=6 * ingroup_size, memory=cactus_clamp_memory(2 * min(ingroup_size, - chain_split_min_size))).rv() + chain_split_min_size)), + # concat every input paf, paffy invert it (65 s/GB + # of paf at p99 in the 577-way blast logs) and + # paffy split_file the result (~69 s/GB, taken + # from the measured paffy filter, which is the + # closest thing to it we have timings for) + walltime=cactus_walltime(200 * ingroup_size / 1e9, + io_bytes=4 * ingroup_size)).rv() # Separately pick the primary of the outgroups to the ingroups. By setting include_inverted_alignments=False # we only get outgroup-to-ingroup alignments and not imgroup-to-ouygroup alignments and therefore primary @@ -655,13 +736,20 @@ def chain_alignments_splitting_ingroups_and_outgroups(job, ingroup_alignment_fil total_sequence_size=total_sequence_size, disk=6 * outgroup_size, memory=cactus_clamp_memory(2 * min(outgroup_size, - chain_split_min_size))).rv() + chain_split_min_size)), + # same pipeline as the ingroup side above, minus + # the paffy invert + walltime=cactus_walltime(120 * outgroup_size / 1e9, + io_bytes=3 * outgroup_size)).rv() # Calculate approximately total alignment file size total_file_size = sum(alignment_file.size for alignment_file in ingroup_alignment_files + outgroup_alignment_files) # Merge the resulting two alignment files into a single set of alignments - return job.addFollowOnJobFn(merge_alignments, chained_ingroup_alignments, chained_outgroup_alignments).rv() + # (both inputs are promises, so this is merge_alignments' coordination instance -- it schedules + # the sized one, which is where the merging happens) + return job.addFollowOnJobFn(merge_alignments, chained_ingroup_alignments, chained_outgroup_alignments, + walltime=cactus_walltime()).rv() def concat_global_files(job, file_ids, output_path): """Download and concatenate file_ids into output_path, deleting each from the job store after use.""" @@ -673,6 +761,13 @@ def concat_global_files(job, file_ids, output_path): job.fileStore.deleteGlobalFile(file_id) +# Seconds per GB of input paf for the whole chain -> tile -> trim -> filter -> rechain -> filter +# pipeline of chain_tile_trim_filter_one_contig. Pairing per-command seconds against paf bytes in +# 444 of the 577-way blast logs gives 240 s/GB at p90, 351 at p99 and 722 at the worst -- and that +# worst case is a 1.7 GB input, i.e. it is the fixed cost showing through rather than a slow rate. +CHAIN_SECS_PER_GB = 400 + + def chain_alignments(job, alignment_files, alignment_names, reference_event_name, params, include_inverted_alignments=True, total_sequence_size=0): """Merge all input PAFs, split by query contig, run the full chain/tile/trim/filter @@ -701,7 +796,8 @@ def chain_alignments(job, alignment_files, alignment_names, reference_event_name return job.addChildJobFn( chain_tile_trim_filter_one_contig, merged_file_id, reference_event_name, params, disk=4 * merged_size, - memory=cactus_clamp_memory(4 * merged_size) + memory=cactus_clamp_memory(4 * merged_size), + walltime=cactus_walltime(CHAIN_SECS_PER_GB * merged_size / 1e9, io_bytes=2 * merged_size) ).rv() # Large input: split by query contig and run per-contig jobs in parallel @@ -720,7 +816,9 @@ def chain_alignments(job, alignment_files, alignment_names, reference_event_name processed_rvs.append( job.addChildJobFn(chain_tile_trim_filter_one_contig, split_file_id, reference_event_name, params, disk=4 * split_size, - memory=cactus_clamp_memory(4 * split_size)).rv() + memory=cactus_clamp_memory(4 * split_size), + walltime=cactus_walltime(CHAIN_SECS_PER_GB * split_size / 1e9, + io_bytes=2 * split_size)).rv() ) return job.addFollowOnJobFn(merge_processed_alignments, processed_rvs, disk=2 * merged_size, @@ -813,14 +911,18 @@ def sanitize_then_make_paf_alignments(job, event_tree_string, event_names_to_seq ancestor_event_string, params, walltime=cactus_walltime()) # gzip the output if requested if output_path and output_path.endswith('.gz'): - gzip_job = paf_job.addFollowOnJobFn(zip_gz, output_path, paf_job.rv()) + # gzipping the final paf: one per run in the 577-way blast logs, p50 165s, p99 575s, max + # 788s. A constant rather than a formula because the only input here is paf_job.rv(), + # a promise, which has no size until it resolves. + gzip_job = paf_job.addFollowOnJobFn(zip_gz, output_path, paf_job.rv(), + walltime=cactus_walltime(900)) return gzip_job.rv() return paf_job.rv() def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancestor_event_string, params): # a job should never set its own follow-on, so we hang everything off the root_job here to encapsulate - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) logger.info("Parsing species tree: {}".format(event_tree_string)) @@ -861,7 +963,7 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params, walltime=cactus_walltime()) for i,ingroup in enumerate(ingroups): event_names_to_sequences[ingroup] = unmask_job.rv(i) - new_root_job = Job() + new_root_job = Job(walltime=cactus_walltime()) root_job.addFollowOn(new_root_job) root_job = new_root_job @@ -880,7 +982,9 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest # pre-unmask size is the right number anyway. memory=chunked_alignment_memory(input_sequence_map[ingroup.iD], input_sequence_map[ingroup2.iD], params), - disk=2*total_sequence_size).rv()) + disk=2*total_sequence_size, + walltime=chunked_alignment_walltime(input_sequence_map[ingroup.iD], + input_sequence_map[ingroup2.iD])).rv()) ingroup_alignment_names.append('{}-{}_vs_{}'.format(ancestor_event_string, ingroup.iD, ingroup2.iD)) distances = get_distances(event_tree) # Distances between all pairs of nodes @@ -903,7 +1007,9 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest distances[ingroup, outgroup], params, memory=chunked_alignment_memory(input_sequence_map[ingroup.iD], input_sequence_map[outgroup.iD], params), - disk=2*total_sequence_size).rv() + disk=2*total_sequence_size, + walltime=chunked_alignment_walltime(input_sequence_map[ingroup.iD], + input_sequence_map[outgroup.iD])).rv() for ingroup in ingroup_events for outgroup in outgroup_events] # for better logs outgroup_alignment_names = ['{}-og_{}'.format(ancestor_event_string, i) for i in range(len(outgroup_alignments))] @@ -927,7 +1033,12 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest ancestor_event_string, params, total_sequence_size=total_sequence_size, disk=6 * total_sequence_size, - memory=cactus_clamp_memory(2 * total_sequence_size)).rv() + memory=cactus_clamp_memory(2 * total_sequence_size), + # every alignment file here is a promise, so the sequence is the + # only size in scope: 600 s/GB of it covers the concat, invert + # and split_file at the p99 paf-to-sequence ratio of about 7 + walltime=cactus_walltime(600 * total_sequence_size / 1e9, + io_bytes=8 * total_sequence_size)).rv() def trim_unaligned_sequences(job, sequences, alignments, params, has_resources=False): @@ -936,9 +1047,14 @@ def trim_unaligned_sequences(job, sequences, alignments, params, has_resources=F if not has_resources: seq_size = sum([seq.size for seq in sequences]) + # paffy to_bed --includeInverted over the paf, which is the heavier both-ends variant of the + # 99 s/GB-of-paf p99 measured in the 577-way blast logs, then one faffy extract per sequence + # and a paffy upconvert, both of which stream at a few s/GB return job.addChildJobFn(trim_unaligned_sequences, sequences, alignments, params, has_resources=True, disk=4*seq_size + 2*alignments.size, - memory=cactus_clamp_memory(4*seq_size + 2*alignments.size)).rv() + memory=cactus_clamp_memory(4*seq_size + 2*alignments.size), + walltime=cactus_walltime(150 * alignments.size / 1e9 + 10 * seq_size / 1e9, + io_bytes=2 * seq_size + 2 * alignments.size)).rv() work_dir = job.fileStore.getLocalTempDir() alignments_file = os.path.join(work_dir, 'alignments.paf') diff --git a/src/cactus/pipeline/cactus_workflow.py b/src/cactus/pipeline/cactus_workflow.py index 63f7e4e05..b96f3786d 100644 --- a/src/cactus/pipeline/cactus_workflow.py +++ b/src/cactus/pipeline/cactus_workflow.py @@ -32,6 +32,20 @@ ############################################################ ############################################################ +# bar is 63.5% of cactus_consolidated's time across the 576 VGP alignments, caf 20.3%, +# reference 15.4%. The parallel part stops improving somewhere around 24 cores -- which is why +# --consCores above that buys memory rather than speed -- so a job given fewer than that, and +# only then, takes proportionally longer. The fits themselves all ran at 64 cores, on the +# plateau. +CONS_CORE_BASELINE = 24 +CONS_PARALLEL_FRACTION = 0.64 + +def cons_core_scale(cores, baseline=CONS_CORE_BASELINE, parallel=CONS_PARALLEL_FRACTION): + """ how much longer cactus_consolidated takes when given `cores` rather than a full node """ + if cores and 0 < cores < baseline: + return (1.0 - parallel) + parallel * (float(baseline) / cores) + return 1.0 + def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, cons_cores = None, cons_memory = None, intermediate_results_url = None, chrom_name = None, cons_retain_pages = None): @@ -155,23 +169,51 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma name, bytes2human(mem), bytes2human(max_system_memory))) mem = max_system_memory - # Runtime, unlike peak memory, tracks input volume closely: fitted to the same 576 VGP - # alignments, secs = 800 * (disk/1e9)**0.95 lands within 65-75% of the walltime every one - # of them actually needed, once cactus_walltime()'s safety factor is applied on top and the - # 2x cactus_consolidated speedup since those logs is taken out. `disk` is used as the size - # term because it already combines the sequence and paf sizes in the proportions that drive - # the work (5:2), so there is one number to key off rather than two. - wt_coef = getOptionalAttrib(cons_node, 'walltime_coefficient_secs', typeFn=float, default=800.0) - wt_exp = getOptionalAttrib(cons_node, 'walltime_input_exponent', typeFn=float, default=0.95) + # Runtime has two regimes, and which one you are in is decided by (what + # --maxLen sets). They are different in shape, not just in scale: + # + # unbanded, 1 Mb (progressive) secs = 214 * disk_gb**0.906 r = 0.64 + # banded, 10 kb (pangenome) secs = 2522 * disk_gb**0.251 r = 0.26 + # + # fitted to 576 VGP 577-way alignments and to the 50 chromosome alignments of two HPRC + # pangenomes. Banding to 10 kb bounds bar's work per column, so the cost follows the number + # of reference columns rather than the sequence volume -- which is why the pangenome + # exponent is nearly flat, and why adding haplotypes barely moves it even though it moves + # `disk` a great deal. Applying the progressive fit to a pangenome chromosome overshoots by + # 13-17x at the median: HPRC chr5 takes 4.1 h and would have been given 312. + # + # The coefficients below are the fits scaled so that cactus_walltime()'s factor covers the + # worst residual, then divided by 4: the 2x cactus_consolidated speedup that has already + # landed since both sets of logs, and a further 2x that is expected but NOT yet measured + # here. If that second 2x underdelivers the cost is small and bounded -- replaying the + # fits against today's times, 5 of 576 progressive and 3 of 50 pangenome alignments would + # run over, none by more than 1.5x, so a single --doubleTime retry rescues every one. The + # gain is not small: it takes the median progressive request from 15.6 h to 7.8 h and the + # number of them over 24 h from 181 to 7. Raise these two attributes if that turns out to + # be optimistic. `disk` is the size term because it already combines the sequence and paf + # sizes in the proportions that drive the work (5:2). + # + # Only the two measured banding values are in real use, so this selects between them rather + # than interpolating a curve through data that does not exist. Anything between them takes + # the unbanded model, which is the conservative side. + banding_limit = getOptionalAttrib(findRequiredNode(config_node, 'bar'), 'bandingLimit', typeFn=int, default=0) + banding_threshold = getOptionalAttrib(cons_node, 'walltime_banding_threshold', typeFn=int, default=100000) + if banding_limit and banding_limit < banding_threshold: + wt_coef = getOptionalAttrib(cons_node, 'walltime_banded_coefficient_secs', typeFn=float, default=550.0) + wt_exp = getOptionalAttrib(cons_node, 'walltime_banded_exponent', typeFn=float, default=0.35) + else: + wt_coef = getOptionalAttrib(cons_node, 'walltime_coefficient_secs', typeFn=float, default=400.0) + wt_exp = getOptionalAttrib(cons_node, 'walltime_input_exponent', typeFn=float, default=0.95) walltime_secs = wt_coef * ((disk / 1e9) ** wt_exp) if disk > 0 else 0 # bar is 63.5% of consolidated's time across those 576 alignments, caf 20.3%, reference # 15.4%; the parallel part of that stops improving somewhere around 24 cores (which is why # --consCores above ~24 buys memory, not speed), so scale up only when a job is given fewer # than that. The fit's own jobs all ran at 64 cores, i.e. already on the plateau. - wt_core_baseline = getOptionalAttrib(cons_node, 'walltime_core_scale_baseline', typeFn=int, default=24) - wt_parallel = getOptionalAttrib(cons_node, 'walltime_parallel_fraction', typeFn=float, default=0.64) - if cons_cores and 0 < cons_cores < wt_core_baseline: - walltime_secs *= (1.0 - wt_parallel) + wt_parallel * (float(wt_core_baseline) / cons_cores) + walltime_secs *= cons_core_scale(cons_cores, + getOptionalAttrib(cons_node, 'walltime_core_scale_baseline', typeFn=int, + default=CONS_CORE_BASELINE), + getOptionalAttrib(cons_node, 'walltime_parallel_fraction', typeFn=float, + default=CONS_PARALLEL_FRACTION)) cons_job = job.addChildJobFn(cactus_cons, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, intermediate_results_url=intermediate_results_url, chrom_name=chrom_name, cores = cons_cores, diff --git a/src/cactus/preprocessor/cactus_preprocessor.py b/src/cactus/preprocessor/cactus_preprocessor.py index afafb37cc..154ecdf6b 100644 --- a/src/cactus/preprocessor/cactus_preprocessor.py +++ b/src/cactus/preprocessor/cactus_preprocessor.py @@ -19,7 +19,7 @@ from sonLib.bioio import getTempDirectory from toil.common import Toil from toil.job import Job -from cactus.shared.common import cactus_call, cactus_walltime +from cactus.shared.common import cactus_call, cactus_walltime, WALLTIME_COORDINATION from cactus.shared.common import RoundedJob from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import runGetChunks @@ -50,6 +50,30 @@ from cactus.preprocessor.checkPreprocessedSequence import preprocessed_fasta_id from cactus.progressive.cactus_prepare import human2bytesN +# Seconds per GB of fasta for the jobs here that make one linear pass over a whole genome. +# Micro-benchmarked on a 210 MB fasta, and unchanged between a 52-scaffold and a 10309-contig +# version of it: the Bio.SeqIO parse-and-rewrite of CheckUniqueHeaders 12 s/GB, the unmaskFasta +# uppercase loop 9 s/GB, checkPreprocessedSequence's crc32 digest 1.5 s/GB per file. The one +# tool in the group, `faffy merge`, is covered too: faffy chunk and faffy extract managed +# ~33 s/GB across the VGP 577-way (n=8040, p99 67 s on ~2 GB genomes). 40 is the slowest of +# those with margin for a contended cluster worker; cactus_walltime's factor takes the +# effective allowance to 100 s/GB. +FASTA_PASS_SECS_PER_GB = 40 + +# Seconds per GB of a --maskFile. See the call site in maskJobOverride below for provenance. +MASK_FILE_SECS_PER_GB = 300 + +def fasta_pass_walltime(size, passes=1, io_multiple=2): + """ Walltime for a job that makes `passes` linear passes over a fasta of `size` bytes and + stages `io_multiple` times that size through the jobstore. + + size is None where the input is still a promise, which has no size at scheduling time; that + gives no estimate at all rather than a wrong one, leaving the job on --defaultWalltime. """ + if not size: + return cactus_walltime(None) + return cactus_walltime(passes * FASTA_PASS_SECS_PER_GB * size / 1e9, + io_bytes=io_multiple * size) + class PreprocessorOptions: def __init__(self, chunkSize, memory, cpu, check, proportionToSample, unmask, preprocessJob, checkAssemblyHub=None, lastzOptions=None, minPeriod=None, @@ -95,7 +119,8 @@ def __init__(self, prepOptions, inChunkID): disk = 2*inChunkID.size memory = max(prepOptions.memory, inChunkID.size) RoundedJob.__init__(self, memory=memory, cores=prepOptions.cpu, disk=disk, - preemptable=True) + preemptable=True, + walltime=fasta_pass_walltime(inChunkID.size)) self.prepOptions = prepOptions self.inChunkID = inChunkID @@ -111,7 +136,7 @@ def run(self, fileStore): class MergeChunks(RoundedJob): def __init__(self, prepOptions, chunkIDList): - RoundedJob.__init__(self, preemptable=True) + RoundedJob.__init__(self, preemptable=True, walltime=cactus_walltime()) self.prepOptions = prepOptions self.chunkIDList = chunkIDList @@ -121,9 +146,11 @@ def run(self, fileStore): class MergeChunks2(RoundedJob): """merge a list of chunks into a fasta file""" def __init__(self, prepOptions, chunkIDList): - disk = 2*sum([chunkID.size for chunkID in chunkIDList]) + total_size = sum([chunkID.size for chunkID in chunkIDList]) + disk = 2*total_size RoundedJob.__init__(self, cores=prepOptions.cpu, memory=prepOptions.memory, disk=disk, - preemptable=True) + preemptable=True, + walltime=fasta_pass_walltime(total_size)) self.prepOptions = prepOptions self.chunkIDList = chunkIDList @@ -143,9 +170,16 @@ class PreprocessSequence(RoundedJob): """Cut a sequence into chunks, process, then merge """ def __init__(self, prepOptions, inSequenceID, chunksToCompute=None): - disk = 3*inSequenceID.size if hasattr(inSequenceID, "size") else None + in_size = inSequenceID.size if hasattr(inSequenceID, "size") else None + disk = 3*in_size if in_size else None + # in the default unchunked config this job is only a genome-sized jobstore round trip, + # but Toil chains its single mask-job child into it whenever the child's memory, cores + # and disk fit: the VGP 577-way preprocess log demanded 577 CheckUniqueHeaders and only + # ever issued 259 of them as batch jobs, so the other ~318 ran inside their parent. So + # it has to carry that child's fasta pass and its output as well as our own staging. RoundedJob.__init__(self, memory=prepOptions.memory, disk=disk, - preemptable=True) + preemptable=True, + walltime=fasta_pass_walltime(in_size, passes=2, io_multiple=3)) self.prepOptions = prepOptions self.inSequenceID = inSequenceID self.chunksToCompute = chunksToCompute @@ -268,11 +302,17 @@ def unmaskFasta(inFasta, outFasta): out.write(line) class BatchPreprocessor(RoundedJob): - def __init__(self, prepXmlElems, inSequenceID, iteration = 0): + def __init__(self, prepXmlElems, inSequenceID, iteration = 0, inSequenceSize=None): self.prepXmlElems = prepXmlElems self.inSequenceID = inSequenceID self.iteration = iteration - RoundedJob.__init__(self, preemptable=True) + if inSequenceSize is None and hasattr(inSequenceID, "size"): + inSequenceSize = inSequenceID.size + # not the coordination tier: a preprocessor with unmask="1" -- dna-brnn in the default + # config, and whichever masker unmasking.py picks in remask mode -- reads the whole + # genome out of the jobstore, uppercases it in python and writes it back + RoundedJob.__init__(self, preemptable=True, + walltime=fasta_pass_walltime(inSequenceSize)) def run(self, fileStore): # Parse the "preprocessor" config xml element @@ -282,6 +322,7 @@ def run(self, fileStore): prepNode = self.prepXmlElems[self.iteration] checkJob = None + inSize = self.inSequenceID.size if hasattr(self.inSequenceID, "size") else None if getOptionalAttrib(prepNode, "active", typeFn = bool, default=True): prepOptions = PreprocessorOptions(chunkSize = int(prepNode.get("chunkSize", default="-1")), preprocessJob=prepNode.attrib["preprocessJob"], @@ -315,15 +356,20 @@ def run(self, fileStore): fileStore.deleteGlobalFile(self.inSequenceID) self.inSequenceID = fileStore.writeGlobalFile(unmaskedInputFile) - ppJob = self.addChild(PreprocessSequence(prepOptions, self.inSequenceID)) + # the size has to be handed over separately: PreprocessSequence sets its own walltime + # in __init__, and self.inSequenceID is a promise on every iteration after the first + ppJob = self.addChild(PreprocessSequence(prepOptions, self.inSequenceID, + inSequenceSize=getattr(self.inSequenceID, 'size', None))) outSeqID = ppJob.rv() # make sure the step only masked/renamed, and did not corrupt the sequence. # this has to run before clean_if_different, which drops the input file. - inSize = self.inSequenceID.size if hasattr(self.inSequenceID, "size") else None + # it pulls both whole genomes out of the jobstore before either crc32 pass starts, + # so it is an I/O job rather than a coordination one checkJob = self.addFollowOnJobFn(check_preprocessed_sequence, self.inSequenceID, outSeqID, prepOptions.preprocessJob, prepOptions.eventName, prepOptions.dnabrnnAction, - disk=3*inSize if inSize else None) + disk=3*inSize if inSize else None, + walltime=fasta_pass_walltime(inSize)) checkJob.addFollowOnJobFn(clean_if_different, self.inSequenceID, outSeqID, walltime=cactus_walltime()) else: @@ -335,7 +381,10 @@ def run(self, fileStore): # starts by deleting its input, which is the very file the check reads as # its output, and follow-ons of the same job would otherwise race previous = checkJob if checkJob is not None else self - return previous.addFollowOn(BatchPreprocessor(self.prepXmlElems, outSeqID, self.iteration + 1)).rv() + # outSeqID is a promise, so the next iteration cannot size itself off it; no + # preprocessor changes the sequence length except by clipping, so our input stands in + return previous.addFollowOn(BatchPreprocessor(self.prepXmlElems, outSeqID, self.iteration + 1, + inSequenceSize=inSize)).rv() else: return outSeqID @@ -375,7 +424,22 @@ class CactusPreprocessor(RoundedJob): """Modifies the input genomes, doing things like masking/checking, etc. """ def __init__(self, inputSequenceIDs, configNode, eventNames=[]): - RoundedJob.__init__(self, disk=sum([id.size for id in inputSequenceIDs if hasattr(id, 'size')]), preemptable=True) + in_bytes = sum([id.size for id in inputSequenceIDs if hasattr(id, 'size')]) + # run() only deepcopies the config and adds one child per genome -- it stages nothing. + # In the VGP 577-way preprocess log this job was issued at 20:58:28 and its 577 children + # at 21:00:05: 97 s covering the Slurm queue, worker startup and 577 child + # serializations, so ~0.1 s per child. 0.25 s per child is twice that, and below ~1900 + # genomes it is indistinguishable from the bare coordination tier. + # + # The exception is a one-genome fan-out (unmasking.py's remask, cactus_progressive.py's + # per-ingroup preprocess): Toil chains a lone successor into the same worker, so there + # the parent has to cover the child's pass over the fasta as well. A real fan-out does + # not, and must not price in every genome it schedules. + chained_bytes = in_bytes if len(inputSequenceIDs) == 1 else 0 + RoundedJob.__init__(self, disk=in_bytes, preemptable=True, + walltime=cactus_walltime(WALLTIME_COORDINATION + 0.25 * len(inputSequenceIDs) + + FASTA_PASS_SECS_PER_GB * chained_bytes / 1e9, + io_bytes=2 * chained_bytes)) self.inputSequenceIDs = inputSequenceIDs self.configNode = configNode self.eventNames = eventNames @@ -407,7 +471,12 @@ def getOutputSequenceFiles(inputSequences, outputSequenceDir): class CactusPreprocessor2(RoundedJob): def __init__(self, inputSequenceID, configNode): - RoundedJob.__init__(self, preemptable=True) + # the body only adds one BatchPreprocessor child, but Toil chains a single successor + # into its predecessor when the successor's memory, cores and disk fit, and these two + # both take the defaults -- so this has to cover a BatchPreprocessor iteration, which + # reads and rewrites the whole genome when the preprocessor it runs has unmask="1" + in_size = inputSequenceID.size if hasattr(inputSequenceID, "size") else None + RoundedJob.__init__(self, preemptable=True, walltime=fasta_pass_walltime(in_size)) self.inputSequenceID = inputSequenceID self.configNode = configNode @@ -481,8 +550,15 @@ def unzip_then_pp(job, config_node, input_fa_paths, input_fa_ids, input_event_na unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids, walltime=cactus_walltime()) if mask_file_id is not None: mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id], walltime=cactus_walltime()) + # a PAF mask file gets one pafcoverage scan of the whole thing; the nearest measured + # analogue is filter-paf-deletions over a whole-pangenome PAF (HPRC v2.1, n=26, p99 + # 701 s), hence MASK_FILE_SECS_PER_GB. mask_file_id.size is the *compressed* size when + # the mask file is gzipped, which is why the rate is this high and why disk is 20x. For + # a plain BED the job is pure XML rewriting and this collapses to the walltime floor. config_node = mask_unzip_job.addFollowOnJobFn(maskJobOverride, config_node, mask_file_path, mask_unzip_job.rv(0), mask_file_action, min_length, - disk=mask_file_id.size*20).rv() + disk=mask_file_id.size*20, + walltime=cactus_walltime(MASK_FILE_SECS_PER_GB * mask_file_id.size / 1e9, + io_bytes=4 * mask_file_id.size)).rv() pp_job = unzip_job.addFollowOn(CactusPreprocessor([unzip_job.rv(i) for i in range(len(input_fa_ids))], config_node, eventNames=input_event_names)) zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0], walltime=cactus_walltime()) return zip_job.rv() diff --git a/src/cactus/preprocessor/checkUniqueHeaders.py b/src/cactus/preprocessor/checkUniqueHeaders.py index 6d8825a7f..55e97786b 100644 --- a/src/cactus/preprocessor/checkUniqueHeaders.py +++ b/src/cactus/preprocessor/checkUniqueHeaders.py @@ -6,6 +6,16 @@ import os from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime + +# Seconds per GB of *input* fasta for sanitize_fasta_header, the busiest job this module +# defines. Rank-pairing its command time against its input size gives 42-49 s/GB at the median +# in three independent runs (VGP 577-way n=5748, HPRC v2.1 n=22221, HPRC v2.0 n=11385), but the +# tail under heavy concurrency is what a walltime has to survive: in HPRC v2.0 the two commands +# this job runs peaked at 367 s and 223 s over ~0.98 GB gzipped inputs. Those maxima are from +# different instances, so their sum is an upper bound rather than one job's time; 250 s/GB is +# aimed at that bound rather than at the median. +SANITIZE_SECS_PER_GB = 250 def checkUniqueHeaders(inputFile, outputFile, eventName, checkAlphaNumeric=False, checkUCSC=False, checkAssemblyHub=True): """Check that headers are unique and meet certain requirements.""" @@ -39,9 +49,13 @@ def sanitize_fasta_headers(job, fasta_id_map, pangenome=False, log_stats=True): """ input must be map of event -> fasta id""" out_fasta_id_map = {} for event, fasta_id in fasta_id_map.items(): + # fasta_id.size is the compressed size when the input is gzipped, while the job writes + # the fasta back out uncompressed -- the same reason disk is 7x -- so io_bytes is 4x out_fasta_id_map[event] = job.addChildJobFn(sanitize_fasta_header, fasta_id, event, pangenome, log_stats, memory=cactus_clamp_memory(fasta_id.size * 4), - disk=fasta_id.size*7).rv() + disk=fasta_id.size*7, + walltime=cactus_walltime(SANITIZE_SECS_PER_GB * fasta_id.size / 1e9, + io_bytes=4 * fasta_id.size)).rv() return out_fasta_id_map def sanitize_fasta_header(job, fasta_id, event, pangenome, log_stats): diff --git a/src/cactus/preprocessor/cutHeaders.py b/src/cactus/preprocessor/cutHeaders.py index 6a7a58040..90b57e64a 100644 --- a/src/cactus/preprocessor/cutHeaders.py +++ b/src/cactus/preprocessor/cutHeaders.py @@ -12,13 +12,24 @@ from Bio.SeqRecord import SeqRecord from cactus.shared.common import RoundedJob +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Seconds per GB of input fasta for the Bio.SeqIO parse-and-rewrite this job does. +# Micro-benchmarked at 8-9 s/GB on a 210 MB fasta, and the same on a 52-scaffold and on a +# 10309-contig version of it, so the record count does not matter. Rounded up to 12 for the +# benchmark being one file on one machine; the cluster margin is --walltimeFactor's job. This +# one really does run on every genome of every --pangenome run, where the VGP 577-way issued +# 635 of them over inputs with a p99 of 9.2 GB. +CUT_HEADERS_SECS_PER_GB = 12 + class CutHeadersJob(RoundedJob): def __init__(self, fastaID, cutBefore, cutBeforeOcc, cutAfter): disk = 2*(fastaID.size) memory = fastaID.size - RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True, + walltime=cactus_walltime(CUT_HEADERS_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=2 * fastaID.size)) self.fastaID = fastaID self.cutBefore = cutBefore self.cutBeforeOcc = cutBeforeOcc diff --git a/src/cactus/preprocessor/dnabrnnMasking.py b/src/cactus/preprocessor/dnabrnnMasking.py index 615e82152..c379ff459 100644 --- a/src/cactus/preprocessor/dnabrnnMasking.py +++ b/src/cactus/preprocessor/dnabrnnMasking.py @@ -17,9 +17,20 @@ from cactus.shared.common import getOptionalAttrib from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Seconds per GB of input fasta for dna-brnn. No direct evidence: dna-brnn is active="0" in the +# default config (--maskMode brnn turns it on) and did not run in any of the mined logs. This is +# from first principles -- dna-brnn is a recurrent-network base classifier that streams the +# genome, and the job asks for a flat 4 GiB and 2 cores, so it is CPU-bound and roughly linear in +# genome size. With cactus_walltime's factor the effective allowance is 1500 s/GB, ~1.3 hours +# for a 3 Gbp genome and ~4.4 hours for the largest VGP one. The bedtools sort/merge/subtract +# and cactus_fasta_softmask_intervals.py tail that follows is a rounding error next to that: the +# latter was p99 181 s over genomes up to 10.5 GB in the VGP 577-way. +DNABRNN_SECS_PER_GB = 600 + def loadDnaBrnnModel(toil, configNode, maskAlpha = False): """ store the model in a toil file id so it can be used in any workflow """ for prepXml in configNode.findall("preprocessor"): @@ -37,7 +48,9 @@ def __init__(self, fastaID, dnabrnnOpts, cpu, minLength=None, action=None, input memory = 4*1024*1024*1024 disk = 2*(fastaID.size) cores = min(cactus_cpu_count(), cpu) - RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, preemptable=True, + walltime=cactus_walltime(DNABRNN_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=3 * fastaID.size)) self.fastaID = fastaID self.minLength = minLength self.action = action diff --git a/src/cactus/preprocessor/fastanMasking.py b/src/cactus/preprocessor/fastanMasking.py index bfb90cce1..161614bde 100644 --- a/src/cactus/preprocessor/fastanMasking.py +++ b/src/cactus/preprocessor/fastanMasking.py @@ -18,17 +18,30 @@ from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime from cactus.preprocessor.maskingCommon import prefilter_cmd, masked_base_count from cactus.preprocessor.maskingCommon import soft_mask_intervals, log_masking_delta +from cactus.preprocessor.redMasking import RED_SECS_PER_GB, RED_SPEEDUP from toil.realtimeLogger import RealtimeLogger +# FasTAN is active="0" by default and has never run in a logged workflow, so there is no rate +# for it. This job is RedMaskJob's twin -- same prefilter, same bed extraction, same interval +# application, one different repeat masker in the middle -- so Red's measured rate stands in. +# RED_SECS_PER_GB has Red's 3x speedup divided out of it, and FasTAN has no claim to that +# discount, so it is multiplied back: this is Red's rate as measured, before Red got faster. +# --doubleTime buys the retry if FasTAN turns out slower still. +FASTAN_SECS_PER_GB = RED_SECS_PER_GB * RED_SPEEDUP + + class FasTANMaskJob(RoundedJob): def __init__(self, fastaID, fastanOpts, fastanPrefilterOpts, eventName=None, unmask=False): memory = cactus_clamp_memory(12*fastaID.size) disk = 5*(fastaID.size) - RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True, + walltime=cactus_walltime(FASTAN_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=2 * fastaID.size)) self.fastaID = fastaID self.fastanOpts = fastanOpts self.fastanPrefilterOpts = fastanPrefilterOpts diff --git a/src/cactus/preprocessor/fileMasking.py b/src/cactus/preprocessor/fileMasking.py index 0a8f182bd..d999fb16e 100644 --- a/src/cactus/preprocessor/fileMasking.py +++ b/src/cactus/preprocessor/fileMasking.py @@ -21,14 +21,27 @@ from cactus.shared.common import getOptionalAttrib from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Seconds per GB of input fasta for FileMaskingJob. The --maskFile path did not run in any of +# the mined logs, but its two dominant commands did, in the VGP 577-way preprocess log: +# cactus_softmask2hardmask n=601 p99 51 s over genomes up to 10.5 GB (~10 s/GB) and +# cactus_fasta_softmask_intervals.py n=577 p99 181 s (~30 s/GB). 80 doubles that sum to cover +# the bed merge loop, which is pure python and has been observed in no log; the margin beyond +# that is --walltimeFactor's job. Caveat for whoever tunes this: the input BED/PAF arrives as an XML +# attribute, so its size is not in scope here at all -- a pathological mask file is invisible to +# this formula and only --doubleTime would save the job. +FILE_MASKING_SECS_PER_GB = 80 + class FileMaskingJob(RoundedJob): def __init__(self, fastaID, minLength=None, action=None, inputBedID=None, eventName=None): disk = 2*(fastaID.size) memory = fastaID.size - RoundedJob.__init__(self, disk=disk, memory=memory, preemptable=True) + RoundedJob.__init__(self, disk=disk, memory=memory, preemptable=True, + walltime=cactus_walltime(FILE_MASKING_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=3 * fastaID.size)) self.fastaID = fastaID self.minLength = minLength self.action = action diff --git a/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py b/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py index 4707d041a..243d223e0 100644 --- a/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py +++ b/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py @@ -13,8 +13,21 @@ from cactus.shared.common import cactus_call from cactus.shared.common import RoundedJob +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Estimated seconds for one repeat-masking lastz. This preprocessor is deprecated and +# active="0", so it never ran in any mined log; the stand-in is the blast phase's lastz, whose +# p99 across the VGP 577-way (n=826872) was 7469 s, or ~3700 s for the 2x-faster lastz we ship +# now. That is generous, which is the point: this job aligns one chunk (10 MB from the config, +# forced to 6 GB in gpu mode) against proportionToSample of the genome, a smaller job than a +# blast-phase chunk pair. Scaled by what this job is actually handed, the same way +# get_lastz_walltime scales the blast phase's own table: the p99 above was measured on a pair of +# full 90 MB chunks, so a repeat-mask job aligning the config's 10 MB chunk against a sampled +# target set is a fraction of it. +LASTZ_REPEAT_MASK_SECS = 3700 +LASTZ_REPEAT_MASK_REFERENCE_BYTES = 2 * 90e6 + class RepeatMaskOptions: def __init__(self, fragment=200, @@ -59,7 +72,10 @@ def __init__(self, repeatMaskOptions, queryID, targetIDs): disk = max(4*(queryID.size + targetsSize), memory) cores = repeatMaskOptions.cpu accelerators = ['cuda:{}'.format(repeatMaskOptions.gpu)] if repeatMaskOptions.gpu else None - RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, accelerators=accelerators, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, accelerators=accelerators, preemptable=True, + walltime=cactus_walltime(LASTZ_REPEAT_MASK_SECS * + (queryID.size + targetsSize) / LASTZ_REPEAT_MASK_REFERENCE_BYTES, + io_bytes=2 * (queryID.size + targetsSize))) self.repeatMaskOptions = repeatMaskOptions self.queryID = queryID self.targetIDs = targetIDs diff --git a/src/cactus/preprocessor/redMasking.py b/src/cactus/preprocessor/redMasking.py index 2aa5209c7..5771e31f5 100644 --- a/src/cactus/preprocessor/redMasking.py +++ b/src/cactus/preprocessor/redMasking.py @@ -66,12 +66,16 @@ def red_memory_estimate(fasta_size, longest_record_bytes): return int(1.25 * (table_bytes + 8 * longest_record_bytes)) +# How much faster Red is than it was when the VGP 577-way logs were made. Named because +# FasTAN borrows Red's rate and has to multiply this back out -- it got no such speedup. +RED_SPEEDUP = 3.0 + # Seconds of Red per GB of input fasta. Across the 625 Red runs of the VGP 577-way (0.13 to -# 10 Gb of genome) the p99 was 2799 s/Gb and the worst 5147 s/Gb; Red is about 3x faster than -# it was for those, hence the /3. Everything else this job runs -- the prefilter, the -# softmask/hardmask conversions, extracting and applying the intervals -- came to well under -# 100 s each even on the largest genome, and is covered by cactus_walltime()'s safety factor. -RED_SECS_PER_GB = 2799 / 3.0 +# 10 Gb of genome) the p99 was 2799 s/Gb and the worst 5147 s/Gb. Everything else this job +# runs -- the prefilter, the softmask/hardmask conversions, extracting and applying the +# intervals -- came to well under 100 s each even on the largest genome, and is covered by +# cactus_walltime()'s safety factor. +RED_SECS_PER_GB = 2799 / RED_SPEEDUP class RedMaskJob(RoundedJob): diff --git a/src/cactus/preprocessor/unmasking.py b/src/cactus/preprocessor/unmasking.py index 78e19c761..1577f4c88 100644 --- a/src/cactus/preprocessor/unmasking.py +++ b/src/cactus/preprocessor/unmasking.py @@ -10,15 +10,34 @@ from toil.realtimeLogger import RealtimeLogger from cactus.preprocessor.cactus_preprocessor import CactusPreprocessor +# Seconds per GB of input fasta for the unmask scan. The inner loop below walks every base of +# every contig in python (`for c in seq_str: ... c.isupper()`), micro-benchmarked at 131 s/GB on +# a 210 MB fasta -- by far the slowest per-byte operation in the preprocessor, ~15x the +# Bio.SeqIO parse-and-rewrite rate on the same file. Rounded up to 150 because the benchmark +# was one file on one machine; the margin for a contended worker is --walltimeFactor's job, not +# this constant's. Two mitigations are deliberately not priced in: the loop stops early once +# threshold unmasked bases are seen, and contigs under min_length are skipped -- but the +# heavily-masked contigs this job exists to find are exactly the ones scanned to the end. +UNMASK_SCAN_SECS_PER_GB = 150 + +# Seconds per GB of fasta for merge_fa: two Bio.SeqIO parse-and-write passes (8-12 s/GB each, +# micro-benchmarked) plus two whole-genome cactus_analyseAssembly runs, which dominate and whose +# tail under heavy concurrency is the worst part of the picture -- n=11392, p99 270 s, max 367 s +# in the HPRC v2.0 pangenome. +MERGE_FA_SECS_PER_GB = 120 + def unmask_contigs_all(job, event_names_to_sequences, ingroup_events, params): """ unmask the given events and return and updated fasta ids for each ingroup as list """ - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) output_ids = [] for event in ingroup_events: + event_size = event_names_to_sequences[event].size unmask_job = root_job.addChildJobFn(unmask_contigs_one, event, event_names_to_sequences[event], params, - memory=event_names_to_sequences[event].size * 2, - disk=event_names_to_sequences[event].size * 5) + memory=event_size * 2, + disk=event_size * 5, + walltime=cactus_walltime(UNMASK_SCAN_SECS_PER_GB * event_size / 1e9, + io_bytes=3 * event_size)) output_ids.append(unmask_job.rv()) return output_ids @@ -94,8 +113,12 @@ def unmask_contigs_one(job, event, fasta_id, params): pp_job = job.addChild(CactusPreprocessor([copy_fasta_id], params, eventNames=[event])) pp_id = pp_job.rv(0) # mix in the masking for the to-mask contigs with our original file + # pp_id is a promise, but it is a preprocessed copy of the same genome, so + # fasta_id.size sizes both: two reads in and one write out, plus margin fa_merge_job = pp_job.addFollowOnJobFn(merge_fa, event, fasta_id, pp_id, set(unmasked_contigs.keys()), - disk=fasta_id.size * 4) + disk=fasta_id.size * 4, + walltime=cactus_walltime(MERGE_FA_SECS_PER_GB * fasta_id.size / 1e9, + io_bytes=4 * fasta_id.size)) fasta_id = fa_merge_job.rv() # delete the pp_id fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id], walltime=cactus_walltime()) diff --git a/src/cactus/progressive/cactus_prepare.py b/src/cactus/progressive/cactus_prepare.py index e29e98cfc..70715eaa9 100644 --- a/src/cactus/progressive/cactus_prepare.py +++ b/src/cactus/progressive/cactus_prepare.py @@ -34,6 +34,7 @@ from cactus.shared.common import write_s3, has_s3, get_aws_region from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime from toil.job import Job from toil.common import Toil @@ -44,6 +45,40 @@ from toil.lib.conversions import human2bytes, bytes2human from toil.lib.accelerators import count_nvidia_gpus +# --toil mode runs each cactus-* command as a nested single-node Toil workflow, so these are +# estimates for a whole command rather than for one of the jobs inside it. All of them come +# from the VGP 577-way, which was run through cactus-prepare's script mode: 576 cactus-blast +# runs, 576 cactus-align runs, one cactus-preprocess run over all 577 genomes. Those nested +# workflows were given 64 cores. + +# Per-ancestor lastz CPU-seconds for cactus-blast: the p99 of the summed command time of those +# 576 blast logs is 12.1M seconds, 99.7% of it lastz, halved for the 2x lastz speedup since. +# lastz is the one command here that is embarrassingly parallel (827k independent jobs in that +# run), so dividing by the cores the nested workflow gets is the right model. +BLAST_CPU_SECS = 6.0e6 + +# cactus-align is cactus_consolidated and little else: it ran p50 10,014 s, p90 34,771 and p99 +# 120,138 over those 576 ancestors -- halve for the 2x consolidated speedup -- while everything +# else in the phase (the paf ops, export_hal, cactus-hal2fasta) summed to under 1,500 s even at +# the p99. Not scaled by cores: consolidated's parallel part plateaus around 24 of them (see +# ), and the fit's own jobs were well past that. +ALIGN_SECS = 60000 + +# One genome through cactus-preprocess. Grouping the 577-way preprocess log by genome gives +# p50 2,835 s, p90 7,172, p99 11,420; taking the 3x Red speedup out of the Red component of +# each (Red is 88% of the total) leaves p50 1,218, p90 2,859, p99 4,776. +PREPROCESS_SECS = 4800 + +# Seconds per halAppendSubtree. The 12 cactus-halAppendSubtrees runs of the 577-way did 578 +# appends in 346,706 s of command time -- 600 s each, and flat across batches of 25 to 53. +HAL_APPEND_SECS_PER_SUBTREE = 600 + +# Extra seconds per append for staging, for the call site where the sizes are promises and so +# cannot go in io_bytes. Those same 12 runs took 414,562 s of wall against 346,706 s of +# command time: 117 s per append of pulling the target HAL and the subtree HALs down and +# pushing the grown target back up. +HAL_APPEND_STAGING_SECS_PER_SUBTREE = 120 + def main_toil(): return main(toil_mode=True) @@ -481,7 +516,7 @@ def get_plan(options, inSeqFile, outSeqFile, configWrapper, toil): # kick things off with an empty job which we will hook subsequent jobs onto # (using RoundedJob because root job must be sublcass of Job, # https://github.com/ComparativeGenomicsToolkit/cactus/pull/284#issuecomment-684125478) - start_job = RoundedJob() + start_job = RoundedJob(walltime=cactus_walltime()) parent_job = start_job job_idx = {} @@ -501,7 +536,8 @@ def get_plan(options, inSeqFile, outSeqFile, configWrapper, toil): job_idx[("preprocess", leaves[i])] = parent_job.addChildJobFn(toil_call_preprocess, options, inSeqFile, outSeqFile, leaves[i], cores=options.preprocessCores, memory=options.preprocessMemory, - disk=options.preprocessDisk) + disk=options.preprocessDisk, + walltime=cactus_walltime(PREPROCESS_SECS)) else: plan += 'cactus-preprocess {} {} {} --inputNames {} {} {}{}{}{}{}{}\n'.format( get_jobstore(options), options.seqFile, options.outSeqFile, ' '.join(pre_batch), @@ -587,7 +623,7 @@ def cigarPath(event): if options.toil and sub_idx == 0: # advance toil phase (only once per original round) # todo: recapitulate exact dependencies - parent_job = parent_job.addFollowOn(Job()) + parent_job = parent_job.addFollowOn(Job(walltime=cactus_walltime())) if options.script: plan += 'pids=()\n' for event in sub_group: @@ -609,7 +645,8 @@ def cigarPath(event): *fa_promises, cores=options.blastCores, memory=options.blastMemory, - disk=options.preprocessDisk) + disk=options.preprocessDisk, + walltime=cactus_walltime(BLAST_CPU_SECS / max(int(options.blastCores or 1), 1))) job_idx[("align", event)] = job_idx[("blast", event)].addFollowOnJobFn(toil_call_align, options, outSeqFile, mc_tree, @@ -622,7 +659,8 @@ def cigarPath(event): leaf_deps + anc_deps, *fa_promises, cores=options.alignCores, memory=options.alignMemory, - disk=options.alignDisk) + disk=options.alignDisk, + walltime=cactus_walltime(ALIGN_SECS)) else: # todo: support cactus interface (it's easy enough here, but cactus_progressive.py needs changes to handle) cactus_options = options.cactusOptions @@ -659,7 +697,7 @@ def cigarPath(event): # advance toil phase if options.toil: - parent_job = parent_job.addFollowOn(Job()) + parent_job = parent_job.addFollowOn(Job(walltime=cactus_walltime())) # stitch together the final tree plan += '\n## HAL merging\n' @@ -711,7 +749,9 @@ def cigarPath(event): *[job_idx[('align', e)].rv(1) for e in event_list], cores=1, memory=options.alignMemory, - disk=options.halAppendDisk) + disk=options.halAppendDisk, + walltime=cactus_walltime((HAL_APPEND_SECS_PER_SUBTREE + + HAL_APPEND_STAGING_SECS_PER_SUBTREE) * len(event_list))) if options.wdl: prev_event = mc_tree.getRootName() @@ -1312,9 +1352,14 @@ def main_hal2fasta(): fa_id = toil.restart() else: hal_id = toil.importFile(options.halFile) + # hal2fasta itself is trivial -- 576 runs on the VGP 577-way were p99 120 s, max + # 167 s -- so the job is really the HAL coming down and the (bgzipped) fasta going + # back up. cactus-prepare emits one of these per ancestor, so it is worth keeping + # them out of a long partition. fa_id = toil.start(Job.wrapJobFn(hal2fasta, hal_id, options.halFile, options.genome, options.outputFastaFile, memory=cactus_clamp_memory(3000000000), - disk=int(hal_id.size * 1.3))) + disk=int(hal_id.size * 1.3), + walltime=cactus_walltime(200, io_bytes=int(1.5 * hal_id.size)))) # export the alignments toil.exportFile(fa_id, makeURL(options.outputFastaFile)) @@ -1381,9 +1426,16 @@ def main_hal_append_subtrees(): else: hal_id = toil.importFile(options.tgtFile) sub_hal_ids = [toil.importFile(sub_hal) for sub_hal in options.subFiles] + # halAppendSubtree averaged 600 s over the 578 appends of the 577-way, and this job + # runs one per subtree in a serial loop; io_bytes is the target HAL coming down + # mutable, every subtree HAL coming down, and the grown target going back up, which + # is what dominates once the target is a few hundred GiB. + sub_hal_bytes = sum([f.size for f in sub_hal_ids]) out_hal_id = toil.start(Job.wrapJobFn(hal_append_subtrees, hal_id, sub_hal_ids, options, memory=cactus_clamp_memory(10 * max([f.size for f in sub_hal_ids])), - disk=2 * (hal_id.size + sum([f.size for f in sub_hal_ids])))) + disk=2 * (hal_id.size + sub_hal_bytes), + walltime=cactus_walltime(HAL_APPEND_SECS_PER_SUBTREE * len(sub_hal_ids), + io_bytes=2 * (hal_id.size + sub_hal_bytes)))) # export the alignments toil.exportFile(out_hal_id, makeURL(options.outHalFile)) diff --git a/src/cactus/progressive/cactus_progressive.py b/src/cactus/progressive/cactus_progressive.py index dd489534b..f8677c171 100755 --- a/src/cactus/progressive/cactus_progressive.py +++ b/src/cactus/progressive/cactus_progressive.py @@ -51,6 +51,14 @@ from sonLib.nxnewick import NXNewick from sonLib.bioio import getTempDirectory +# Seconds of halAppendCactusSubtree per GiB of the c2h + fasta it reads. The cost per GiB is +# steady across two very different data sets: the VGP 577-way spent 64.0 h appending ~7.0 TiB +# in 576 subtrees (33 s/GiB on average, p90 41), and the HPRC v2.1 pangenome's 25 chromosome +# subtrees ran p50 2570 s on ~90 GiB apiece (29 s/GiB, p99 61). export_hal appends every +# subtree of its tree in one serial loop, so it is the average, not the per-append tail, that +# sets a whole-tree export's runtime. +HAL_APPEND_SECS_PER_GIB = 45 + def logAssemblyStats(job, message, name, sequenceID, preemptable=True): sequenceFile = job.fileStore.readGlobalFile(sequenceID) analysisString = cactus_call(parameters=["cactus_analyseAssembly", sequenceFile], check_output=True) @@ -58,7 +66,7 @@ def logAssemblyStats(job, message, name, sequenceID, preemptable=True): def preprocess_all(job, options, config_node, input_seq_id_map): ''' run prepreprocessor on every input sequence ''' - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) events = list(input_seq_id_map.keys()) seq_ids = list(input_seq_id_map.values()) @@ -85,12 +93,15 @@ def save_preprocessed_files(job, options, config_node, seq_id_map): # Log the stats for the preprocessed assemblies for name, sequence in list(seq_id_map.items()): - job.addChildJobFn(logAssemblyStats, "After preprocessing", name, sequence) + # cactus_analyseAssembly never took more than 121 s in 27,938 runs across the VGP + # 577-way and the HPRC v2.1 pangenome; downloading the sequence is the rest of the job + job.addChildJobFn(logAssemblyStats, "After preprocessing", name, sequence, + walltime=cactus_walltime(120, io_bytes=sequence.size)) def progressive_schedule(job, options, config_node, seq_id_map, tree, og_map, root_event): ''' create job for every internal node, use child dependencies to make tree (+ outgroups)''' - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) config_wrapper = ConfigWrapper(config_node) @@ -184,7 +195,8 @@ def progressive_step(job, options, config_node, seq_id_map, tree, og_map, event) # trim the outgroups if outgroups and int(config_node.find("blast").attrib["trimOutgroups"]): # Trim the outgroup sequences trim_sequences = paf_job.addChildJobFn(trim_unaligned_sequences, - [subtree_eventmap[i] for i in outgroups], paf_job.rv(), config_node) + [subtree_eventmap[i] for i in outgroups], paf_job.rv(), config_node, + walltime=cactus_walltime()) cons_job = paf_job.addFollowOnJobFn(progressive_step_2, trim_sequences.rv(), options, config_node, subtree_eventmap, spanning_tree, og_map, event, walltime=cactus_walltime()) @@ -275,16 +287,23 @@ def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=Non cactus_call(parameters=args, job_memory=job.memory) if not has_resources: - disk = 3 * sum([file_id.size for file_id in fa_file_ids + c2h_file_ids]) + total_size = sum([file_id.size for file_id in fa_file_ids + c2h_file_ids]) + disk = 3 * total_size mem = cactus_clamp_memory(5 * (max([file_id.size for file_id in fa_file_ids]) + max([file_id.size for file_id in c2h_file_ids]))) # allows pass-through of memory override from --consMemory if memory_override: mem = memory_override + # this is the pass that actually runs halAppendCactusSubtree, once per subtree root, so + # its cost is the whole c2h+fa volume rather than any one subtree's. io_bytes reads all + # of that back out of the jobstore and writes the merged HAL, which came to ~13% of the + # input on the 577-way. return job.addChildJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, results, event=event, cacheBytes=cacheBytes, cacheMDC=cacheMDC, cacheRDC=cacheRDC, cacheW0=cacheW0, chunk=chunk, inMemory=inMemory, checkpointInfo=checkpointInfo, acyclicEvent=acyclicEvent, has_resources=True, - disk=disk, memory=mem).rv() + disk=disk, memory=mem, + walltime=cactus_walltime(HAL_APPEND_SECS_PER_GIB * total_size / 2**30, + io_bytes=int(1.2 * total_size))).rv() cactus_call(parameters=["halSetMetadata", hal_path, "CACTUS_COMMIT", cactus_commit]) config_path = os.path.join(work_dir, 'config.xml') @@ -326,11 +345,19 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i scaled_tree = get_ancestor_scaled_tree(mc_tree, root_event, max_div, branch_scale=options.branchScale, upweight_ancestors=upweight_ancestors) - progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event, walltime=cactus_walltime()) + # progressive_schedule builds one job per ancestor, and the get_subtree() it does for each + # is O(tree size): 31 ms per ancestor measured on the 577-way's 1153-node tree, so ~18 s for + # the whole of it, plus Toil writing out the 576 job descriptions. Keying off the ancestor + # count keeps a tree much bigger than that from being clipped; anything ordinary floors. + progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event, + walltime=cactus_walltime(0.2 * len(scaled_tree.getSubtreeRootNames()))) # then do the hal export + # the first export_hal pass stages nothing: it resolves the c2h/fasta promises into sizes and + # re-dispatches itself with them, and that resourced pass sizes its own walltime hal_export_job = progressive_job.addFollowOnJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, - progressive_job.rv(), event=root_event, memory_override=options.consMemory) + progressive_job.rv(), event=root_event, memory_override=options.consMemory, + walltime=cactus_walltime()) return hal_export_job.rv() diff --git a/src/cactus/refmap/cactus_graphmap.py b/src/cactus/refmap/cactus_graphmap.py index 1fba7cecd..a70934e4f 100644 --- a/src/cactus/refmap/cactus_graphmap.py +++ b/src/cactus/refmap/cactus_graphmap.py @@ -18,6 +18,7 @@ from cactus.progressive.seqFile import SeqFile from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime +from cactus.shared.common import GZIP_COMPRESS_BYTES_PER_SEC from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles @@ -322,15 +323,48 @@ def add_separate_ref_contigs_job(batch_job, options, config, input_dict): from cactus.refmap.cactus_graphmap_split import separate_ref_contigs_batch reference = options.reference[0] if type(options.reference) is list else options.reference return batch_job.addFollowOnJobFn(separate_ref_contigs_batch, config, input_dict, batch_job.rv(), reference, - getattr(options, 'permissiveContigFilter', None)) + getattr(options, 'permissiveContigFilter', None), walltime=cactus_walltime()) def minigraph_batch_separate_workflow(job, options, config, input_dict, graph_event, sanitize, pansn_gfa_input=True): """ minigraph_batch_workflow followed by the separation pass, for callers that just want the final result and add nothing after it """ batch_job = job.addChildJobFn(minigraph_batch_workflow, options, config, input_dict, graph_event, sanitize, - pansn_gfa_input) + pansn_gfa_input, walltime=cactus_walltime()) return add_separate_ref_contigs_job(batch_job, options, config, input_dict).rv() +# Walltime estimates for the graphmap jobs. Everything below was measured on the two HPRC +# pangenome runs, the biggest graphmaps we have logs for: the whole-panel pass of v2.0 (one 8.3 +# GiB raw GFA, a 34.4 GiB merged PAF) and the per-chromosome pass of v2.1 (~0.6 GiB of GFA and +# ~2.2 GiB of PAF each). + +# The merged minigraph PAF comes out at about 4x the raw GFA it was mapped against: 34.4 GiB +# against 8.3 GiB whole-panel, 2.2 GiB against 0.6 GiB per chromosome. The PAF is only a promise +# while the workflow is being built, so this is how its size reaches the walltimes of the jobs +# that make it, read it and copy it. +PAF_BYTES_PER_GFA_BYTE = 4 + +# A bgzipped GFA or PAF decompresses to about 10x its size, the figure the disk requests here are +# already reckoned at. +RAW_BYTES_PER_GZ_BYTE = 10 + +# Seconds per GB of *compressed* GFA to rename it between PanSN and Cactus. Nothing measures this +# one directly -- cactus-pangenome passes pansn_gfa_input=False, so it only runs from the +# standalone cactus-graphmap and cactus-graphmap-split entry points and it fired in none of the +# runs we have logs for. It rewrites every S-line of the decompressed GFA in python (~40 MB/s) and +# bgzips the result back up (~25 MB/s, the rate of the whole-panel bgzips that were measured), both +# single-threaded and both over a raw GFA ~10x the compressed input it is handed. +GFA_RENAME_SECS_PER_GB = 700 + +# Seconds per GB of raw GFA for rgfa2paf: 402s on the 8.3 GiB whole-panel GFA against a 112s worst +# case on the 0.6 GiB per-chromosome ones, ie ~40 s/GB on top of a ~90s fixed cost. +RGFA2PAF_SECS_PER_GB = 40 + +# Seconds per GB of raw GFA for filter_paf_deletions: 4337s for filter-paf-deletions plus ~500s +# for the vg convert that precedes it on the whole-panel GFA, against 802s worst case on the +# per-chromosome ones. +FILTER_PAF_DELETIONS_SECS_PER_GB = 500 + + def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sanitize, ref_collapse_paf_id, pansn_gfa_input=True): """ Overall workflow takes command line options and returns (paf-id, (optional) fa-id) """ fa_id = None @@ -341,7 +375,7 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa if type(options.reference) is list: options.reference = options.reference[0] - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) mg_cores = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "cpu", typeFn=int, default=1) @@ -353,16 +387,22 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa # add unique prefixes to the input PAF if ref_collapse_paf_id: + # one awk pass over the PAF, then staging it in and out. --collapseRefPAF was not used in + # any of the runs we have logs for, so the ~50 MB/s awk rate is inferred, not measured ref_collapse_paf_id = root_job.addChildJobFn(add_paf_prefixes, ref_collapse_paf_id, options.reference, - disk=2*ref_collapse_paf_id.size).rv() + disk=2*ref_collapse_paf_id.size, + walltime=cactus_walltime(20 * ref_collapse_paf_id.size / 1e9, + io_bytes=2*ref_collapse_paf_id.size)).rv() # convert the GFA from PanSN to Cactus names if pansn_gfa_input: # the renaming pass decompresses the GFA before bgzipping it back up, so it needs room for # the raw copy (reckoned at 10x, as elsewhere) on top of the compressed input and output rename_gfa_job = root_job.addChildJobFn(minigraph_gfa_from_pansn, genome_names, options.minigraphGFA, gfa_id, - disk=gfa_id.size*12) - new_root_job = Job() + disk=gfa_id.size*12, + walltime=cactus_walltime(GFA_RENAME_SECS_PER_GB * gfa_id.size / 1e9, + io_bytes=2*gfa_id.size)) + new_root_job = Job(walltime=cactus_walltime()) root_job.addFollowOn(new_root_job) root_job = new_root_job gfa_id = rename_gfa_job.rv() @@ -371,16 +411,26 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa if options.outputFasta: # convert GFA to fasta scale = 5 if zipped_gfa else 1 + # gfatools barely scales with GFA size -- 199s on the whole-panel GFA against a 239s worst + # case per chromosome -- so a flat compute term plus the staging is the honest shape fa_job = root_job.addChildJobFn(make_minigraph_fasta, gfa_id, options.outputFasta, graph_event, - disk=scale*2*gfa_id_size, memory=cactus_clamp_memory(2*scale*gfa_id_size)) + disk=scale*2*gfa_id_size, memory=cactus_clamp_memory(2*scale*gfa_id_size), + walltime=cactus_walltime(300, io_bytes=2*gfa_id_size)) fa_id = fa_job.rv() if zipped_gfa: # gaf2paf needs unzipped gfa, so we take care of that upfront - gfa_unzip_job = root_job.addChildJobFn(unzip_gz, options.minigraphGFA, gfa_id, delete_original=False, disk=5*gfa_id_size) + # gunzip itself is fast (27s for the 0.83 GiB compressed whole-panel GFA); what costs is + # writing the ~10x bigger raw GFA back to the jobstore + gfa_unzip_job = root_job.addChildJobFn(unzip_gz, options.minigraphGFA, gfa_id, delete_original=False, disk=5*gfa_id_size, + walltime=cactus_walltime(60, io_bytes=(1 + RAW_BYTES_PER_GZ_BYTE) * gfa_id_size)) gfa_id = gfa_unzip_job.rv() gfa_id_size *= 10 options.minigraphGFA = options.minigraphGFA[:-3] + + # size of the merged PAF every job below either makes, reads or copies + paf_bytes = PAF_BYTES_PER_GFA_BYTE * gfa_id_size + paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event, walltime=cactus_walltime()) root_job.addFollowOn(paf_job) @@ -390,7 +440,9 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa # if --refFromGFA is specified, we get the entire alignment from that, otherwise we just take contigs # that didn't get mapped by anything else gfa2paf_job = Job.wrapJobFn(extract_paf_from_gfa, gfa_id, options.minigraphGFA, options.reference, graph_event, paf_job.rv(0) if not options.refFromGFA else None, - disk=gfa_id_size, memory=cactus_clamp_memory(gfa_id_size)) + disk=gfa_id_size, memory=cactus_clamp_memory(gfa_id_size), + walltime=cactus_walltime(120 + RGFA2PAF_SECS_PER_GB * gfa_id_size / 1e9, + io_bytes=gfa_id_size + (0 if options.refFromGFA else paf_bytes))) if options.refFromGFA: root_job.addChild(gfa2paf_job) else: @@ -401,10 +453,12 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa if ref_collapse_paf_id: collapse_paf_id = collapse_job.addFollowOnJobFn(merge_pafs, {"1":collapse_job.rv(), "2":ref_collapse_paf_id}, - disk=gfa_id_size).rv() + disk=gfa_id_size, + walltime=merge_pafs_walltime(paf_bytes)).rv() else: collapse_paf_id = collapse_job.rv() - merge_paf_job = Job.wrapJobFn(merge_pafs, {"1" : paf_job.rv(0), "2" : gfa2paf_job.rv()}, disk=gfa_id_size) + merge_paf_job = Job.wrapJobFn(merge_pafs, {"1" : paf_job.rv(0), "2" : gfa2paf_job.rv()}, disk=gfa_id_size, + walltime=merge_pafs_walltime(paf_bytes)) paf_job.addFollowOn(merge_paf_job) gfa2paf_job.addFollowOn(merge_paf_job) out_paf_id = merge_paf_job.rv() @@ -424,9 +478,15 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa del_filter_job = prev_job.addFollowOnJobFn(filter_paf_deletions, out_paf_id, gfa_id, del_filter, del_filter_threshold, del_size_threshold, disk=8*gfa_id_size, cores=mg_cores, - memory=cactus_clamp_memory(30*gfa_id_size)) + memory=cactus_clamp_memory(30*gfa_id_size), + walltime=cactus_walltime(600 + FILTER_PAF_DELETIONS_SECS_PER_GB * gfa_id_size / 1e9, + io_bytes=gfa_id_size + 2*paf_bytes)) + # the 600s floor is for the tail: per chromosome this gzip has a 79s median but a 1390s + # worst case, which is contention on the shared filesystem, not PAF size unfiltered_paf_id = prev_job.addFollowOnJobFn(zip_gz, 'mg.paf.unfiltered', out_paf_id, delete_original=False, - disk=gfa_id_size).rv() + disk=gfa_id_size, + walltime=cactus_walltime(600 + paf_bytes / GZIP_COMPRESS_BYTES_PER_SEC, + io_bytes=2*paf_bytes)).rv() out_paf_id = del_filter_job.rv(0) filtered_paf_log = del_filter_job.rv(1) paf_was_filtered = del_filter_job.rv(2) @@ -434,7 +494,8 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa if collapse_paf_id: # note: the collapse paf doesn't get merged into unfiltered_paf - merge_collapse_job = prev_job.addFollowOnJobFn(merge_pafs, {"1" : out_paf_id, "2" : collapse_paf_id}, disk=gfa_id_size) + merge_collapse_job = prev_job.addFollowOnJobFn(merge_pafs, {"1" : out_paf_id, "2" : collapse_paf_id}, disk=gfa_id_size, + walltime=merge_pafs_walltime(paf_bytes)) out_paf_id = merge_collapse_job.rv() return out_paf_id, fa_id if options.outputFasta else None, paf_job.rv(1), unfiltered_paf_id, filtered_paf_log, paf_was_filtered @@ -469,10 +530,26 @@ def make_minigraph_fasta(job, gfa_file_id, gfa_file_path, name): return job.fileStore.writeGlobalFile(fa_path) +# minigraph mapping, per job and per GB of sanitized fasta. Grouping the HPRC toil-rt lines by +# job gives p90 4062s for a whole-genome haplotype (v2.0, n=474, ~3.1 GB of fasta) against a p99 +# of ~1200s for a per-chromosome one (v2.1, ~0.12 GB), which is a line at ~1200 s/GB. The +# intercept is raised to 1500 to cover the per-chromosome tail (max 3521s, 3x its own p99), which +# is cluster contention rather than anything the fasta size can see. +MINIGRAPH_MAP_SECS = 1500 +MINIGRAPH_MAP_SECS_PER_GB = 1200 + +# What a genome's minigraph output weighs as a fraction of the fasta it was mapped from: the v2.0 +# whole panel merged ~1.4 TB of sanitized fasta into a 34.4 GiB PAF (2.6%) and a GAF whose +# single-threaded bgzip took 649s (v2.0) / 868s (v2.1) at ~25 MB/s, ie ~16-22 GB (~1.5%); per +# chromosome on v2.1 the PAF ratio is 4.3%. Both merges are handed promises, so the fastas are the +# only handle on how much they move. +PAF_BYTES_PER_FASTA_BYTE = 0.05 +GAF_BYTES_PER_FASTA_BYTE = 0.025 + def minigraph_map_all(job, options, config, gfa_id, fa_id_map, graph_event): """ top-level job to run the minigraph mapping in parallel, returns paf """ # hang everything on this job, to self-contain workflow - top_job = Job() + top_job = Job(walltime=cactus_walltime()) job.addChild(top_job) mg_cores = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "cpu", typeFn=int, default=1) @@ -490,13 +567,19 @@ def minigraph_map_all(job, options, config, gfa_id, fa_id_map, graph_event): event_name = '{}.{}'.format(event, options.mg_chrom_name) minigraph_map_job = top_job.addChildJobFn(minigraph_map_one, config, event_name, fa_id, gfa_id, cores=mg_cores, disk=5*fa_id.size + gfa_id.size, - memory=cactus_clamp_memory(mem)) + memory=cactus_clamp_memory(mem), + walltime=cactus_walltime(MINIGRAPH_MAP_SECS + MINIGRAPH_MAP_SECS_PER_GB * fa_id.size / 1e9, + io_bytes=2*fa_id.size + gfa_id.size)) gaf_id_map[event] = minigraph_map_job.rv(0) paf_id_map[event] = minigraph_map_job.rv(1) - # merge up - paf_merge_job = top_job.addFollowOnJobFn(merge_pafs, paf_id_map) - gaf_merge_job = top_job.addFollowOnJobFn(merge_pafs, gaf_id_map, gzip=True) + # merge up. these two concatenate every genome's output, so they are the biggest movers in + # the workflow: the whole-panel PAF merge is 34.4 GiB out + fa_bytes = sum(fa_id.size for fa_id in fa_id_map.values()) + paf_merge_job = top_job.addFollowOnJobFn(merge_pafs, paf_id_map, + walltime=merge_pafs_walltime(PAF_BYTES_PER_FASTA_BYTE * fa_bytes)) + gaf_merge_job = top_job.addFollowOnJobFn(merge_pafs, gaf_id_map, gzip=True, + walltime=merge_pafs_walltime(GAF_BYTES_PER_FASTA_BYTE * fa_bytes, gzip=True)) return paf_merge_job.rv(), gaf_merge_job.rv() @@ -594,6 +677,19 @@ def minigraph_map_one(job, config, event_name, fa_file_id, gfa_file_id): # return the stable gaf (minigraph output) and the unstable paf return job.fileStore.writeGlobalFile(pansn_gaf_path), job.fileStore.writeGlobalFile(unstable_paf_path) +# What is left of a merge_pafs job once its staging is accounted for: catFiles runs no command, so +# this is worker startup and the python copy loop. +MERGE_PAF_SECS = 60 + +def merge_pafs_walltime(merged_bytes, gzip=False): + """ walltime for a merge_pafs job whose output comes to roughly merged_bytes. The job is all + I/O -- every input is read out of the jobstore and the concatenation written back -- except + with gzip=True, which bgzips the result single-threaded on the way out. """ + secs = MERGE_PAF_SECS + if gzip: + secs += merged_bytes / GZIP_COMPRESS_BYTES_PER_SEC + return cactus_walltime(secs, io_bytes=2*merged_bytes) + def merge_pafs(job, paf_file_id_map, gzip=False): """ merge up some pafs """ paf_paths = [job.fileStore.readGlobalFile(paf_id) for paf_id in paf_file_id_map.values()] @@ -629,11 +725,17 @@ def extract_paf_from_gfa(job, gfa_id, gfa_path, ref_event, graph_event, ignore_p cactus_call(parameters=cmd, outfile=paf_path) return job.fileStore.writeGlobalFile(paf_path) +# Seconds per GB of fasta for a minimap2 -xasm5 self-alignment. There is no measurement behind +# this one: defaults to "none", so self_align ran in none of the runs we have +# logs for. It is minigraph's own mapping rate standing in, and wants replacing with a +# measurement the first time --collapse is used at panel scale. +SELF_ALIGN_SECS_PER_GB = 1200 + def self_align_all(job, config, seq_id_map, reference, collapse_mode): """ run self-alignment. if reference event given, just run on that, otherwise do all genomes """ assert collapse_mode in ['reference', 'all', 'nonref'] assert reference or collapse_mode == 'all' - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) events = [] for event in seq_id_map.keys(): @@ -648,11 +750,14 @@ def self_align_all(job, config, seq_id_map, reference, collapse_mode): collapse_job = root_job.addChildJobFn(self_align, config, event, seq_id_map[event], disk=4*seq_id_map[event].size, memory=4*seq_id_map[event].size, - cores=mg_cores) + cores=mg_cores, + walltime=cactus_walltime(600 + SELF_ALIGN_SECS_PER_GB * seq_id_map[event].size / 1e9, + io_bytes=2*seq_id_map[event].size)) paf_dict[event] = collapse_job.rv() merge_paf_job = root_job.addFollowOnJobFn(merge_pafs, paf_dict, - disk=4*paf_size) + disk=4*paf_size, + walltime=merge_pafs_walltime(PAF_BYTES_PER_FASTA_BYTE * paf_size)) return merge_paf_job.rv() def self_align(job, config, seq_name, seq_id): @@ -690,6 +795,12 @@ def apply_mgsplit_filter_overrides(config_node): graphmap_node.attrib["minGAFBlockLength"] = "0" graphmap_node.attrib["delFilter"] = "-1" +# Seconds per GB of PAF for filter_paf: gaffilter is measured at 1475s on the 34.4 GiB whole-panel +# PAF and 270s worst case on the 2.2 GiB per-chromosome ones, and the python line-by-line pass that +# always runs adds about as much again at ~60 MB/s. Lives here, next to the job, because +# cactus-graphmap-split and cactus-align both schedule it. +FILTER_PAF_SECS_PER_GB = 60 + def filter_paf(job, paf_id, config, reference=None): """ run basic paf-filtering. these are quick filters that are best to do on-the-fly when reading the paf and as such, they are called by cactus-graphmap-split and cactus-align, not here diff --git a/src/cactus/refmap/cactus_graphmap_join.py b/src/cactus/refmap/cactus_graphmap_join.py index 598c9ef28..ef3dc0f96 100644 --- a/src/cactus/refmap/cactus_graphmap_join.py +++ b/src/cactus/refmap/cactus_graphmap_join.py @@ -66,6 +66,177 @@ import pysam +# --- per-job walltime estimates --------------------------------------------------------------- +# +# Nearly every job below scales with the size of the per-chromosome VG files that came in, so the +# estimates are expressed as seconds per GB of those files. They were fit on three whole-genome +# HPRC runs -- hprc-v2.0-mc-chm13-join, hprc-v2.1-mc-chm13 and its eval variant -- each 25 +# chromosomes of 0.005 to 33.4 GB summing to 408 GB, by grouping every logged command by the +# worker temp dir it ran in to recover per-job totals. Each number sits at or just above the +# worst per-GB ratio the three produced; --walltimeFactor is the margin on top of that. They are +# also all 64-core measurements, so a run with far fewer --indexCores is slower than this and +# gets there through --doubleTime. + +def scaled_walltime(secs_per_gb, size_bytes, io_multiple=2): + """ Walltime for a job whose cost tracks the size of what it reads: secs_per_gb of compute per + GB of size_bytes -- one chromosome's graph, or the sum over all of them -- plus io_multiple + times that many bytes staged through the jobstore. size_bytes must be a real FileID size (a + promise has none). """ + return cactus_walltime(secs_per_gb * size_bytes / 1e9, io_bytes=int(size_bytes * io_multiple)) + +# clip_vg, the whole load/GFAffix/clip/normalise/validate pipeline. n=75: the 'full' phase (the +# only one that runs GFAffix) came to 597-630 s/GB at p50 and 729-951 s/GB at p99, the 'clip' +# phase to 336-443 and 454-590. the largest chromosome, 33.4 GB, took 6.1 hours in 'full' +CLIP_VG_SECS_PER_GB = {'full' : 800, 'clip' : 600} + +# vg_clip_vg: vg clip three times plus vg validate. n=75, 292-349 s/GB at p50, 382-440 at the max +VG_CLIP_VG_SECS_PER_GB = 450 + +# vg_to_og: vg convert -f | grep, then odgi build. n=75, 307-310 s/GB at p50, 357-370 at the max +VG_TO_OG_SECS_PER_GB = 400 + +# vg_to_gfa: vg convert -f. n=225 (25 chromosomes x 3 phases), 161-196 s/GB at p99 and 212-228 at +# the max. the unchopped variant streams through an extra vg mod -u, which no logged run +# exercised, so its +40% is a guess. the GFA written back is several times the VG, hence the +# wider I/O multiple at those call sites +VG_TO_GFA_SECS_PER_GB = 250 +VG_TO_GFA_UNCHOPPED_SECS_PER_GB = 350 + +# drop_graph_event: one vg paths -d over the full-phase graph, written back out. it post-dates +# these logs, so it is taken from vg convert -f on the same graphs, which loads and re-serialises +# the same way at 36-230 s/GB, at the fast end of that since -d does much less in between +DROP_GRAPH_EVENT_SECS_PER_GB = 60 + +# snarl_stats: vg stats -R --snarl-sample. n=75, 24-26 s/GB at p50 and 38-64 at p99 (the high +# ratios are the small chromosomes, where fixed overhead dominates); the max was 1119 s on 33.4 GB +SNARL_STATS_SECS_PER_GB = 60 + +# make_odgi_viz: odgi sort, odgi paths -L and odgi viz. n=75, 76-83 s/GB at p50, 132-158 at p99. +# --draw instead runs odgi layout and odgi draw, which the option's own help calls very slow and +# which appear in no log at all, so 4x the viz number is a guess +ODGI_VIZ_SECS_PER_GB = 150 +ODGI_DRAW_SECS_PER_GB = 600 + +# compute_gref_paths: vg paths -L, -d and -u, then a rewritten graph. --gref was in no logged +# run; this is below the measured band for the per-chromosome jobs that load, modify and +# re-serialise a graph (292-590 s/GB), because it gets --indexCores and can short-circuit +COMPUTE_GREF_PATHS_SECS_PER_GB = 200 + +# path_coverage_job: vg paths -E -v (n=75, ~20 s/GB at the top) followed by interval merging in +# python, which issues no command and so is invisible in the logs. 3x the measured half +PATH_COVERAGE_SECS_PER_GB = 60 + +# ref_gaps_job: vg depth -m0, in no logged run. its real driver is the length of the reference +# contig -- it emits one line per reference base -- and that is not in scope at the call site, so +# this is per GB of VG instead, at ~5x the slowest per-chromosome vg command measured on this data +REF_GAPS_SECS_PER_GB = 150 + +# deconstruct, keyed on the phase graph (make_vcf is itself a job, so its ids are resolved). +# n=75, 59-80 s/GB at p50 but 127-256 at the max: the spread is wide because deconstruct tracks +# snarl and allele structure, not just bytes +DECONSTRUCT_SECS_PER_GB = 300 + +# vcfbub and chunked_vcfwave, per GB of the graph deconstruct ran on -- the VCFs they read are +# promises at the call site, but scale with the same panel. n=75 each: vcfbub 7-10 s/GB at p50 +# and 11-24 at the max; chunked_vcfwave (its own bcftools pipeline and chunk bgzips, not the +# vcfwave jobs it fans out to) 11-24 at p50 and 13-37 at p99 +VCFBUB_SECS_PER_GB = 20 +CHUNKED_VCFWAVE_SECS_PER_GB = 40 + +# join_vg: one vg ids -j over every chromosome, 3308-4909 s on 408 GB of graph (8-12 s/GB). it +# reads all of them mutable and writes all of them back, so the staging term is the bigger half +JOIN_VG_SECS_PER_GB = 15 + +# make_vg_indexes: merge the per-chromosome GFAs, then vg gbwt --gbz-format, bgzip and vg snarls. +# n=9, 7.0-9.1 ks over 408 GB, i.e. ~22 s/GB, at 64 cores. the unchopped call site asks for one +# core, and its bgzip of a whole-genome GFA is what does not survive that (371-553 s at 64 +# threads), hence the separate single-core number +MAKE_VG_INDEXES_SECS_PER_GB = 25 +MAKE_VG_INDEXES_1CORE_SECS_PER_GB = 60 + +# extract_gbz_fasta: vg paths -F over the whole-genome GBZ, once per --vcfReference. 508-522 s +# over 408 GB in all three runs (1.27 s/GB); 4x that covers staging the GBZ, which is a promise +# here and so cannot go in io_bytes +EXTRACT_GBZ_FASTA_SECS_PER_GB = 5 + +# extract_vg_fasta: the same extraction done as one vg paths -F per chromosome, staging every +# chromosome VG first. neither call site was exercised, so the compute term is 25 graph loads +# benchmarked against vg paths -E -v on the same graphs (86-666 s each); the staging is the +# honest dominant term and is exactly known here +EXTRACT_VG_FASTA_SECS_PER_GB = 20 + +# make_xg: vg convert -x, in no logged run. this is just above the top of the range of the +# whole-genome index builds measured on the identical GBZ (vg gbwt, vg index -j, vg snarls, vg +# haplotypes: 5-19 s/GB), XG being the largest of those formats +MAKE_XG_SECS_PER_GB = 30 + +# make_giraffe_indexes: vg index -j and vg minimizer. n=3, 2604-3279 s over 408 GB (6.4-8.0 +# s/GB) at 64 cores. 12 leaves room for the second, long-read minimizer pass --lrGiraffe adds, +# which none of the runs built +MAKE_GIRAFFE_INDEXES_SECS_PER_GB = 12 + +# make_haplo_index: vg index -j, vg gbwt -r and vg haplotypes -H. n=3, 8.5-12.1 ks over 408 GB +# (21-30 s/GB) at 64 cores +MAKE_HAPLO_INDEX_SECS_PER_GB = 35 + +# odgi_squeeze: no logged run asked for a whole-genome --odgi. squeeze concatenates graphs that +# are already built, so it belongs at the odgi sort end of the measured odgi work (286-707 s per +# chromosome) rather than the odgi build end (4.2-9.8 ks); staging every .og dominates either way +ODGI_SQUEEZE_SECS_PER_GB = 10 + +# merge_snarl_stats and merge_gref_segs: concatenate the per-chromosome TSVs, sort, compress. +# merge_snarl_stats was measured once at 177 s over 408 GB of graph (0.43 s/GB). the real driver +# is the TSV bytes, which are promises here, so this is per GB of graph at ~5x that observation +MERGE_CHROM_TSV_SECS_PER_GB = 2 + +# run_panacus, per GB of graph and per phase: no logged run passed --panacus. each phase is one +# bgzip -d of a merged GFA that make_vg_indexes' own bgzip and grep passes put at 400-550 GB, +# then one panacus histgrowth pass over it per countType (2 by default) +PANACUS_SECS_PER_GB = 20 + +# compute_exclusions_job: interval work in python over every chromosome's coverage table, whose +# only command is a tar. it post-dates these logs, so this is reasoned from shape, not measured +COMPUTE_EXCLUSIONS_SECS_PER_GB = 15 + +# vcf_cat over a whole genome, per GB of graph -- the per-chromosome VCFs are promises, which is +# why the disk request is reckoned the same way. measured: bcftools query/merge/concat came to +# 10-13 ks on the raw VCFs, 2.3-3.2 ks on the bub ones and ~0.8 ks on the wave ones, up to ~30 +# s/GB. the rest covers fix_vcf_ploidies, which is pure python and so absent from the logs +VCF_CAT_SECS_PER_GB = 45 + +# merge_hal, per GB of HAL: the longest job in the module, measured four times at 43.9-47.7 ks +# over ~445 GB of chromosome HALs (~103 s/GB), with ~2.5 ks of halRemoveGenome on top +MERGE_HAL_SECS_PER_GB = 110 + +# merge_sv_gfa, per GB of gzipped minigraph GFA: pure python, so it issues no command and is +# invisible in the logs. a gzipped GFA expands ~8x, and re-compressing that in-process runs at +# the ~16 MB/s this file already measures for python gzip (see fix_vcf_ploidies) +MERGE_SV_GFA_SECS_PER_GB = 500 + +# vcf_cat of the vcfwave chunks, per GB of the raw VCF they were cut from. n=75, 205-272 s/GB at +# p50 and 289-341 at the max, single-core (that call site asks for no cores) +VCF_CAT_CHUNK_SECS_PER_GB = 400 + +# vcfnorm, per GB of the VCF that drives it. the vcfwave site is measured n=75 at 4.9-6.7 ks per +# GB of raw VCF at p50 and 5.9-7.9 at p99, nearly all of it in the closing bcftools norm | +# vcffixup | bgzip. the vcfbub site (bcftoolsNorm, off in every logged run) keys off the bub VCF +# instead, which is only ~0.6x the raw one at HPRC scale, so its number is scaled up to match +VCFNORM_WAVE_SECS_PER_GB = 7000 +VCFNORM_BUB_SECS_PER_GB = 12000 + +# vcfwave, per MB of its bgzipped chunk and per core. pooled over the 5577 chunks of three runs: +# 25998 MB in 1835 hours, i.e. 254 s/MB. that is ~5x the median rate, because those runs used +# the vcflib build from before commit 8f76312f, whose pathological tail is baked into the +# aggregate; that build also ignored --vcfwaveCores, which is why this is a single-thread rate +VCFWAVE_SECS_PER_MB = 254 + +# cat_stats: concatenate the per-chromosome clip and path stats and gzip them. those TSVs are +# promises at the call site and nothing in scope stands in for them, so this is a flat number. +# it is well above what the logs imply (the equivalent tar there ran in 1.4-2.1 s) because +# path-stats can reach millions of rows on a heavily fragmented panel +CAT_STATS_SECS = 600 + + def main(): parser = Job.Runner.getDefaultArgumentParser() add_cactus_toil_options(parser) @@ -699,7 +870,7 @@ def vcflib_checks(job, options, config_node): """ run the vcflib checks""" # vcfwave isn't included in the static binary release, so we start by checking it's available if options.vcfwave and options.vcf: - vcfwave_check_job = job.addFollowOnJobFn(check_vcfwave) + vcfwave_check_job = job.addFollowOnJobFn(check_vcfwave, walltime=cactus_walltime()) job = vcfwave_check_job # vcffixup isn't included in the static binary release, so we start by checking it's available @@ -707,7 +878,7 @@ def vcflib_checks(job, options, config_node): wave_norm = getOptionalAttrib(findRequiredNode(config_node, "graphmap_join"), "vcfwaveNorm", typeFn=bool, default=True) bub_norm = getOptionalAttrib(findRequiredNode(config_node, "graphmap_join"), "bcftoolsNorm", typeFn=bool, default=False) if options.vcf and merge_dup and (bub_norm or (options.vcfwave and wave_norm)): - vcffixup_check_job = job.addFollowOnJobFn(check_vcffixup) + vcffixup_check_job = job.addFollowOnJobFn(check_vcffixup, walltime=cactus_walltime()) job = vcffixup_check_job return job @@ -715,7 +886,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, bypass_full_ids=None, bypass_clip_ids=None, bypass_filter_ids=None, contig_sizes_id=None, split_log_id=None): - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) root_job = vcflib_checks(root_job, options, config.xmlRoot) @@ -767,12 +938,14 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, assert len(options.vg) == len(vg_ids) for vg_path, vg_id in zip(options.vg, vg_ids): full_job = Job.wrapJobFn(clip_vg, options, config, vg_path, vg_id, 'full', - disk=vg_id.size * 20, memory=max(2**31, min(vg_id.size * 20, max_mem))) + disk=vg_id.size * 20, memory=max(2**31, min(vg_id.size * 20, max_mem)), + walltime=scaled_walltime(CLIP_VG_SECS_PER_GB['full'], vg_id.size)) root_job.addChild(full_job) full_vg_ids.append(full_job.rv(0)) if 'full' in options.odgi + options.chrom_og + options.viz + options.draw: full_og_job = full_job.addFollowOnJobFn(vg_to_og, options, config, vg_path, full_job.rv(0), - disk=vg_id.size * 16, memory=min(max(og_min_size, vg_id.size * 32), max_mem)) + disk=vg_id.size * 16, memory=min(max(og_min_size, vg_id.size * 32), max_mem), + walltime=scaled_walltime(VG_TO_OG_SECS_PER_GB, vg_id.size, io_multiple=3)) og_chrom_ids['full']['og'].append(full_og_job.rv()) prev_job = root_job @@ -780,7 +953,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # join the ids join_job = prev_job.addFollowOnJobFn(join_vg, options, config, full_vg_ids, disk=sum([f.size for f in vg_ids]), - memory=min(max([f.size for f in vg_ids]) * 4, max_mem)) + memory=min(max([f.size for f in vg_ids]) * 4, max_mem), + walltime=scaled_walltime(JOIN_VG_SECS_PER_GB, + sum(f.size for f in vg_ids))) full_vg_ids = [join_job.rv(i) for i in range(len(vg_ids))] prev_job = join_job @@ -791,7 +966,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vg_path, vg_id, full_vg_id in zip(options.vg, vg_ids, full_vg_ids): drop_graph_event_job = join_job.addFollowOnJobFn(drop_graph_event, config, vg_path, full_vg_id, disk=vg_id.size * 3, - memory=cactus_clamp_memory(min(vg_id.size * 6, max_mem))) + memory=cactus_clamp_memory(min(vg_id.size * 6, max_mem)), + walltime=scaled_walltime(DROP_GRAPH_EVENT_SECS_PER_GB, + vg_id.size)) output_full_vg_ids.append(drop_graph_event_job.rv(0)) full_vg_empty.append(drop_graph_event_job.rv(1)) else: @@ -802,41 +979,49 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, clip_vg_ids = [] clipped_stats = None if options.clip or options.filter: - clip_root_job = Job() + clip_root_job = Job(walltime=cactus_walltime()) prev_job.addFollowOn(clip_root_job) clip_vg_stats = [] assert len(options.vg) == len(full_vg_ids) == len(vg_ids) for vg_path, vg_id, input_vg_id in zip(options.vg, full_vg_ids, vg_ids): clip_job = Job.wrapJobFn(clip_vg, options, config, vg_path, vg_id, 'clip', - disk=input_vg_id.size * 20, memory=max(2**31, min(input_vg_id.size * 20, max_mem))) + disk=input_vg_id.size * 20, memory=max(2**31, min(input_vg_id.size * 20, max_mem)), + walltime=scaled_walltime(CLIP_VG_SECS_PER_GB['clip'], input_vg_id.size)) clip_root_job.addChild(clip_job) clip_vg_ids.append(clip_job.rv(0)) clip_vg_stats.append(clip_job.rv(1)) if 'clip' in options.odgi + options.chrom_og + options.viz + options.draw: clip_og_job = clip_job.addFollowOnJobFn(vg_to_og, options, config, vg_path, clip_job.rv(0), disk=input_vg_id.size * 16, - memory=min(max(og_min_size, input_vg_id.size * 32), max_mem)) + memory=min(max(og_min_size, input_vg_id.size * 32), max_mem), + walltime=scaled_walltime(VG_TO_OG_SECS_PER_GB, + input_vg_id.size, io_multiple=3)) og_chrom_ids['clip']['og'].append(clip_og_job.rv()) # join the stats - clipped_stats = clip_root_job.addFollowOnJobFn(cat_stats, clip_vg_stats).rv() + clipped_stats = clip_root_job.addFollowOnJobFn(cat_stats, clip_vg_stats, + walltime=cactus_walltime(CAT_STATS_SECS)).rv() prev_job = clip_root_job # run the "filter" phase to do the vg clip clipping filter_vg_ids = [] if options.filter: - filter_root_job = Job() + filter_root_job = Job(walltime=cactus_walltime()) prev_job.addFollowOn(filter_root_job) assert len(options.vg) == len(clip_vg_ids) == len(vg_ids) for vg_path, vg_id, input_vg_id in zip(options.vg, clip_vg_ids, vg_ids): filter_job = filter_root_job.addChildJobFn(vg_clip_vg, options, config, vg_path, vg_id, disk=input_vg_id.size * 20, - memory=max(2**31, min(input_vg_id.size * 22, max_mem))) + memory=max(2**31, min(input_vg_id.size * 22, max_mem)), + walltime=scaled_walltime(VG_CLIP_VG_SECS_PER_GB, + input_vg_id.size)) filter_vg_ids.append(filter_job.rv()) if 'filter' in options.odgi + options.chrom_og + options.viz + options.draw: filter_og_job = filter_job.addFollowOnJobFn(vg_to_og, options, config, vg_path, filter_job.rv(), disk=input_vg_id.size * 16, - memory=min(max(og_min_size, input_vg_id.size * 64), max_mem)) + memory=min(max(og_min_size, input_vg_id.size * 64), max_mem), + walltime=scaled_walltime(VG_TO_OG_SECS_PER_GB, + input_vg_id.size, io_multiple=3)) og_chrom_ids['filter']['og'].append(filter_og_job.rv()) @@ -850,7 +1035,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, hal_merge_job = job.addChildJobFn(merge_hal, options, config, hal_ids, cores = 1, disk=sum(f.size for f in hal_ids) * 2, - memory=min(max(f.size for f in hal_ids) * 2, max_mem)) + memory=min(max(f.size for f in hal_ids) * 2, max_mem), + walltime=scaled_walltime(MERGE_HAL_SECS_PER_GB, + sum(f.size for f in hal_ids))) hal_id_dict = hal_merge_job.rv() out_dicts.append(hal_id_dict) # delete the chromosome hals @@ -859,7 +1046,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # optional minigraph gfa merge if sv_gfa_ids: sv_gfa_merge_job = job.addChildJobFn(merge_sv_gfa, options, sv_gfa_ids, - disk=sum(f.size for f in sv_gfa_ids) * 3) + disk=sum(f.size for f in sv_gfa_ids) * 3, + walltime=scaled_walltime(MERGE_SV_GFA_SECS_PER_GB, + sum(f.size for f in sv_gfa_ids))) sv_gfa_id_dict = sv_gfa_merge_job.rv() out_dicts.append(sv_gfa_id_dict) # delete the chromosome gfas @@ -890,7 +1079,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, ref_vg_ids = bypass_full_ids or bypass_clip_ids or bypass_filter_ids ref_fasta_job = root_job.addFollowOnJobFn(extract_vg_fasta, options, ref_vg_ids, disk=sum(f.size for f in vg_ids) * 2, - memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4)) + memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4), + walltime=scaled_walltime(EXTRACT_VG_FASTA_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=1)) if not options.bypass: workflow_phases = [('full', full_vg_ids, join_job)] @@ -917,7 +1109,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for workflow_phase, phase_vg_ids, phase_root_job in workflow_phases: # make a gfa for each - gfa_root_job = Job() + gfa_root_job = Job(walltime=cactus_walltime()) phase_root_job.addFollowOn(gfa_root_job) gfa_ids = [] current_out_dict = None @@ -929,7 +1121,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vg_path, vg_id, input_vg_id in zip(options.vg, phase_vg_ids, vg_ids): gfa_job = gfa_root_job.addChildJobFn(vg_to_gfa, options, config, vg_path, vg_id, disk=input_vg_id.size * 10, - memory=min(max(2**31, input_vg_id.size * 16), max_mem)) + memory=min(max(2**31, input_vg_id.size * 16), max_mem), + walltime=scaled_walltime(VG_TO_GFA_SECS_PER_GB, + input_vg_id.size, io_multiple=5)) gfa_ids.append(gfa_job.rv()) gfa_merge_job = gfa_root_job.addFollowOnJobFn(make_vg_indexes, options, config, gfa_ids, @@ -937,7 +1131,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, do_gbz=do_gbz, cores=options.indexCores, disk=sum(f.size for f in vg_ids) * 6, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(MAKE_VG_INDEXES_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(gfa_merge_job.rv()) prev_job = gfa_merge_job current_out_dict = gfa_merge_job.rv() @@ -946,21 +1142,29 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, ref_fasta_job = gfa_merge_job.addFollowOnJobFn(extract_gbz_fasta, options, current_out_dict, tag=workflow_phase + '.', memory=index_mem, - disk=sum(f.size for f in vg_ids) * 2) + disk=sum(f.size for f in vg_ids) * 2, + # one vg paths -F pass per --vcfReference + walltime=scaled_walltime( + EXTRACT_GBZ_FASTA_SECS_PER_GB * max(1, len(options.vcfReference or [])), + sum(f.size for f in vg_ids), io_multiple=0.5)) # optional unchopped gfa if workflow_phase in options.unchopped_gfa: unchopped_gfa_ids = [] for vg_path, vg_id, input_vg_id in zip(options.vg, phase_vg_ids, vg_ids): unchopped_gfa_job = gfa_root_job.addChildJobFn(vg_to_gfa, options, config, vg_path, vg_id, unchopped=True, disk=input_vg_id.size * 10, - memory=min(max(2**31, input_vg_id.size * 16), max_mem)) + memory=min(max(2**31, input_vg_id.size * 16), max_mem), + walltime=scaled_walltime(VG_TO_GFA_UNCHOPPED_SECS_PER_GB, + input_vg_id.size, io_multiple=5)) unchopped_gfa_ids.append(unchopped_gfa_job.rv()) unchopped_gfa_merge_job = gfa_root_job.addFollowOnJobFn(make_vg_indexes, options, config, unchopped_gfa_ids, tag=workflow_phase + '.unchopped.', do_gbz=False, cores=1, - disk=sum(f.size for f in vg_ids) * 3) + disk=sum(f.size for f in vg_ids) * 3, + walltime=scaled_walltime(MAKE_VG_INDEXES_1CORE_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(unchopped_gfa_merge_job.rv()) @@ -970,7 +1174,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, tag=workflow_phase + '.', cores=max(options.indexCores, 4), disk = sum(f.size for f in vg_ids) * 10, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(MAKE_XG_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=1)) out_dicts.append(xg_job.rv()) # optional vcf @@ -988,7 +1195,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, l_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, workflow_phase, index_mem, vcf_ref, phase_vg_ids, ref_fasta_job.rv() if ref_fasta_job else None, - decon_L=options.vcfL) + decon_L=options.vcfL, walltime=cactus_walltime()) if ref_fasta_job: ref_fasta_job.addFollowOn(l_job) out_dicts.append(l_job.rv()) @@ -1013,7 +1220,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # than a human one will exceed the cap and # get there through --doubleMem memory=cactus_clamp_memory( - min(index_mem * 3, 128 * 2**30))) + min(index_mem * 3, 128 * 2**30)), + walltime=scaled_walltime(MAKE_GIRAFFE_INDEXES_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.5)) out_dicts.append(giraffe_job.rv()) # optional haplo index @@ -1030,14 +1240,19 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # and 760Gi. scale up for the small end, where # it barely overflowed, and cap the big end memory=cactus_clamp_memory( - min(int(index_mem * 1.5), 128 * 2**30))) + min(int(index_mem * 1.5), 128 * 2**30)), + walltime=scaled_walltime(MAKE_HAPLO_INDEX_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.5)) out_dicts.append(haplo_job.rv()) # optional full-genome odgi if workflow_phase in options.odgi: odgi_job = gfa_root_job.addChildJobFn(odgi_squeeze, config, options.vg, og_chrom_ids[workflow_phase]['og'], tag=workflow_phase + '.', disk=sum(f.size for f in vg_ids) *4, - memory=index_mem, cores=options.indexCores) + memory=index_mem, cores=options.indexCores, + walltime=scaled_walltime(ODGI_SQUEEZE_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(odgi_job.rv()) # optional viz @@ -1051,7 +1266,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, viz_job = gfa_root_job.addChildJobFn(make_odgi_viz, config, options, vg_path, og_id, tag=workflow_phase, viz=do_viz, draw=do_draw, cores=options.indexCores, disk = input_vg_id.size * 10, - memory=min(max(og_min_size, input_vg_id.size * 32), max_mem)) + memory=min(max(og_min_size, input_vg_id.size * 32), max_mem), + walltime=scaled_walltime(ODGI_DRAW_SECS_PER_GB if do_draw + else ODGI_VIZ_SECS_PER_GB, + input_vg_id.size)) else: viz_job = None if do_viz: @@ -1067,12 +1285,17 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, snarl_stats_job = gfa_root_job.addChildJobFn(snarl_stats, options, config, vg_path, vg_id, disk=input_vg_id.size * 2, memory=cactus_clamp_memory(input_vg_id.size * 10), - cores=options.indexCores) + cores=options.indexCores, + walltime=scaled_walltime(SNARL_STATS_SECS_PER_GB, + input_vg_id.size)) snarl_stats_ids.append(snarl_stats_job.rv()) snarl_stats_merge_job = gfa_root_job.addFollowOnJobFn(merge_snarl_stats, options.vg, snarl_stats_ids, tag=workflow_phase + '.', - disk=sum(f.size for f in vg_ids) * 2) + disk=sum(f.size for f in vg_ids) * 2, + walltime=scaled_walltime(MERGE_CHROM_TSV_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.1)) out_dicts.append(snarl_stats_merge_job.rv()) # collect this phase's merged GFA; a single combined panacus report is built after the loop @@ -1089,7 +1312,11 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, panacus_job = Job.wrapJobFn(run_panacus, options, config, panacus_phase_dicts, cores=options.indexCores, disk=sum(f.size for f in vg_ids) * 6 * len(panacus_phase_dicts), - memory=index_mem) + memory=index_mem, + # every phase is a full decompress-and-scan of its own merged GFA + walltime=scaled_walltime(PANACUS_SECS_PER_GB * len(panacus_phase_dicts), + sum(f.size for f in vg_ids), + io_multiple=0.5 * len(panacus_phase_dicts))) for merge_job in panacus_merge_jobs: merge_job.addFollowOn(panacus_job) out_dicts.append(panacus_job.rv()) @@ -1121,7 +1348,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, gref_sample = gref_sample_name(vcf_ref) # augment each chromosome in parallel - gref_root_job = Job() + gref_root_job = Job(walltime=cactus_walltime()) gref_source_root_job.addFollowOn(gref_root_job) gref_vg_ids = [] @@ -1130,7 +1357,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, gref_job = gref_root_job.addChildJobFn(compute_gref_paths, config, options, vg_path, vg_id, vcf_ref, disk=input_vg_id.size * 20, memory=max(2**31, min(input_vg_id.size * 20, max_mem)), - cores=options.indexCores) + cores=options.indexCores, + walltime=scaled_walltime(COMPUTE_GREF_PATHS_SECS_PER_GB, + input_vg_id.size)) gref_vg_ids.append(gref_job.rv(0)) gref_segs_ids.append(gref_job.rv(1)) @@ -1142,7 +1371,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, gref_fasta_job = gref_root_job.addFollowOnJobFn(extract_vg_fasta, options, gref_vg_ids, vcf_ref=gref_sample, disk=sum(f.size for f in vg_ids) * 2, - memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4)) + memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4), + walltime=scaled_walltime(EXTRACT_VG_FASTA_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=1)) gref_fasta_dict = gref_fasta_job.rv() gref_parent_job = gref_fasta_job @@ -1158,7 +1390,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # merge gref segments gref_segs_merge_job = gref_root_job.addFollowOnJobFn(merge_gref_segs, options.vg, gref_segs_ids, - disk=sum(f.size for f in vg_ids)) + disk=sum(f.size for f in vg_ids), + walltime=scaled_walltime(MERGE_CHROM_TSV_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.1)) out_dicts.append(gref_segs_merge_job.rv()) # All of the exclusion work hangs off one barrier, and that barrier is a descendant of @@ -1169,7 +1404,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, exclusion_ids = None if do_exclusions and workflow_phases: deepest_phase, deepest_vg_ids, deepest_root_job = workflow_phases[-1] - excl_root_job = Job() + excl_root_job = Job(walltime=cactus_walltime()) deepest_root_job.addFollowOn(excl_root_job) exclusion_coverage = {} @@ -1181,7 +1416,8 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, cov_job = excl_root_job.addChildJobFn( path_coverage_job, config, vg_path, phase_vg_id, chrom_name, workflow_phase, disk=input_vg_id.size * 3, - memory=cactus_clamp_memory(min(max(2**31, input_vg_id.size * 6), max_mem))) + memory=cactus_clamp_memory(min(max(2**31, input_vg_id.size * 6), max_mem)), + walltime=scaled_walltime(PATH_COVERAGE_SECS_PER_GB, input_vg_id.size)) phase_coverage.append((cov_job.rv(0), cov_job.rv(1), cov_job.rv(2), cov_job.rv(3))) exclusion_coverage[workflow_phase] = phase_coverage @@ -1206,7 +1442,8 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # with ~30% headroom while 16Gi covers the walk to a 324Mbp reference contig. # a longer contig than that still gets there through --doubleMem memory=cactus_clamp_memory(min(16 * 2**30 + int(input_vg_id.size * 1.75), - max_mem))) + max_mem)), + walltime=scaled_walltime(REF_GAPS_SECS_PER_GB, input_vg_id.size)) gap_ids.append(gap_job.rv()) exclusion_refgap_ids[ref_event] = gap_ids @@ -1218,7 +1455,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, compute_exclusions_job, config, options, exclusion_coverage, contig_sizes_id, split_log_id, exclusion_refgap_ids, chrom_names, disk=sum(f.size for f in vg_ids) * 2, - memory=cactus_clamp_memory(min(max(2**31, max(f.size for f in vg_ids) * 16), max_mem))) + memory=cactus_clamp_memory(min(max(2**31, max(f.size for f in vg_ids) * 16), max_mem)), + walltime=scaled_walltime(COMPUTE_EXCLUSIONS_SECS_PER_GB, sum(f.size for f in vg_ids), + io_multiple=0.25)) exclusion_ids = exclusion_job.rv() return (output_full_vg_ids, clip_vg_ids, clipped_stats, filter_vg_ids, out_dicts, og_chrom_ids, @@ -1698,7 +1937,7 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r ones. it never makes a wave VCF: -L has already merged away the alleles vcfwave would realign """ - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) if vcftag is None: vcftag = vcf_ref + '.' + workflow_phase if vcf_ref != options.reference[0] else workflow_phase @@ -1713,7 +1952,8 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r tag=os.path.splitext(os.path.basename(vg_path))[0] + '.' + vcftag + '.', cores=options.indexCores, disk = vg_id.size * 6, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(DECONSTRUCT_SECS_PER_GB, vg_id.size)) raw_vcf_id, raw_tbi_id = deconstruct_job.rv(0), deconstruct_job.rv(1) raw_vcf_tbi_ids.append((raw_vcf_id, raw_tbi_id)) @@ -1725,7 +1965,9 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r ref_fasta_dict, tag=os.path.splitext(os.path.basename(vg_path))[0] + '.' + vcftag + '.', disk = vg_id.size * 6, - memory=cactus_clamp_memory(vg_id.size * 2)) + memory=cactus_clamp_memory(vg_id.size * 2), + walltime=scaled_walltime(VCFBUB_SECS_PER_GB, vg_id.size, + io_multiple=0.5)) bub_vcf_id, bub_tbi_id = vcfbub_job.rv(0), vcfbub_job.rv(1) bub_vcf_tbi_ids.append((bub_vcf_id, bub_tbi_id)) @@ -1737,7 +1979,9 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r tag=os.path.splitext(os.path.basename(vg_path))[0] + '.' + vcftag + '.', cores=options.vcfwaveCores, disk=vg_id.size * 6, - memory=cactus_clamp_memory(options.vcfwaveMemory)) + memory=cactus_clamp_memory(options.vcfwaveMemory), + walltime=scaled_walltime(CHUNKED_VCFWAVE_SECS_PER_GB, + vg_id.size, io_multiple=0.5)) wave_vcf_id, wave_tbi_id = vcfwave_job.rv(0), vcfwave_job.rv(1) wave_vcf_tbi_ids.append((wave_vcf_id, wave_tbi_id)) @@ -1751,7 +1995,10 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r fix_ploidies=True, cores = cat_cores, disk = sum(f.size for f in vg_ids) * 26, - memory = cactus_clamp_memory(sum(f.size for f in vg_ids))) + memory = cactus_clamp_memory(sum(f.size for f in vg_ids)), + walltime=scaled_walltime(VCF_CAT_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.125)) out_dict = {'{}.raw.vcf.gz'.format(vcftag) : merge_vcf_job.rv(0), '{}.raw.vcf.gz.tbi'.format(vcftag) : merge_vcf_job.rv(1) } if bub_vcf_tbi_ids: @@ -1759,7 +2006,10 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r fix_ploidies=True, cores = cat_cores, disk = sum(f.size for f in vg_ids) * 26, - memory = cactus_clamp_memory(sum(f.size for f in vg_ids))) + memory = cactus_clamp_memory(sum(f.size for f in vg_ids)), + walltime=scaled_walltime(VCF_CAT_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.125)) out_dict['{}.vcf.gz'.format(vcftag)] = merge_bub_job.rv(0) out_dict['{}.vcf.gz.tbi'.format(vcftag)] = merge_bub_job.rv(1) if wave_vcf_tbi_ids: @@ -1767,7 +2017,10 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r fix_ploidies=True, cores = cat_cores, disk = sum(f.size for f in vg_ids) * 26, - memory = cactus_clamp_memory(sum(f.size for f in vg_ids))) + memory = cactus_clamp_memory(sum(f.size for f in vg_ids)), + walltime=scaled_walltime(VCF_CAT_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.125)) out_dict['{}.wave.vcf.gz'.format(vcftag)] = merge_wave_job.rv(0) out_dict['{}.wave.vcf.gz.tbi'.format(vcftag)] = merge_wave_job.rv(1) @@ -1949,7 +2202,9 @@ def vcfbub(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_allele, fasta if fasta_ref_dict is not None and \ getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap_join"), "bcftoolsNorm", typeFn=bool, default=False): norm_job = job.addChildJobFn(vcfnorm, config, vcf_ref, bub_vcf_id, vcfbub_path, bub_tbi_id, fasta_ref_dict, - disk=bub_vcf_id.size * 6) + disk=bub_vcf_id.size * 6, + walltime=scaled_walltime(VCFNORM_BUB_SECS_PER_GB, bub_vcf_id.size, + io_multiple=6)) return norm_job.rv() else: return bub_vcf_id, bub_tbi_id @@ -2224,7 +2479,7 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle chunk_paths = [vcfbub_path] # distribute on the chunks - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) chunk_vcf_tbi_ids = [] for chunk_path in chunk_paths: @@ -2237,7 +2492,10 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle wave_mem = min(job.memory, max(6 * 2**30, job.cores * 960 * 2**20, chunk_id.size * 800)) vcfwave_job = root_job.addChildJobFn(vcfwave, config, chunk_path, chunk_id, disk=chunk_id.size * 10, cores=job.cores, - memory=wave_mem) + memory=wave_mem, + walltime=cactus_walltime( + VCFWAVE_SECS_PER_MB * chunk_id.size / 1e6 / job.cores, + io_bytes=chunk_id.size * 4)) chunk_vcf_tbi_ids.append(vcfwave_job.rv()) # combine the chunks @@ -2246,7 +2504,9 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle vcfwave_cat_job = root_job.addFollowOnJobFn(vcf_cat, chunk_vcf_tbi_ids, tag, sort=True, fix_ploidies=False, disk=vcf_id.size * 10, - memory=cactus_clamp_memory(vcf_id.size*5)) + memory=cactus_clamp_memory(vcf_id.size*5), + walltime=scaled_walltime(VCF_CAT_CHUNK_SECS_PER_GB, vcf_id.size, + io_multiple=4)) # normalize the output if fasta_ref_dict is not None and \ @@ -2256,7 +2516,9 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle norm_job = vcfwave_cat_job.addFollowOnJobFn(vcfnorm, config, vcf_ref, vcfwave_cat_job.rv(0), vcfwave_path, vcfwave_cat_job.rv(1), fasta_ref_dict, disk=vcf_id.size*12, - memory=cactus_clamp_memory(vcf_id.size*5)) + memory=cactus_clamp_memory(vcf_id.size*5), + walltime=scaled_walltime(VCFNORM_WAVE_SECS_PER_GB, vcf_id.size, + io_multiple=6)) return norm_job.rv() else: return vcfwave_cat_job.rv() @@ -2522,13 +2784,15 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, out_dicts = [] # per-chromosome GFA conversion - gfa_root_job = Job() + gfa_root_job = Job(walltime=cactus_walltime()) parent_job.addFollowOn(gfa_root_job) gfa_ids = [] for vg_path, vg_id, input_vg_id in zip(options.vg, phase_vg_ids, vg_ids): gfa_job = gfa_root_job.addChildJobFn(vg_to_gfa, options, config, vg_path, vg_id, disk=input_vg_id.size * 10, - memory=min(max(2**31, input_vg_id.size * 16), max_mem)) + memory=min(max(2**31, input_vg_id.size * 16), max_mem), + walltime=scaled_walltime(VG_TO_GFA_SECS_PER_GB, + input_vg_id.size, io_multiple=5)) gfa_ids.append(gfa_job.rv()) # GBZ from merged GFA. the gref graph reuses the base graph's snarls (identical topology), @@ -2538,7 +2802,9 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, ref_samples=(options.reference + [vcf_ref]) if is_gref else None, cores=options.indexCores, disk=sum(f.size for f in vg_ids) * 6, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(MAKE_VG_INDEXES_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(gbz_job.rv()) index_dict = gbz_job.rv() @@ -2556,7 +2822,7 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, l_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, tag.rstrip('.'), index_mem, vcf_ref, phase_vg_ids, ref_fasta_dict, vcftag=vcftag, - decon_L=decon_L) + decon_L=decon_L, walltime=cactus_walltime()) out_dicts.append(l_job.rv()) # optional haplo index @@ -2567,7 +2833,10 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, disk=sum(f.size for f in vg_ids) * 16, # see the other make_haplo_index call site memory=cactus_clamp_memory( - min(int(index_mem * 1.5), 128 * 2**30))) + min(int(index_mem * 1.5), 128 * 2**30)), + walltime=scaled_walltime(MAKE_HAPLO_INDEX_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.5)) out_dicts.append(haplo_job.rv()) return out_dicts diff --git a/src/cactus/refmap/cactus_graphmap_split.py b/src/cactus/refmap/cactus_graphmap_split.py index 61a4238d5..c1b7f3980 100644 --- a/src/cactus/refmap/cactus_graphmap_split.py +++ b/src/cactus/refmap/cactus_graphmap_split.py @@ -29,6 +29,7 @@ from cactus.preprocessor.fileMasking import get_mask_bed_from_fasta from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.refmap.cactus_graphmap import filter_paf, apply_mgsplit_filter_overrides +from cactus.refmap.cactus_graphmap import FILTER_PAF_SECS_PER_GB, GFA_RENAME_SECS_PER_GB, RAW_BYTES_PER_GZ_BYTE from cactus.refmap.cactus_minigraph import check_sample_names, minigraph_gfa_from_pansn from toil.job import Job from toil.common import Toil @@ -181,10 +182,16 @@ def cactus_graphmap_split(options): #export the split data export_split_data(toil, wf_output[0], wf_output[1], wf_output[2], wf_output[3], options.outDir, config) +# Seconds per GB of GFA-plus-PAF for rgfa-split: 1220s to split the 8.3 GiB GFA and 34.4 GiB PAF +# of the HPRC v2.0 whole panel, ie ~27 s/GB, with the per-chromosome sed renames that follow adding +# ~130s more. This is the heaviest job in cactus-graphmap-split, and separate_ref_contigs runs the +# same binary on the same kind of input. +RGFA_SPLIT_SECS_PER_GB = 32 + def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_id, gfa_path, paf_id, paf_path, sanitize=True, pansn_gfa_input=True): - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # can be a list coming in from cactus-pangenome, but we only need first item @@ -207,12 +214,17 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() else: - sanitize_job = Job() + sanitize_job = Job(walltime=cactus_walltime()) root_job.addChild(sanitize_job) # auto-set --refContigs if not options.refContigs: - refcontig_job = sanitize_job.addFollowOnJobFn(detect_ref_contigs, config, options, seq_id_map) + # samtools faidx of a whole reference is 21s measured; the rest is downloading the fasta, + # whose size is not in scope here (seq_id_map is a promise), so it goes in the constant -- + # 600s is a ~3 GB reference downloaded on a badly contended filesystem, and a bigger one + # still fits inside the safety factor + refcontig_job = sanitize_job.addFollowOnJobFn(detect_ref_contigs, config, options, seq_id_map, + walltime=cactus_walltime(600)) ref_contigs = refcontig_job.rv() options.otherContig = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap_split"), "otherContigName", typeFn=str, default="chrOther") sanitize_job = refcontig_job @@ -225,24 +237,33 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ # the renaming pass decompresses the GFA before bgzipping it back up, so it needs room for # the raw copy (reckoned at 10x, as below) on top of the compressed input and output rename_gfa_job = root_job.addChildJobFn(minigraph_gfa_from_pansn, genome_names, gfa_path, gfa_id, - disk=gfa_size*12) - new_root_job = Job() + disk=gfa_size*12, + walltime=cactus_walltime(GFA_RENAME_SECS_PER_GB * gfa_size / 1e9, + io_bytes=2*gfa_size)) + new_root_job = Job(walltime=cactus_walltime()) root_job.addFollowOn(new_root_job) root_job = new_root_job gfa_id = rename_gfa_job.rv() # use file extension to sniff out compressed input if gfa_path.endswith(".gz"): - gfa_id = root_job.addChildJobFn(unzip_gz, gfa_path, gfa_id, delete_original=False, disk=gfa_size * 10).rv() + # gunzip is fast (27s for the 0.83 GiB compressed whole-panel GFA); the cost is writing the + # ~10x bigger raw file back to the jobstore, which is what io_bytes is counting + gfa_id = root_job.addChildJobFn(unzip_gz, gfa_path, gfa_id, delete_original=False, disk=gfa_size * 10, + walltime=cactus_walltime(60, io_bytes=(1 + RAW_BYTES_PER_GZ_BYTE) * gfa_size)).rv() gfa_size *= 10 if paf_path.endswith(".gz"): - paf_id = root_job.addChildJobFn(unzip_gz, paf_path, paf_id, delete_original=False, disk=paf_id.size * 10).rv() + # the same, but on a PAF that reaches 34 GiB raw at panel scale + paf_id = root_job.addChildJobFn(unzip_gz, paf_path, paf_id, delete_original=False, disk=paf_id.size * 10, + walltime=cactus_walltime(120, io_bytes=(1 + RAW_BYTES_PER_GZ_BYTE) * paf_size)).rv() paf_size *= 10 # do some basic paf filtering paf_filter_mem = max(paf_id.size * 10, 2**32) paf_filter_job = root_job.addFollowOnJobFn(filter_paf, paf_id, config, reference=options.reference, - disk = paf_id.size * 10, memory=cactus_clamp_memory(paf_filter_mem)) + disk = paf_id.size * 10, memory=cactus_clamp_memory(paf_filter_mem), + walltime=cactus_walltime(60 + FILTER_PAF_SECS_PER_GB * paf_size / 1e9, + io_bytes=2*paf_size)) paf_id = paf_filter_job.rv() root_job = paf_filter_job @@ -254,7 +275,9 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ split_gfa_job = root_job.addFollowOnJobFn(split_gfa, config, gfa_id, [paf_id], ref_contigs, options.otherContig, options.reference, mask_bed_id, disk=(gfa_size + paf_size) * 5, - memory=cactus_clamp_memory((gfa_size + paf_size) * 3)) + memory=cactus_clamp_memory((gfa_size + paf_size) * 3), + walltime=cactus_walltime(120 + RGFA_SPLIT_SECS_PER_GB * (gfa_size + paf_size) / 1e9, + io_bytes=2 * (gfa_size + paf_size))) # use the output of the above splitting to do the fasta splitting split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0), walltime=cactus_walltime()) @@ -333,8 +356,14 @@ def get_mask_bed(job, seq_id_map, min_length): for event in seq_id_map.keys(): fa_id = seq_id_map[event] fa_path = '{}.fa'.format(event) - beds.append(job.addChildJobFn(get_mask_bed_from_fasta, event, fa_id, fa_path, min_length, disk=fa_id.size * 5).rv()) - return job.addFollowOnJobFn(cat_beds, beds).rv() + # cactus_softmask2hardmask scanning a vertebrate genome is p99 51s / max 89s over the 625 + # runs of the VGP 577-way; the rest is the fasta in and the BED out + beds.append(job.addChildJobFn(get_mask_bed_from_fasta, event, fa_id, fa_path, min_length, disk=fa_id.size * 5, + walltime=cactus_walltime(120, io_bytes=2*fa_id.size)).rv()) + # one BED per genome -- order tens of GiB across a whole panel -- concatenated and written back. + # bed_ids are promises and no fasta size is in scope either, so the whole cost has to be a + # constant: 1200s is ~30 GiB of round-trip staging at a pessimistic 30 MB/s + return job.addFollowOnJobFn(cat_beds, beds, walltime=cactus_walltime(1200)).rv() def cat_beds(job, bed_ids): in_beds = [job.fileStore.readGlobalFile(bed_id) for bed_id in bed_ids] @@ -506,7 +535,9 @@ def separate_ref_contigs_batch(job, config, graphmap_input_dict, graphmap_batch_ tot_size = gfa_id.size * 10 + paf_id.size output_dict[chrom] = job.addChildJobFn(separate_ref_contigs, config, chrom, gfa_id, gm_result, reference_event, disk=tot_size * 5, - memory=cactus_clamp_memory(tot_size * 3)).rv() + memory=cactus_clamp_memory(tot_size * 3), + walltime=cactus_walltime(120 + RGFA_SPLIT_SECS_PER_GB * tot_size / 1e9, + io_bytes=tot_size)).rv() return output_dict def separate_ref_contigs(job, config, chrom, gfa_id, gm_result, reference_event): @@ -575,13 +606,20 @@ def line_count(path): return (job.fileStore.writeGlobalFile(separated_paf_path),) + tuple(gm_result[1:]) + \ (job.fileStore.writeGlobalFile(log_path),) +# Seconds per reference contig for split_fa_into_contigs. Grouping the HPRC toil-rt lines by job +# gives ~330s per job for the 23-26 reference contigs of a CHM13/GRCh38 split, with a ceiling of +# 396s across 465 jobs in each of the two runs, ie ~13 s/contig. The contig count is the driver +# rather than the fasta size because each contig costs its own faidx and bgzip, and defaults to 128 -- five times the human case. +SPLIT_FA_SECS_PER_REF_CONTIG = 15 + def split_fas(job, seq_id_map, seq_name_map, split_id_map): """ Use samtools to split a bunch of fasta files into reference contigs, using the output of rgfa-split as a guide""" if (seq_id_map, split_id_map) == (None, None): return None - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # map event name to dict of contgs. ex fa_contigs["CHM13"]["chr13"] = file_id @@ -596,7 +634,9 @@ def split_fas(job, seq_id_map, seq_name_map, split_id_map): if fa_id.size: split_job = root_job.addChildJobFn(split_fa_into_contigs, event, fa_id, fa_path, split_id_map, strip_prefix=False, - disk=fa_id.size * 3) + disk=fa_id.size * 3, + walltime=cactus_walltime(60 + SPLIT_FA_SECS_PER_REF_CONTIG * len(split_id_map), + io_bytes=2*fa_id.size)) fa_contigs[event] = split_job.rv(0) fa_contig_sizes[event] = split_job.rv(1) diff --git a/src/cactus/refmap/cactus_minigraph.py b/src/cactus/refmap/cactus_minigraph.py index ff7aed0cf..3f1f1da57 100644 --- a/src/cactus/refmap/cactus_minigraph.py +++ b/src/cactus/refmap/cactus_minigraph.py @@ -287,7 +287,7 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde sanitized_seq_id_map = sanitize_job.rv() else: sanitized_seq_id_map = seq_id_map - sanitize_job = Job() + sanitize_job = Job(walltime=cactus_walltime()) job.addChild(sanitize_job) xml_node = findRequiredNode(config_node, "graphmap") sort_type = getOptionalAttrib(xml_node, "minigraphSortInput", str, default=None) @@ -302,25 +302,49 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde if options.lastTrain and len(seq_id_map) > 1: # note: somehow last training memory overruns don't seem to be detected by slurm so we # give 12G at least whenever possible, as --doubleMem won't help... + # lastdb dominates the runtime and is erratic: over the 24 per-chromosome runs of the + # HPRC v2.1 pangenome (2.0e8-byte reference, 8 cores) it took 55 s at the p50 but + # 3934 s at the worst, while the 16x bigger whole-genome reference of HPRC v2.0 + # (32 cores) took 2821 s. last-train itself adds 409-680 s on top. So the estimate is + # mostly a flat allowance for that tail, with a small linear term so that small inputs + # aren't over-provisioned. The I/O is the reference plus the training partner, which + # last_train() only picks inside the job but constrains to at least half the reference. last_train_job = prev_job.addFollowOnJobFn(last_train, config_node, seq_order, sanitized_seq_id_map, cores=options.mgCores, disk=8*ref_size, - memory=cactus_clamp_memory(max(8*ref_size, 12*10**9))) + memory=cactus_clamp_memory(max(8*ref_size, 12*10**9)), + walltime=cactus_walltime(3000 + ref_size / 1e6, + io_bytes=3 * ref_size)) train_id = last_train_job.rv() return minigraph_job.rv(0), minigraph_job.rv(1), train_id +# Bytes of reference fasta `mash sketch` gets through per second. It took 225 s on the whole +# 3.15e9-byte CHM13 of HPRC v2.0 (1.4e7 B/s) and a p50 of 3.4 s on the ~1.3e8-byte chromosome +# references of HPRC v2.1 (3.9e7 B/s); this is the slow end of that. +MASH_SKETCH_BYTES_PER_SEC = 1e7 + +# Bytes of query fasta one mash_dist job gets through per second. `mash dist` alone runs at +# 7e6-1.9e7 B/s (694 calls over the 232 mash_dist jobs of HPRC v2.0: p50 168 s, p99 429 s for a +# 3.1e9-byte haplotype), but the job also concatenates the sample's haplotypes and counts every +# base of each with Bio.SeqIO.parse, neither of which shows up in the logs as its own command. +# Budgeting those two at ~2e7 and ~2e8 B/s respectively lands the whole job here. +MASH_DIST_BYTES_PER_SEC = 3e6 + def sort_minigraph_input_with_mash(job, options, config_node, seq_id_map, seq_order): """ Sort the input """ # (dist, length) pairs which will be sorted decreasing on dist, breaking ties with increasing on length # assumption : reference is first mash_dists = [(0, sys.maxsize)] # start by sketching the reference to avoid a bunch of recomputation + ref_bytes = seq_id_map[seq_order[0]].size sketch_job = job.addChildJobFn(mash_sketch, seq_order[0], seq_id_map, - disk = seq_id_map[seq_order[0]].size * 2) + disk = ref_bytes * 2, + walltime=cactus_walltime(ref_bytes / MASH_SKETCH_BYTES_PER_SEC, + io_bytes=ref_bytes)) ref_sketch_id = sketch_job.rv() - dist_root_job = Job() + dist_root_job = Job(walltime=cactus_walltime()) sketch_job.addFollowOn(dist_root_job) xml_node = findRequiredNode(config_node, "graphmap") @@ -342,8 +366,11 @@ def sort_minigraph_input_with_mash(job, options, config_node, seq_id_map, seq_or # list of dictionary (promises) that map genome name to mash distance output dist_maps = [] for sample, names in seq_by_sample.items(): + sample_bytes = sum(seq_id_map[x].size for x in names) dist_map = dist_root_job.addChildJobFn(mash_dist, names, seq_order[0], seq_id_map, ref_sketch_id, - disk = 2 * sum(seq_id_map[x].size for x in names) + seq_id_map[seq_order[0]].size).rv() + disk = 2 * sample_bytes + ref_bytes, + walltime=cactus_walltime(sample_bytes / MASH_DIST_BYTES_PER_SEC, + io_bytes=sample_bytes)).rv() dist_maps.append(dist_map) return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps, walltime=cactus_walltime()).rv() @@ -450,6 +477,21 @@ def mash_distance_order(job, options, config_node, seq_order, mash_output_maps): return mash_order +# Bytes of input fasta `minigraph -xggs` gets through per second per core. From the 237 +# construct commands of the HPRC v2.1 pangenome (8 cores, 50 sequences of up to 2.5e8 bytes per +# batch): p50 14681 s, p99 36241 s, max 41592 s, which is 4.3e4 B/s/core at the p99. The 11 +# whole-genome batches of HPRC v2.0 (32 cores, ~1.5e11 bytes per batch) give 5.9e4 B/s/core at +# their worst, so dividing by the core count is what reconciles two very differently shaped +# runs. (These are not the `minigraph|bgzip` rows of the aggregated evidence, which only catch +# the last batch of each chain -- the rest land in the `minigraph` row alongside the mapping.) +MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE = 4e4 + +# Fixed cost of a construct batch on top of the alignment itself: staging in the previous +# batch's GFA, which is a promise here and so cannot be sized, and -- on the final batch only -- +# the in-python PanSN rename plus its bgzip, which is under 310 s even for the 830 MB +# whole-genome GFA of HPRC v2.0. +MINIGRAPH_CONSTRUCT_OVERHEAD_SECS = 600 + def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_order, gfa_path): """ Make minigraph in sequential batches""" @@ -490,10 +532,14 @@ def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_or else: out_gfa_path = '{}.{}'.format(gfa_path, i) pan_sn_output = False + batch_bytes = sum(seq_id_map[e].size for e in input_seq_order) minigraph_job = Job.wrapJobFn(minigraph_construct, options, config_node, seq_id_map, input_seq_order, out_gfa_path, prev_job.rv() if prev_job else None, prev_gfa_path, pan_sn_output, - disk=disk, memory=mem, cores=options.mgCores) + disk=disk, memory=mem, cores=options.mgCores, + walltime=cactus_walltime(MINIGRAPH_CONSTRUCT_OVERHEAD_SECS + + batch_bytes / (MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE * options.mgCores), + io_bytes=batch_bytes)) if prev_job: prev_job.addFollowOn(minigraph_job) # delete the output of the previous batch from the job store diff --git a/src/cactus/refmap/cactus_pangenome.py b/src/cactus/refmap/cactus_pangenome.py index 6090bb222..e2de28d14 100644 --- a/src/cactus/refmap/cactus_pangenome.py +++ b/src/cactus/refmap/cactus_pangenome.py @@ -505,7 +505,7 @@ def export_join_wrapper(job, options, wf_output, contig_sizes_id=None): def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id): """ chain the entire workflow together, doing exports after each step to mitigate annoyance of failures """ - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) config_node = config_wrapper.xmlRoot @@ -514,7 +514,10 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ # size. The assemblies that went in are the one real size we have here, and since the # pangenome collapses the sequence they share, every one of those outputs is smaller than # their total. So it is used as the bound for the export jobs' I/O below: generous for the - # small ones, but these run once per workflow, and the alternative is guessing. + # small ones, but these run once per workflow, and the alternative is guessing. The two jobs + # that pull the split tree back into the jobstore (import_minigraph_batch_wrapper and + # make_batch_align_jobs_wrapper) move the same sequence in the other direction, so they use it + # too. input_seq_bytes = sum(seq_id.size for seq_id in seq_id_map.values()) # make sure this is done up front @@ -536,7 +539,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ prev_job = sanitize_job.addFollowOnJobFn( contig_sizes_job, seq_id_map, getOptionalAttrib(findRequiredNode(config_node, "graphmap"), "assemblyName", - default="_MINIGRAPH_")) + default="_MINIGRAPH_"), walltime=cactus_walltime()) contig_sizes_id = prev_job.rv() assert type(options.reference) == list @@ -604,7 +607,8 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ wf_output = split_job.rv() split_log_id = split_job.rv(2) split_out_path = os.path.join(options.outDir, 'chrom-subproblems') - split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper, walltime=cactus_walltime()) + split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) chromfile_path = os.path.join(split_out_path, 'chromfile.txt') # clean out some jobstore files we no longer need @@ -616,7 +620,8 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ if options.mgSplit: # rerun cactus_minigraph but on a per-chromosome basis minigraph_batch_import_job = clean_jobstore_job.addFollowOnJobFn(import_minigraph_batch_wrapper, options, config_wrapper, - chromfile_path) + chromfile_path, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) input_seqfiles = minigraph_batch_import_job.rv(0) raw_input_map = minigraph_batch_import_job.rv(1) sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map, walltime=cactus_walltime()) @@ -626,7 +631,8 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ input_map, None, sanitize=False, walltime=cactus_walltime()) minigraph_batch_results = minigraph_batch_job.rv() minigraph_batch_export_job = minigraph_batch_job.addFollowOnJobFn(export_minigraph_batch_wrapper, options, config_node, - input_seqfiles, input_map, minigraph_batch_results, walltime=cactus_walltime()) + input_seqfiles, input_map, minigraph_batch_results, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) # now rerun cactus_graphmap but on a per-chromosome bassis graphmap_input_dict = minigraph_batch_export_job.rv(0) @@ -643,7 +649,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ graphmap_batch_results = separate_job.rv() graphmap_batch_export_job = separate_job.addFollowOnJobFn(export_graphmap_batch_wrapper, options, config_node, graphmap_batch_results, input_seqfiles, - walltime=cactus_walltime()) + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) graphmap_file_ids = graphmap_batch_export_job.rv(0) chromfile_path = graphmap_batch_export_job.rv(1) # clean out the jobstore, as cactus_align reads everything from disk @@ -654,12 +660,14 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ # cactus_align options.scoresFromChromfile = options.lastTrain and options.mgSplit align_jobs_make_job = clean_jobstore_job.addFollowOnJobFn(make_batch_align_jobs_wrapper, options, chromfile_path, config_wrapper, - last_scores_id) + last_scores_id, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) align_jobs = align_jobs_make_job.rv() align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs, walltime=cactus_walltime()) results_dict = align_job.rv() - align_export_job = align_job.addFollowOnJobFn(export_align_wrapper, options, results_dict) + align_export_job = align_job.addFollowOnJobFn(export_align_wrapper, options, results_dict, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) join_options, vg_ids, hal_ids = align_export_job.rv(0), align_export_job.rv(1), align_export_job.rv(2) # cactus_graphmap_join diff --git a/src/cactus/refmap/cactus_panpatch.py b/src/cactus/refmap/cactus_panpatch.py index 5c0449c94..075a7cacb 100644 --- a/src/cactus/refmap/cactus_panpatch.py +++ b/src/cactus/refmap/cactus_panpatch.py @@ -589,6 +589,31 @@ def pangenome_dir(options, run): """ where a run's intermediate cactus-pangenome output goes (deleted on success) """ return os.path.join(options.outDir, run['name'] + '.cactus-scratch') +# Walltime estimates for the jobs below. cactus-panpatch is newer than any run we have logs for, +# so none of these three rates was measured on it: each is the slow end of the nearest thing that +# was measured, and each covers one job per sample or one per chromosome graph. + +# Bytes of input assembly per second for the error-masking pass, which rewrites the whole assembly +# in a pure-python per-line loop -- an order of magnitude slower per byte than the compiled +# cactus_sanitizeFastaHeaders that reads its output. Reckoned against the FileID's size, which is +# the *compressed* size when the input is gzipped even though the loop runs over the ~3x larger +# sequence, so keep it pessimistic. +MASK_FASTA_BYTES_PER_SEC = 4e6 + +# Bytes of chromosome graph per second for one single-threaded panpatch job. panpatch itself takes +# a couple of minutes on a whole human pangenome (see the module docstring); what dominates is +# loading the graph, and it gets loaded twice -- once by the `vg paths -L -x` that lists the samples +# in it, once by panpatch. 20 MB/s is the slow end of the vg graph loads measured on the HPRC v2.1 +# chromosome graphs (`vg paths` p90 165s, `vg paths|awk` p90 471s, on graphs of ~24 GiB). +PANPATCH_BYTES_PER_SEC = 20e6 + +# Bytes of output assembly per second for the gather pass: single-threaded bgzip of each output +# haplotype (21-27 MB/s, measured on the HPRC PAFs -- see GZIP_COMPRESS_BYTES_PER_SEC in +# cactus.shared.common), +# plus two whole-genome python passes over it (the masked-target revert and the dropped-contig +# rescue) plus panpatch's telomere report over each finished fasta. +GATHER_BYTES_PER_SEC = 8e6 + def panpatch_batch_workflow(job, options, config_wrapper, run_inputs, exclude_bed_id): """ patch each sample. the runs are completely independent of each other: they share no input files in the jobstore (see the import loop in main()) """ @@ -668,7 +693,9 @@ def panpatch_run_workflow(job, options, pg_options, config_wrapper, run, seq_id_ for sample, bed_id in mask_bed_ids.items(): mask_job = job.addChildJobFn(mask_assembly_errors, seq_id_map[sample], bed_id, disk=seq_id_map[sample].size * 4 + 2**30, - memory=cactus_clamp_memory(max(2**32, seq_id_map[sample].size * 2))) + memory=cactus_clamp_memory(max(2**32, seq_id_map[sample].size * 2)), + walltime=cactus_walltime(seq_id_map[sample].size / MASK_FASTA_BYTES_PER_SEC, + io_bytes=seq_id_map[sample].size * 4)) masked[sample] = mask_job.rv() # the pangenome runs as a follow-on so any masking children have completed first (their promises @@ -720,7 +747,9 @@ def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_ma mem = min(mem, options.indexMemory) cj = job.addChildJobFn(run_panpatch_chrom, options, run, vg_id, vg_name, exclude_bed_id, target_error_bed_ids, - cores=1, memory=mem, disk=vg_id.size * 4 + 2**30) + cores=1, memory=mem, disk=vg_id.size * 4 + 2**30, + walltime=cactus_walltime(vg_id.size / PANPATCH_BYTES_PER_SEC, + io_bytes=vg_id.size)) chrom_jobs.append(cj) # gather the per-chromosome outputs, rescue dropped contigs, and bgzip. the concatenated fastas @@ -734,7 +763,9 @@ def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_ma gather_job = job.addFollowOnJobFn(gather_panpatch, options, run, [cj.rv() for cj in chrom_jobs], target_fasta_ids, target_error_bed_ids, memory=cactus_clamp_memory(max(2**32, gather_mem)), - disk=gather_disk) + disk=gather_disk, + walltime=cactus_walltime(run['ploidy'] * ref_size / GATHER_BYTES_PER_SEC, + io_bytes=gather_disk)) # this copies the patched assemblies out, and with --keepGraphs every chromosome graph too, # so it is bounded by the same bytes its disk request is export_bytes = sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30 @@ -1145,7 +1176,10 @@ def export_panpatch_wrapper(job, options, run, output_id_map, full_vg_ids, vg_na job.fileStore.exportFile(vg_id, makeURL(os.path.join(chrom_dir, vg_name))) if not options.keepPangenome: - job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run, walltime=cactus_walltime()) + # deleting the scratch tree costs one operation per file (~n_chroms * n_genomes of them), + # not one per byte: seconds of rmtree locally, but a serial paginated delete loop on an + # s3:// outDir, which is why it is off the coordination tier + job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run, walltime=cactus_walltime(600)) def concat_reports(chrom_report_paths, cap_lines, out_path): """ concatenate the per-chromosome panpatch reports (in the order given) into one sample report: diff --git a/src/cactus/refmap/cactus_refmap.py b/src/cactus/refmap/cactus_refmap.py index 06f5a15dc..74c620368 100644 --- a/src/cactus/refmap/cactus_refmap.py +++ b/src/cactus/refmap/cactus_refmap.py @@ -168,6 +168,15 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b """ lead_job = job.addChildJobFn(empty, walltime=cactus_walltime()) + # None of the runs we have logs for exercise cactus-refmap, so the walltimes below are sized + # from the input assemblies rather than measured. minimap2 asm5 is budgeted at 5e5 bytes of + # combined input a second: the whole-genome minigraph mapping rate (1.3e6 B/s at 8 cores) + # halved, since this job asks for no cores and minimap2 defaults to 3 threads. The python + # passes downstream of it are sized off the assembly too -- the pafs they actually read are + # promises here and have no size at scheduling time -- taking a paf to be a fifth of the + # bytes of the assembly that produced it and a line-by-line python pass to run at ~5e7 B/s. + total_bytes = sum(seq_id.size for seq_id in assembly_files.values()) + # map all assemblies to the reference. Don't map reference to reference, though. ref_mappings = dict() secondary_mappings = dict() @@ -176,14 +185,26 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b if assembly != reference: # map to a to b print("about to run map a to b. a:", assembly, job.fileStore.readGlobalFile(assembly_file), "b (ref):", reference, job.fileStore.readGlobalFile(assembly_files[reference])) - map_job = lead_job.addChildJobFn(map_a_to_b, assembly_file, assembly_files[reference], (dipcall_bed_filter or dipcall_vcf_filter)) + pair_bytes = assembly_file.size + assembly_files[reference].size + map_job = lead_job.addChildJobFn(map_a_to_b, assembly_file, assembly_files[reference], (dipcall_bed_filter or dipcall_vcf_filter), + walltime=cactus_walltime(pair_bytes / 5e5, io_bytes=pair_bytes)) ref_mappings[assembly] = map_job.rv() if dipcall_bed_filter: - secondaries_filter_job = map_job.addFollowOnJobFn(filter_out_secondaries_from_paf, ref_mappings[assembly]) + secondaries_filter_job = map_job.addFollowOnJobFn(filter_out_secondaries_from_paf, ref_mappings[assembly], + walltime=cactus_walltime(assembly_file.size / 2.5e8, + io_bytes=2 * assembly_file.size / 5)) primary_paf = secondaries_filter_job.rv() - dipcall_bed_filter_job = secondaries_filter_job.addFollowOnJobFn(apply_dipcall_bed_filter.apply_dipcall_bed_filter, primary_paf) + # the bed filter is not a pass over the paf at all: extract_single_mappings() + # scans every single-mapping region for every large mapping (its own todo at + # the top of apply_dipcall_bed_filter.py), so it is quadratic in the mappings + # of a chromosome, and it print()s the whole region dict into the worker log + # on the way. A rate per byte of assembly is only a stand-in for that, so it + # is a slow one -- ~3100 s for a human assembly. + dipcall_bed_filter_job = secondaries_filter_job.addFollowOnJobFn(apply_dipcall_bed_filter.apply_dipcall_bed_filter, primary_paf, + walltime=cactus_walltime(assembly_file.size / 1e6, + io_bytes=2 * assembly_file.size / 5)) bed_filtered_primary_mappings = dipcall_bed_filter_job.rv() paf_mappings = bed_filtered_primary_mappings else: @@ -195,7 +216,9 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b secondary_mappings[assembly] = None # consolidate the primary mappings into a single file; same for secondary mappings. - all_primary = lead_job.addFollowOnJobFn(consolidate_mappings, primary_mappings).rv() + all_primary = lead_job.addFollowOnJobFn(consolidate_mappings, primary_mappings, + walltime=cactus_walltime(total_bytes / 2.5e8, + io_bytes=2 * total_bytes / 5)).rv() return all_primary def map_a_to_b(job, a, b, dipcall_filter): @@ -339,7 +362,12 @@ def main(): ## Save alignments: if options.dipcall_vcf_filter: # this is substantially less restrictive than the dipcall_bed_filter. - dipcall_filtered = toil.start(Job.wrapJobFn(apply_dipcall_vcf_filter, alignments)) + # variation_length() print()s once per cigar token of every line and the toil worker + # captures all of it, so this job is paced by the printing, not by the parsing: + # reckon on ~1e6 bytes of input a second until that print goes away. + dipcall_filtered = toil.start(Job.wrapJobFn(apply_dipcall_vcf_filter, alignments, + walltime=cactus_walltime(alignments.size / 1e6, + io_bytes=2 * alignments.size))) toil.exportFile(dipcall_filtered, makeURL(options.outputFile)) else: toil.exportFile(alignments, makeURL(options.outputFile)) diff --git a/src/cactus/refmap/pangenome_exclusions.py b/src/cactus/refmap/pangenome_exclusions.py index 4b510a58f..dffd1fc22 100644 --- a/src/cactus/refmap/pangenome_exclusions.py +++ b/src/cactus/refmap/pangenome_exclusions.py @@ -25,7 +25,7 @@ import re import shutil -from cactus.shared.common import cactus_call, getOptionalAttrib, findRequiredNode +from cactus.shared.common import cactus_call, getOptionalAttrib, findRequiredNode, cactus_walltime from toil.realtimeLogger import RealtimeLogger logger = logging.getLogger(__name__) @@ -1134,9 +1134,13 @@ def contig_sizes_job(job, seq_id_map, graph_event): continue safe_event_filename(event) fa_id = seq_id_map[event] + # samtools faidx of a whole genome is 21s measured; the rest is downloading the fasta per_event[event] = job.addChildJobFn(contig_sizes_for_event, fa_id, event, - disk=fa_id.size * 3).rv() - return job.addFollowOnJobFn(merge_contig_sizes, per_event).rv() + disk=fa_id.size * 3, + walltime=cactus_walltime(60, io_bytes=fa_id.size)).rv() + # the rows arrive through promises, so this stages no file at all: it deserialises a few + # million tuples and writes one small gzipped TSV. no size is in scope to key off + return job.addFollowOnJobFn(merge_contig_sizes, per_event, walltime=cactus_walltime(300)).rv() def contig_sizes_for_event(job, fa_id, event): diff --git a/src/cactus/setup/cactus_align.py b/src/cactus/setup/cactus_align.py index 1ae3799e1..14b299525 100644 --- a/src/cactus/setup/cactus_align.py +++ b/src/cactus/setup/cactus_align.py @@ -29,6 +29,8 @@ from cactus.shared.common import write_s3, has_s3, get_aws_region, unzip_gzs, unzip_gz from cactus.shared.common import cactusRootPath from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import unzip_gz_walltime +from cactus.pipeline.cactus_workflow import cons_core_scale from cactus.shared.common import clean_jobstore_files from cactus.shared.version import cactus_commit from cactus.shared.configWrapper import ConfigWrapper @@ -47,6 +49,34 @@ from sonLib.nxnewick import NXNewick from sonLib.bioio import getTempDirectory, getTempFile +# How much bigger a PAF gets when it is unzipped. Measured over the 576 VGP 577-way +# alignments, whose gzipped PAFs decompress to about three times their compressed size. +PAF_GZIP_RATIO = 3 + +# align_toil hosts a whole nested cactus-align on one node. The size term borrows the +# runtime fit's exponent and raises its coefficient for the HAL export and +# optional hal2vg the nested run also pays for; the startup term is nested-Toil startup alone. +ALIGN_TOIL_STARTUP_SECS = 600 +ALIGN_TOIL_SECS_PER_GB = 8000 +ALIGN_TOIL_EXPONENT = 0.78 + +# The pangenome PAF filter, in seconds per GB of PAF. Two sequential passes over every line. +# The python filter loop in filter_paf() runs no subprocess and so is timed in no log; at ~150 +# bytes a line and a split()/int()/float() per line it does ~20 MiB/s, i.e. ~48 s/GB. The +# optional `gaffilter` overlap pass on top of it measures 45 s/GB across the 52 HPRC v2.0 and +# v2.1 chromosomes (1.46 GB in 66.5 s at the median, 34.4 GB in 1475 s at the worst). +FILTER_PAF_SECS_PER_GB = 95 + +# hal2vg, in seconds per GB of HAL. Across the 50 hal2vg jobs of two HPRC pangenomes (177 h +# over 1.1 TB of HAL) the cost per GB is flat: 566 s/GB in aggregate, and the slowest run -- +# 8.5 h -- was at most 830 s/GB against the largest HAL in either run, 36.7 GB. 600 puts the +# largest of them at 1.8x what it took. HAL sizes here are recovered from each job's disk +# request, which is exactly 3x the HAL; the memory request is not usable for this because +# cactus_clamp_memory bounds it, which flattens the small end and manufactures a superlinear +# trend that is not there. The `vg view -g | gzip` that --outGFA adds ran in none of the +# logged runs, so cactus_walltime()'s factor is what covers it. +HAL2VG_SECS_PER_GB = 600 + def main(): parser = Job.Runner.getDefaultArgumentParser() add_cactus_toil_options(parser) @@ -413,15 +443,22 @@ def make_align_job(options, toil, config_wrapper=None, chrom_name=None): def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, paf_id, paf_path, root_name, og_map, checkpointInfo, doVG, doGFA, delay=0, referenceEvents=None, pafMaskFilter=None, paf2Stable=False, cons_cores = None, cons_memory = None, cons_retain_pages = None, do_filter_paf=False, chrom_name=None, scores_id=None, branch_scale=1.0): - head_job = Job() + head_job = Job(walltime=cactus_walltime()) job.addChild(head_job) event_list = input_seq_id_map.keys() - # unzip the PAF if it's gzipped + # unzip the PAF if it's gzipped. Hold on to the size the alignment will actually have once + # unzipped: paf_id becomes a promise below, and a promise has no .size, so everything after + # this point has to size itself from paf_size instead. (filter_paf's disk and memory were + # already reading paf_id.size past this rebinding, so a gzipped PAF with --pangenome raised + # an AttributeError before it ever got as far as scheduling the job.) + paf_size = paf_id.size if paf_path and paf_path.endswith('.gz'): - unzip_job = head_job.addChildJobFn(unzip_gz, paf_path, paf_id, disk=10*paf_id.size) + unzip_job = head_job.addChildJobFn(unzip_gz, paf_path, paf_id, disk=10*paf_id.size, + walltime=unzip_gz_walltime(paf_id.size)) paf_id = unzip_job.rv() + paf_size = int(paf_size * PAF_GZIP_RATIO) # parse the scores file into the config if scores_id: @@ -447,10 +484,12 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # anyway. that is the common path, not just --noSplit. referenceEvents[0] matches the "first # reference" convention cactus-graphmap-split already normalises to. if do_filter_paf: - paf_filter_mem = max(paf_id.size * 10, 2**32) + paf_filter_mem = max(paf_size * 10, 2**32) paf_filter_job = head_job.addChildJobFn(filter_paf, paf_id, config_wrapper, reference=referenceEvents[0] if referenceEvents else None, - disk = paf_id.size * 10, memory=paf_filter_mem) + disk = paf_size * 10, memory=paf_filter_mem, + walltime=cactus_walltime(FILTER_PAF_SECS_PER_GB * paf_size / 1e9, + io_bytes=2 * paf_size)) paf_id = paf_filter_job.rv() # apply tree scaling to reflect branch scaling and/or uncertainty in ancestor placement/sequences @@ -477,18 +516,23 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # run the hal export allow_collapse = getOptionalAttrib(findRequiredNode(config_wrapper.xmlRoot, "graphmap"), "collapse", typeFn=str, default="none") in ['reference', 'all'] + # export_hal called without has_resources stages no file: it resolves the c2h/fasta promises + # into sizes and re-dispatches itself with them, so this pass is coordination. halAppendCactusSubtree + # runs in the resourced pass, which sizes its own walltime. hal_job = cons_job.addFollowOnJobFn(export_hal, sub_tree, config_wrapper.xmlRoot, new_seq_id_map, og_map, results, event=root_name, inMemory=True, checkpointInfo=checkpointInfo, acyclicEvent=referenceEvents[0] if referenceEvents and not allow_collapse else None, - memory_override=cons_memory) + memory_override=cons_memory, walltime=cactus_walltime()) # clean out some of the intermediate jobstore files hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id], walltime=cactus_walltime()) # optionally create the VG if doVG or doGFA: + # as with export_hal above, the pass without a resource_spec only sizes the HAL promise and + # re-dispatches itself; hal2vg runs in the second pass vg_export_job = hal_job.addFollowOnJobFn(export_vg, hal_job.rv(), config_wrapper, doVG, doGFA, referenceEvents, checkpointInfo=checkpointInfo, memory_override=cons_memory, - vg_tag=RAW_VG_SUFFIX if chrom_name else '') + vg_tag=RAW_VG_SUFFIX if chrom_name else '', walltime=cactus_walltime()) vg_file_id, gfa_file_id = vg_export_job.rv(0), vg_export_job.rv(1) else: vg_file_id, gfa_file_id = None, None @@ -515,7 +559,9 @@ def export_vg(job, hal_id, config_wrapper, doVG, doGFA, referenceEvents, checkpo return job.addChildJobFn(export_vg, hal_id, config_wrapper, doVG, doGFA, referenceEvents, checkpointInfo, resource_spec = True, vg_tag=vg_tag, disk=hal_id.size * 3, - memory=cactus_clamp_memory(max(vg_memory, min_memory))).rv() + memory=cactus_clamp_memory(max(vg_memory, min_memory)), + walltime=cactus_walltime(HAL2VG_SECS_PER_GB * hal_id.size / 1e9, + io_bytes=3 * hal_id.size)).rv() work_dir = job.fileStore.getLocalTempDir() hal_path = os.path.join(work_dir, "out.hal") @@ -690,8 +736,22 @@ def align_toil_batch(job, chrom_dict, config_id, options): options.configFile = options.configOverrides[chrom][0] if chrom in options.configOverrides else orig_config config_id = options.configOverrides[chrom][1] if chrom in options.configOverrides else orig_config_id # spawn the chromosome job + # this job hosts an entire nested cactus-align on one node, so its walltime covers that whole + # pipeline -- consolidated, the HAL export, optionally hal2vg -- run one after another with no + # cluster to fan out onto. Only the paf size is in scope (seq_file_id is the seqfile *text*, + # a few hundred bytes), so the estimate keys off it, with the exponent of the + # runtime fit and a coefficient raised over it for the export stages the nested run also pays + # for. The nested run is handed --consCores, so it takes the same core scaling. + # ALIGN_TOIL_STARTUP_SECS is nested-Toil startup only -- deliberately not a floor for the + # work, which would hand an evolver-sized test PAF hours it cannot use. It is the roughest + # estimate here: the seq:paf ratio it assumes is 8x in VGP but 43x in the HPRC pangenome. align_job = job.addChildJobFn(align_toil, chrom, seq_file_id, paf_file_id, config_id, options, - cores=options.alignCores) + cores=options.alignCores, + walltime=cactus_walltime( + ALIGN_TOIL_STARTUP_SECS + + ALIGN_TOIL_SECS_PER_GB * (paf_file_id.size / 1e9) ** ALIGN_TOIL_EXPONENT + * cons_core_scale(options.alignCores), + io_bytes=4 * paf_file_id.size)) results_dict[chrom] = align_job.rv() return results_dict diff --git a/src/cactus/shared/common.py b/src/cactus/shared/common.py index a486aab9c..04799c92d 100644 --- a/src/cactus/shared/common.py +++ b/src/cactus/shared/common.py @@ -1396,6 +1396,25 @@ def enableDumpStack(sig=signal.SIGUSR1): """enable dumping stacks when the specified signal is received""" signal.signal(sig, dumpStacksHandler) +# Single-threaded gzip/bgzip, in bytes of *input* per second. Compression measured at 21-27 +# MB/s on the HPRC PAFs: 1727 s to gzip the 34.4 GiB whole-panel PAF and 868 s to bgzip its +# GAF, against medians of 79 s and 50 s per chromosome. Decompression at ~38 MB/s of +# compressed input, over the 576 VGP alignments' gzipped PAFs. Both are rates over whatever +# gzip is handed, so the compressed side of each pair is the smaller number of bytes. +GZIP_COMPRESS_BYTES_PER_SEC = 21e6 +GZIP_DECOMPRESS_BYTES_PER_SEC = 38e6 + +def unzip_gz_walltime(compressed_bytes): + """ walltime for an unzip_gz job on a file of compressed_bytes. It writes the decompressed + file back to the jobstore, which for a PAF or fasta is roughly 3x what it read. """ + return cactus_walltime(compressed_bytes / GZIP_DECOMPRESS_BYTES_PER_SEC, + io_bytes=4 * compressed_bytes) + +def zip_gz_walltime(uncompressed_bytes): + """ walltime for a zip_gz job on a file of uncompressed_bytes """ + return cactus_walltime(uncompressed_bytes / GZIP_COMPRESS_BYTES_PER_SEC, + io_bytes=2 * uncompressed_bytes) + def unzip_gzs(job, input_paths, input_ids, delete_original=True): """ go through a list of files and unzip any that end with .gz and return a list of updated ids. files that don't end in .gz are just passed through. relying on the extension @@ -1404,7 +1423,8 @@ def unzip_gzs(job, input_paths, input_ids, delete_original=True): for input_path, input_id in zip(input_paths, input_ids): if input_path.endswith('.gz'): unzip_job = job.addChildJobFn(unzip_gz, input_path, input_id, delete_original=delete_original, - disk=10*input_id.size) + disk=10*input_id.size, + walltime=unzip_gz_walltime(input_id.size)) unzipped_ids.append(unzip_job.rv()) else: unzipped_ids.append(input_id) @@ -1433,13 +1453,15 @@ def zip_gzs(job, input_paths, input_ids, list_elems = None, delete_original=True for i, elem in enumerate(input_list): if not list_elems or i in list_elems: output_list.append(job.addChildJobFn(zip_gz, input_path, elem, delete_original=delete_original, - disk=2*elem.size).rv()) + disk=2*elem.size, + walltime=zip_gz_walltime(elem.size)).rv()) else: output_list.append(elem) zipped_ids.append(output_list) else: zipped_ids.append(job.addChildJobFn(zip_gz, input_path, input_list, delete_original=delete_original, - disk=2*input_list.size).rv()) + disk=2*input_list.size, + walltime=zip_gz_walltime(input_list.size)).rv()) else: zipped_ids.append(input_list) return zipped_ids diff --git a/src/cactus/update/cactus_hal2seqfile.py b/src/cactus/update/cactus_hal2seqfile.py index e83fece66..f5bc8815b 100644 --- a/src/cactus/update/cactus_hal2seqfile.py +++ b/src/cactus/update/cactus_hal2seqfile.py @@ -90,16 +90,30 @@ def to_newick(node_id): return to_newick(root_id) + ';', genome_names(root_id) +# Seconds of hal2fasta|bgzip per Gb of extracted genome. Across the 576 per-ancestor exports +# of the VGP 577-way (genomes of 0.007 to 4.3 Gb) the p99 was 57 s/Gb and the worst 70 s/Gb. +HAL2FASTA_SECS_PER_GB = 60 + + def export_subtree_fastas(job, hal_id, hal_name, genomes, lengths): """Fan out one hal2fasta|bgzip job per genome (run in parallel by Toil). Returns a {genome: fasta file id} map.""" fa_ids = {} for genome in genomes: - # the HAL is read by symlink (below), so each job only needs disk for its own FASTA - length = lengths.get(genome, hal_id.size) + # the HAL is read by symlink (below), so each job only needs disk for its own FASTA. + # A genome missing from `lengths` falls back to the whole HAL for disk, which is merely + # wasteful, but as a walltime it would ask for the entire alignment's worth of time for + # one genome -- so an unknown length gets no estimate at all instead. + length = lengths.get(genome) + disk_length = length if length is not None else hal_id.size fa_ids[genome] = job.addChildJobFn(hal2fasta_gz, hal_id, hal_name, genome, memory=cactus_clamp_memory(3000000000), - disk=max(int(length * 2), 2**20)).rv() + disk=max(int(disk_length * 2), 2**20), + # only the bgzipped FASTA is written back; the HAL + # arrives by symlink and is not staged + walltime=cactus_walltime(HAL2FASTA_SECS_PER_GB * length / 1e9, + io_bytes=length // 4) + if length is not None else cactus_walltime(None)).rv() return fa_ids From e8de1383326406bd08774cd62b40ea4dcbbb6a4c Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 10 Sep 2026 09:32:54 -0400 Subject: [PATCH 07/18] Fix what the adversarial pass found in the walltime estimates The worst of it, in cactus-prepare-toil: cactusPrepare() never called cactus_override_toil_options(), the only thing that exports CACTUS_MAX_WALLTIME. Nothing else in the module has needed it before, but the plan now computes walltimes, so every one of them was running with no ceiling on what it could ask a Slurm partition for -- and Toil raises, on the batch system's own thread rather than the job's, when it cannot find a partition that fits. It also meant --walltimeFactor, --minWalltime and --maxWalltime, which this parser advertises, were silently ignored, along with the retryCount and realTimeLogging defaults the rest of Cactus gets. Compounding it: in toil mode the parser is Toil's own, whose --defaultCores defaults to 1, and main() copies that into blastCores/alignCores/preprocessCores. So the default `cactus-prepare-toil --batchSystem slurm` asked 15,000,000 s (173 days) per blast job, unclamped. The estimates now anchor on the p50/p90 rather than the p99 -- cactus_walltime multiplies by 2.5 and --doubleTime takes the tail -- and align and preprocess scale by cores through the same cons_core_scale the consolidated estimator uses, rather than being applied unscaled at their 64-core measurements. A 1-core blast is still a 13-day job; that is honest, and the clamp now bounds what it asks for. Also: guard vcfwave's new per-core divisor against --vcfwaveCores 0; raise merge_pafs to 4x its merged size, since catFiles moves the bytes again locally on top of the jobstore round trip; and give minigraph_map_one a real tail allowance (1500 left 17% over the observed max across 11,390 invocations, which is not one). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- src/cactus/progressive/cactus_prepare.py | 47 +++++++++++++++++------ src/cactus/refmap/cactus_graphmap.py | 15 +++++--- src/cactus/refmap/cactus_graphmap_join.py | 2 +- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/cactus/progressive/cactus_prepare.py b/src/cactus/progressive/cactus_prepare.py index 70715eaa9..42c671443 100644 --- a/src/cactus/progressive/cactus_prepare.py +++ b/src/cactus/progressive/cactus_prepare.py @@ -33,6 +33,7 @@ from cactus.shared.common import makeURL, cactus_call, RoundedJob from cactus.shared.common import write_s3, has_s3, get_aws_region from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options +from cactus.pipeline.cactus_workflow import cons_core_scale from cactus.shared.common import cactus_clamp_memory from cactus.shared.common import cactus_walltime @@ -51,22 +52,38 @@ # runs, 576 cactus-align runs, one cactus-preprocess run over all 577 genomes. Those nested # workflows were given 64 cores. -# Per-ancestor lastz CPU-seconds for cactus-blast: the p99 of the summed command time of those -# 576 blast logs is 12.1M seconds, 99.7% of it lastz, halved for the 2x lastz speedup since. -# lastz is the one command here that is embarrassingly parallel (827k independent jobs in that -# run), so dividing by the cores the nested workflow gets is the right model. -BLAST_CPU_SECS = 6.0e6 +# In toil mode this parser is Toil's own, so --defaultCores is Toil's option and defaults to 1. +# main() copies it into blastCores/alignCores/preprocessCores when they are unset, so the +# default cactus-prepare-toil invocation runs these nested workflows at a single core -- which +# is what the estimates below have to assume unless the user says otherwise. + +def prepare_cores(cores): + """ cores for a nested cactus-* workflow, as a number the estimates can divide by """ + try: + return max(1, int(cores or 1)) + except (TypeError, ValueError): + return 1 + +# Per-ancestor lastz CPU-seconds for cactus-blast: the p50 of the summed command time of those +# 576 blast logs, 99.7% of it lastz, halved for the 2x lastz speedup since. lastz is the one +# command here that is embarrassingly parallel (827k independent jobs in that run), so dividing +# by the cores the nested workflow gets is the right model. The p50 rather than the p99 +# because cactus_walltime multiplies by 2.5 on top and --doubleTime takes the tail: at the p99 +# (12.1M CPU-seconds) the single-core default asks for 173 days, which no partition can run. +BLAST_CPU_SECS = 4.6e5 # cactus-align is cactus_consolidated and little else: it ran p50 10,014 s, p90 34,771 and p99 # 120,138 over those 576 ancestors -- halve for the 2x consolidated speedup -- while everything # else in the phase (the paf ops, export_hal, cactus-hal2fasta) summed to under 1,500 s even at -# the p99. Not scaled by cores: consolidated's parallel part plateaus around 24 of them (see -# ), and the fit's own jobs were well past that. -ALIGN_SECS = 60000 +# the p99. The p90 is the anchor, on the same reasoning as blast. Scaled by cores with the +# same model cactus_cons itself uses, because those 576 jobs ran at 64 cores while the default +# here is one. +ALIGN_SECS = 17400 # One genome through cactus-preprocess. Grouping the 577-way preprocess log by genome gives # p50 2,835 s, p90 7,172, p99 11,420; taking the 3x Red speedup out of the Red component of # each (Red is 88% of the total) leaves p50 1,218, p90 2,859, p99 4,776. +# Red is 88% of that and is multithreaded, so it takes the same core scaling as consolidated. PREPROCESS_SECS = 4800 # Seconds per halAppendSubtree. The 12 cactus-halAppendSubtrees runs of the 577-way did 578 @@ -492,6 +509,14 @@ def cactusPrepare(options): # write the instructions if options.toil: + # this is the one Toil-launching path in the module that never did this, and the plan + # below now depends on it: cactus_override_toil_options is what exports + # CACTUS_MAX_WALLTIME, without which nothing clamps a walltime to what the cluster's + # partitions will actually accept -- and Toil raises, on the batch system's own thread, + # when it cannot find one that fits. It is also what makes --walltimeFactor, + # --minWalltime and --maxWalltime (added to this parser by add_cactus_toil_options) + # mean anything here at all. + cactus_override_toil_options(options) with Toil(options) as toil: if options.restart: toil.restart() @@ -537,7 +562,7 @@ def get_plan(options, inSeqFile, outSeqFile, configWrapper, toil): cores=options.preprocessCores, memory=options.preprocessMemory, disk=options.preprocessDisk, - walltime=cactus_walltime(PREPROCESS_SECS)) + walltime=cactus_walltime(PREPROCESS_SECS * cons_core_scale(prepare_cores(options.preprocessCores)))) else: plan += 'cactus-preprocess {} {} {} --inputNames {} {} {}{}{}{}{}{}\n'.format( get_jobstore(options), options.seqFile, options.outSeqFile, ' '.join(pre_batch), @@ -646,7 +671,7 @@ def cigarPath(event): cores=options.blastCores, memory=options.blastMemory, disk=options.preprocessDisk, - walltime=cactus_walltime(BLAST_CPU_SECS / max(int(options.blastCores or 1), 1))) + walltime=cactus_walltime(BLAST_CPU_SECS / prepare_cores(options.blastCores))) job_idx[("align", event)] = job_idx[("blast", event)].addFollowOnJobFn(toil_call_align, options, outSeqFile, mc_tree, @@ -660,7 +685,7 @@ def cigarPath(event): cores=options.alignCores, memory=options.alignMemory, disk=options.alignDisk, - walltime=cactus_walltime(ALIGN_SECS)) + walltime=cactus_walltime(ALIGN_SECS * cons_core_scale(prepare_cores(options.alignCores)))) else: # todo: support cactus interface (it's easy enough here, but cactus_progressive.py needs changes to handle) cactus_options = options.cactusOptions diff --git a/src/cactus/refmap/cactus_graphmap.py b/src/cactus/refmap/cactus_graphmap.py index a70934e4f..47923c675 100644 --- a/src/cactus/refmap/cactus_graphmap.py +++ b/src/cactus/refmap/cactus_graphmap.py @@ -533,9 +533,10 @@ def make_minigraph_fasta(job, gfa_file_id, gfa_file_path, name): # minigraph mapping, per job and per GB of sanitized fasta. Grouping the HPRC toil-rt lines by # job gives p90 4062s for a whole-genome haplotype (v2.0, n=474, ~3.1 GB of fasta) against a p99 # of ~1200s for a per-chromosome one (v2.1, ~0.12 GB), which is a line at ~1200 s/GB. The -# intercept is raised to 1500 to cover the per-chromosome tail (max 3521s, 3x its own p99), which -# is cluster contention rather than anything the fasta size can see. -MINIGRAPH_MAP_SECS = 1500 +# intercept covers the per-chromosome tail (max 3521s over 11,390 invocations, 3x its own p99), +# which is cluster contention rather than anything the fasta size can see. 2000 leaves ~50% +# over that max once the factor is applied; 1500 left 17%, which is not a tail allowance. +MINIGRAPH_MAP_SECS = 2000 MINIGRAPH_MAP_SECS_PER_GB = 1200 # What a genome's minigraph output weighs as a fraction of the fasta it was mapped from: the v2.0 @@ -684,11 +685,15 @@ def minigraph_map_one(job, config, event_name, fa_file_id, gfa_file_id): def merge_pafs_walltime(merged_bytes, gzip=False): """ walltime for a merge_pafs job whose output comes to roughly merged_bytes. The job is all I/O -- every input is read out of the jobstore and the concatenation written back -- except - with gzip=True, which bgzips the result single-threaded on the way out. """ + with gzip=True, which bgzips the result single-threaded on the way out. + + io_bytes is 4x rather than 2x because the bytes move twice: once staging in and out of the + jobstore, and again locally, where catFiles reads every input back and writes the joined + file. At 2x the whole-panel merge of the HPRC v2.0 PAF came to under half an hour. """ secs = MERGE_PAF_SECS if gzip: secs += merged_bytes / GZIP_COMPRESS_BYTES_PER_SEC - return cactus_walltime(secs, io_bytes=2*merged_bytes) + return cactus_walltime(secs, io_bytes=4*merged_bytes) def merge_pafs(job, paf_file_id_map, gzip=False): """ merge up some pafs """ diff --git a/src/cactus/refmap/cactus_graphmap_join.py b/src/cactus/refmap/cactus_graphmap_join.py index ef3dc0f96..7bcc3c5a3 100644 --- a/src/cactus/refmap/cactus_graphmap_join.py +++ b/src/cactus/refmap/cactus_graphmap_join.py @@ -2494,7 +2494,7 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle disk=chunk_id.size * 10, cores=job.cores, memory=wave_mem, walltime=cactus_walltime( - VCFWAVE_SECS_PER_MB * chunk_id.size / 1e6 / job.cores, + VCFWAVE_SECS_PER_MB * chunk_id.size / 1e6 / max(1, job.cores or 1), io_bytes=chunk_id.size * 4)) chunk_vcf_tbi_ids.append(vcfwave_job.rv()) From 3f899f5e7156159db3da6feb3e36ad12a239ff5c Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 10 Sep 2026 21:14:25 -0400 Subject: [PATCH 08/18] Revert a bogus PreprocessSequence argument that broke every preprocessor job PreprocessSequence.__init__ takes (prepOptions, inSequenceID, chunksToCompute) and already derives its own size from inSequenceID.size on its first line. The inSequenceSize parameter belongs to BatchPreprocessor, further down the same file; I misread one for the other and passed it to the wrong constructor. The result was TypeError: PreprocessSequence.__init__() got an unexpected keyword argument 'inSequenceSize' on every preprocessing job, which failed 16 of 28 evolver tests -- everything that preprocesses a sequence, plus everything downstream of it, since the missing outputs then surfaced as FileNotFoundError. CI job 116490. Neither compileall nor importing the module catches a wrong keyword at a call site, which is why this got through. Every one of the 295 job-scheduling calls in the tree has now been bound against its target's signature with inspect.Signature.bind_partial, and this was the only mismatch. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- src/cactus/preprocessor/cactus_preprocessor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cactus/preprocessor/cactus_preprocessor.py b/src/cactus/preprocessor/cactus_preprocessor.py index 154ecdf6b..15edbe3a6 100644 --- a/src/cactus/preprocessor/cactus_preprocessor.py +++ b/src/cactus/preprocessor/cactus_preprocessor.py @@ -356,10 +356,7 @@ def run(self, fileStore): fileStore.deleteGlobalFile(self.inSequenceID) self.inSequenceID = fileStore.writeGlobalFile(unmaskedInputFile) - # the size has to be handed over separately: PreprocessSequence sets its own walltime - # in __init__, and self.inSequenceID is a promise on every iteration after the first - ppJob = self.addChild(PreprocessSequence(prepOptions, self.inSequenceID, - inSequenceSize=getattr(self.inSequenceID, 'size', None))) + ppJob = self.addChild(PreprocessSequence(prepOptions, self.inSequenceID)) outSeqID = ppJob.rv() # make sure the step only masked/renamed, and did not corrupt the sequence. # this has to run before clean_if_different, which drops the input file. From 115d8c5f5446481551bf18c076b2fc4cc4b33f31 Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Wed, 16 Sep 2026 12:47:53 -0400 Subject: [PATCH 09/18] Raise the walltime floor to 30 minutes and the staging rate to 25 MiB/s From the first at-scale Slurm run with walltimes on (HPRC, GRCh38+CHM13, --doubleTime true). It got through preprocessing with no memory kills and no partition errors -- the ceiling clamped to 7 days off a 14-day partition exactly as intended -- but 60 of its 519 sanitize_fasta_header jobs were killed on time. Every one of them then succeeded on the --doubleTime retry, and none needed a second doubling, so the requirement sat between the ~620 s they asked for and the ~1300 s they were given. The shortfall is staging, not compute: those jobs' own commands measured 68 s and 56 s at the median and 237 s summed at the p99, against a walltime of 620 s. What they were actually doing was moving ~4 GB each through a jobstore with 519 of them on it at once. 100 MiB/s was the first guess for that and it is an order of magnitude out under that load, so it drops to 25. The floor goes to half an hour because a short partition is an hour: every request under that routes identically, so precision below it buys nothing, while being short by a minute costs a whole retry. sanitize_fasta_header now asks 30 minutes against the ~22 it needed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- doc/progressive.md | 2 +- src/cactus/shared/common.py | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/doc/progressive.md b/doc/progressive.md index 5afb8f209..75e0f3670 100644 --- a/doc/progressive.md +++ b/doc/progressive.md @@ -423,7 +423,7 @@ Slurm chooses a job's partition from its time limit, so giving every job one big * `--doubleTime true` is **on by default** (Cactus turns it on the way it raises `--retryCount` to 5). It is the time analogue of `--doubleMem`: if slurm kills a job for running past its walltime, it is retried with double the walltime. This is what makes tight per-job estimates safe -- an occasional underestimate costs one retry, not the run. Pass `--doubleTime false` to turn it off. * `--walltimeFactor` is the safety multiplier applied to every estimate (default `2.5`). Raise it if your cluster is slower than the ones the estimates were measured on and jobs are being killed for running over. Pass `--walltimeFactor 0` to switch per-job walltimes off entirely, which puts every job back on `--defaultWalltime`. -* `--minWalltime` is the floor for any request, in seconds (default `600`). Cactus schedules a great many jobs that finish in well under a second; this keeps worker startup and jobstore I/O from pushing them over their limit. +* `--minWalltime` is the floor for any request, in seconds (default `1800`). A short partition is typically an hour, so every request below that routes the same way and there is nothing to gain by asking for less -- while a request that is short by a minute costs a whole `--doubleTime` retry. Half an hour leaves room for worker startup and for jobstore staging under heavy concurrency, which no size-based estimate can see. * `--maxWalltime` is the ceiling, in seconds. By default, on Slurm, Cactus queries `sinfo` and caps every request at *half* the longest time limit of any partition your jobs could land on. This matters more than it looks: when Toil cannot find a partition that fits a job's walltime it raises rather than falling back to a default, and it does so on the batch system's own thread, taking the whole workflow down rather than just the job. Since `--doubleTime` retries a killed job with twice the walltime, an estimate sitting at the longest partition's limit would turn its own first retry into exactly that crash -- hence the half. A job needing more than half your longest partition still gets there, it just spends one attempt on the way. With `--doubleTime false` there is nothing to retry into and the full limit is used. * `--defaultWalltime` is Toil's fallback for jobs Cactus did not estimate a time for. With the estimates on there is very little left for it to cover, so you can usually leave it alone. * `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU. But this option will allow you to override that. diff --git a/src/cactus/shared/common.py b/src/cactus/shared/common.py index 04799c92d..d1b595a00 100644 --- a/src/cactus/shared/common.py +++ b/src/cactus/shared/common.py @@ -268,21 +268,27 @@ def cactus_clamp_memory(memory_bytes): # bare cactus_walltime() is the right call for a coordination job. WALLTIME_COORDINATION = 120 -# Assumed jobstore throughput, in bytes/second, for the io_bytes term. Staging is what -# makes an otherwise trivial job slow, and a shared cluster filesystem with a few hundred -# concurrent Cactus jobs on it goes nowhere near its headline number. WALLTIME_FACTOR is -# applied on top of this, so the effective worst case is a good deal slower again. -WALLTIME_IO_RATE = 100 * 1024**2 +# Assumed jobstore throughput, in bytes/second, for the io_bytes term. Staging is what makes an +# otherwise trivial job slow, and a shared cluster filesystem with a few hundred concurrent +# Cactus jobs on it goes nowhere near its headline number: in the first at-scale Slurm run with +# walltimes on (HPRC, 519 concurrent sanitize_fasta_header jobs) the ones that overran were +# spending upwards of 900 s moving ~4 GB each, while their own commands took 240 s. 100 MiB/s +# was the first guess and it was an order of magnitude out under that load. +WALLTIME_IO_RATE = 25 * 1024**2 # Multiplier applied to every estimate, overridable with --walltimeFactor. The estimates # at the call sites aim at roughly the p99 of what we have measured on the largest runs we # have logs for, so this is headroom on top of that. WALLTIME_FACTOR = 2.5 -# Floor for any walltime request, overridable with --minWalltime. Nothing is gained by -# asking Slurm for less: worker startup, jobstore round-trips and Slurm's own granularity -# swamp it, and a too-short request just buys a --doubleTime retry. -WALLTIME_MIN = 600 +# Floor for any walltime request, overridable with --minWalltime. A short partition is an hour +# on the clusters this runs on, so every request below that routes the same way and precision +# under it buys nothing -- while a request that is too short by a minute costs a whole +# --doubleTime retry. 600 was too tight for exactly that reason: 60 of the 519 +# sanitize_fasta_header jobs of the run above were killed at ~620 s, and every one then +# succeeded when --doubleTime handed it ~1250 s. Half the short partition leaves room for the +# staging tail that no size model sees, and still lands in it. +WALLTIME_MIN = 1800 # How much of the longest Slurm partition to keep in reserve for --doubleTime to retry into. # 2 leaves room for exactly one doubling; see cactus_override_toil_options. From abebf94b5576f03fedaeb9c2dbcf26eb534e211d Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 07:58:09 -0400 Subject: [PATCH 10/18] Size minigraph construction by Amdahl's law and by the graph already built Two things were wrong with the walltime, and the first at-scale Slurm run showed both. Seven jobs were killed on time, all of them minigraph_construct, and nothing else in the run was. First, it divided the work by the full --mgCores, crediting parallelism minigraph does not deliver. It is not capped at a fixed core count -- it parallelises over query contigs, and the faster fork adds more parallel sections -- but parts of construction stay single-threaded, so it is Amdahl's law and the plateau depends on the data. The serial fraction comes from the CPU factor minigraph reports itself (cputime/elapsed, the `*N` in [M::ggen_map::T*N]), time-weighted over every construct process of the run: 7.96 cores busy at --mgCores 64, i.e. s = 0.112 and an asymptote near nine. Amdahl rather than min(cores, 8) because they agree only at the top: at --mgCores 8 the cap credits 8 effective cores where the curve gives 4.5, so it would have kept under-estimating for anyone not asking for a whole node. Second, every batch was sized as though it were the first. Each one aligns its genomes against everything the batches before it put in the graph, so the same new sequence costs more the later it arrives. Across 23 chromosomes of that run, batch i against batch 0 at the p90: 1.23, 1.58, 1.81, 1.92, 2.08 for i = 1..5, then flat -- saturating rather than compounding, hence the cap. Keyed off the bytes already in the graph rather than the batch index, so the short last batch and the --inGFA seed graph need no special case. The graph itself would predict better but arrives as prev_job.rv(), a promise with no size. Together these cover all 198 chromosome-batches of the run once the walltime factor is applied. At the batch that failed: 1.9 h before, 11.8 h now, against 2.6 h typical and 7.8 h worst observed. Both are fitted to one run, so the asymptote and the growth are measured while the shapes -- Amdahl, and linear-then-flat -- are assumed. chr2 is the one chromosome the growth term does not really explain: its CPU factor collapses from 6.82 to 2.57 across batches where every other chromosome holds steady, so its 4.7x is lost parallelism rather than a bigger graph. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_019aWQPbf73PRcHZkw9YL2gf --- src/cactus/refmap/cactus_minigraph.py | 48 +++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/src/cactus/refmap/cactus_minigraph.py b/src/cactus/refmap/cactus_minigraph.py index 1d690aee1..cd4e40dd6 100644 --- a/src/cactus/refmap/cactus_minigraph.py +++ b/src/cactus/refmap/cactus_minigraph.py @@ -802,12 +802,49 @@ def mash_distance_order(job, options, config_node, seq_order, mash_output_maps, # the last batch of each chain -- the rest land in the `minigraph` row alongside the mapping.) MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE = 4e4 +# ...but not linearly. minigraph parallelises over query contigs, and the faster fork adds more +# parallel sections on top of that, but parts of construction remain single-threaded -- so it is +# Amdahl's law rather than a hard ceiling, and how far it scales depends on the data. Measured +# from the CPU factor minigraph reports itself (cputime/elapsed, the `*N` in its +# [M::ggen_map::T*N] lines), time-weighted over every construct process of an HPRC run: at +# --mgCores 64 on the current fork it averages 7.96 cores busy, which is a serial fraction of +# 0.112 and an asymptote near nine cores. +# +# Fitted at one core count, so the asymptote is measured and the shape is Amdahl's assumption. +# It matters most in the middle: at --mgCores 8 this gives 4.5 effective cores where crediting +# the request in full would give 8, and simply capping at the asymptote would too. +MINIGRAPH_SERIAL_FRACTION = 0.112 + +def minigraph_effective_cores(cores): + """ cores minigraph construction can actually keep busy at this core count, by Amdahl's law """ + cores = max(1, int(cores or 1)) + s = MINIGRAPH_SERIAL_FRACTION + return 1.0 / (s + (1.0 - s) / cores) + # Fixed cost of a construct batch on top of the alignment itself: staging in the previous # batch's GFA (or the seed graph when extending), which is a promise here and so cannot be # sized, and -- on the final batch only -- the in-python PanSN rename plus its bgzip, which is # under 310 s even for the 830 MB whole-genome GFA of HPRC v2.0. MINIGRAPH_CONSTRUCT_OVERHEAD_SECS = 600 +# Each batch aligns its genomes against everything the batches before it already put in the +# graph, so the same amount of new sequence costs more the later it arrives. Measured over 23 +# chromosomes of an HPRC run, as batch i's wall time against batch 0's: the p90 runs 1.23, 1.58, +# 1.81, 1.92, 2.08 for i = 1..5 and then flattens, so the growth saturates rather than +# compounding. Keyed off the bytes already in the graph rather than the batch index, so uneven +# batches and the --inGFA seed graph are handled the same way. +# +# The graph itself is the better predictor, but it reaches this point as prev_job.rv(), a promise +# with no size, so the sequence that went into it is the closest thing in scope. +MINIGRAPH_GRAPH_GROWTH = 0.25 +MINIGRAPH_GRAPH_GROWTH_MAX = 2.5 + +def minigraph_graph_growth(prior_bytes, batch_bytes): + """ how much slower this batch is than the first, for the graph already built ahead of it """ + if batch_bytes <= 0: + return 1.0 + return min(1.0 + MINIGRAPH_GRAPH_GROWTH * (prior_bytes / batch_bytes), MINIGRAPH_GRAPH_GROWTH_MAX) + def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_order, gfa_path, whole_genome_ref=False, seed_gfa_id=None, graph_names=None): """ Make minigraph in sequential batches. @@ -870,14 +907,19 @@ def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_or out_gfa_path = '{}.{}'.format(gfa_path, i) pan_sn_output = False batch_bytes = sum(seq_id_map[e].size for e in input_seq_order) + # everything already in the graph this batch has to align against + prior_bytes = sum(seq_id_map[e].size for e in seq_order[:i * max_batch_size]) minigraph_job = Job.wrapJobFn(minigraph_construct, options, config_node, seq_id_map, input_seq_order, out_gfa_path, prev_job.rv() if prev_job else seed_gfa_id, prev_gfa_path if prev_job else seed_gfa_path, pan_sn_output, graph_names, disk=disk, memory=mem, cores=options.mgCores, - walltime=cactus_walltime(MINIGRAPH_CONSTRUCT_OVERHEAD_SECS + - batch_bytes / (MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE * options.mgCores), - io_bytes=batch_bytes)) + walltime=cactus_walltime( + MINIGRAPH_CONSTRUCT_OVERHEAD_SECS + + batch_bytes * minigraph_graph_growth(prior_bytes, batch_bytes) / + (MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE * + minigraph_effective_cores(options.mgCores)), + io_bytes=batch_bytes)) if prev_job: prev_job.addFollowOn(minigraph_job) # delete the output of the previous batch from the job store From b8f18289f09ecdf6b7668e2e9f4906d8b1b4870e Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 09:18:09 -0400 Subject: [PATCH 11/18] stop the hal export asking for a whole node export_hal took --consMemory verbatim, so a memory figure chosen for bar became the request for a single-threaded halAppendCactusSubtree. On a salamander that meant 1.8 Ti reserved to use 48.7 Gi: a job that can only be placed on the largest node in the cluster, and queues behind whatever is on it. The flag is documented as overriding cactus_consolidated, and the export is not a cactus_consolidated job, so it no longer receives it. The estimate itself was also far too generous. Over 576 VGP alignments halAppendCactusSubtree used a median 13.5% of its request and never more than 23%. Against the term the formula already uses -- the largest subtree's fasta plus its c2h -- actual use is 0.68x at the median, 1.01x at p99 and 1.15x at worst, so the 5x coefficient sat more than four times above anything observed. 2x leaves 1.74x of headroom over that worst case and under-provisions none of the 576, while dropping what they collectively reserve from 34857 to 13950 GiB. --doubleMem retries at twice the request, so a workload outside the envelope recovers at 4x, still below what this replaces. export_vg has the same override pattern and is deliberately left alone: it is a separate estimate and there is no usage data here to justify touching it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- src/cactus/progressive/cactus_progressive.py | 12 ++++++------ src/cactus/setup/cactus_align.py | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/cactus/progressive/cactus_progressive.py b/src/cactus/progressive/cactus_progressive.py index 3ccbb2589..2c4720cb9 100755 --- a/src/cactus/progressive/cactus_progressive.py +++ b/src/cactus/progressive/cactus_progressive.py @@ -208,7 +208,7 @@ def progressive_step_2(job, trimmed_outgroups_and_alignments, options, config_no def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=None, cacheBytes=None, cacheMDC=None, cacheRDC=None, cacheW0=None, chunk=None, inMemory=False, - checkpointInfo=None, acyclicEvent=None, has_resources=False, memory_override=None): + checkpointInfo=None, acyclicEvent=None, has_resources=False): # todo: going through list nonsense because (i think) it helps with promises, should at least clean up work_dir = job.fileStore.getLocalTempDir() @@ -272,10 +272,10 @@ def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=Non if not has_resources: disk = 3 * sum([file_id.size for file_id in fa_file_ids + c2h_file_ids]) - mem = cactus_clamp_memory(5 * (max([file_id.size for file_id in fa_file_ids]) + max([file_id.size for file_id in c2h_file_ids]))) - # allows pass-through of memory override from --consMemory - if memory_override: - mem = memory_override + # 2x the largest subtree's inputs: halAppendCactusSubtree peaked at 1.15x of that across + # 576 VGP alignments, and never used more than 23% of the old 5x. --doubleMem covers a + # workload outside that envelope. + mem = cactus_clamp_memory(2 * (max([file_id.size for file_id in fa_file_ids]) + max([file_id.size for file_id in c2h_file_ids]))) return job.addChildJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, results, event=event, cacheBytes=cacheBytes, cacheMDC=cacheMDC, cacheRDC=cacheRDC, cacheW0=cacheW0, chunk=chunk, inMemory=inMemory, checkpointInfo=checkpointInfo, @@ -326,7 +326,7 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i # then do the hal export hal_export_job = progressive_job.addFollowOnJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, - progressive_job.rv(), event=root_event, memory_override=options.consMemory) + progressive_job.rv(), event=root_event) return hal_export_job.rv() diff --git a/src/cactus/setup/cactus_align.py b/src/cactus/setup/cactus_align.py index 3eb4c5560..dff705a85 100644 --- a/src/cactus/setup/cactus_align.py +++ b/src/cactus/setup/cactus_align.py @@ -477,8 +477,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # run the hal export allow_collapse = getOptionalAttrib(findRequiredNode(config_wrapper.xmlRoot, "graphmap"), "collapse", typeFn=str, default="none") in ['reference', 'all'] hal_job = cons_job.addFollowOnJobFn(export_hal, sub_tree, config_wrapper.xmlRoot, new_seq_id_map, og_map, results, event=root_name, inMemory=True, - checkpointInfo=checkpointInfo, acyclicEvent=referenceEvents[0] if referenceEvents and not allow_collapse else None, - memory_override=cons_memory) + checkpointInfo=checkpointInfo, acyclicEvent=referenceEvents[0] if referenceEvents and not allow_collapse else None) # clean out some of the intermediate jobstore files hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id]) From 90d211999a2fb4a4766c0135e79345503d319c2a Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 12:00:52 -0400 Subject: [PATCH 12/18] size rgfa-collapse from the graph it collapses collapse_inversions asked for 8*ref_size, which is the wrong quantity twice over. In the sep16 HPRC run it resolved to 2 Gi and every one of 22 real chromosomes was OOM-killed; with the estimate working as written it would ask ~180 GiB, the same figure the minigraph jobs get, for a job that needs 6. rgfa-collapse's cost is almost entirely fixed. Across 24 HPRC chromosomes it used 5.9-6.0 GiB of memory and 2.9 GiB of disk while the gfa it was given spanned six thousandfold, 0.03 MB to 190 MB. chrM, at 16 kb, still cost 3.9 GiB. So the floors here are what was measured and the per-byte terms are headroom for graphs larger than any observed -- worth saying plainly, because the coefficients are not calibrated and should not be read as if they were. Sized through a PromisedRequirement so the gfa's real size is used rather than a proxy available at graph-construction time. Toil evaluates these inside a wrapper job on a worker, so the clamp is resolved on the leader and closed over: cactus_clamp_memory reads CACTUS_MAX_MEMORY from the environment, which only the leader has. Nothing here says whether the cost scales with sample count or thread count. The measurements fix the axis of graph size, and that is all they fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- src/cactus/refmap/cactus_minigraph.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/cactus/refmap/cactus_minigraph.py b/src/cactus/refmap/cactus_minigraph.py index e98ee650e..e36a27b85 100644 --- a/src/cactus/refmap/cactus_minigraph.py +++ b/src/cactus/refmap/cactus_minigraph.py @@ -32,6 +32,7 @@ from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.paf.last_scoring import last_train from toil.job import Job +from toil.job import PromisedRequirement from toil.common import Toil from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options @@ -452,11 +453,27 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf uncollapsed_pansn_gfa_id, collapse_report_id = None, None if getOptionalAttrib(xml_node, "collapseInversions", typeFn=bool, default=False) and \ not getattr(options, 'refOnly', False): + # resolved here, not in the lambda: toil evaluates a PromisedRequirement inside a wrapper + # job on a worker, and cactus_clamp_memory reads CACTUS_MAX_MEMORY out of the environment, + # which is only set on the leader. + mem_cap = cactus_clamp_memory(sys.maxsize) collapse_job = minigraph_job.addFollowOnJobFn(collapse_inversions, options, config_node, minigraph_job.rv(1), gfa_path, cores=options.mgCores, - disk=8*ref_size, - memory=cactus_clamp_memory(8*ref_size)) + # sized from the graph it collapses, not the + # reference: rgfa-collapse's cost is almost all + # fixed. Over 24 HPRC chromosomes it used + # 5.9-6.0 GiB and 2.9 GiB disk while the gfa + # spanned 6000x (0.03 MB to 190 MB), so the + # floors are what is measured and the per-byte + # terms are headroom for graphs larger than any + # seen. 8*ref_size asked ~180 GiB for this. + disk=PromisedRequirement( + lambda gfa: max(8 * gfa.size, 8 * 2**30), + minigraph_job.rv(1)), + memory=PromisedRequirement( + lambda gfa: min(max(64 * gfa.size, 12 * 2**30), mem_cap), + minigraph_job.rv(1))) # graph_names, the same set the forward rename uses, and for the same reason its comment # gives: it has to resolve every SN tag in the finished graph, which on the --inGFA extend # path is more genomes than minigraph is given. It is captured above before seq_id_map is From 0a56ea4cea2275752ecd9659ac97dcf6fc4519ed Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 12:20:12 -0400 Subject: [PATCH 13/18] let auto retention keep room to be wrong, and shrink the window for giant genomes Two config settings for cactus_consolidated, both aimed at the case the memory estimate handles worst: a genome far larger than anything the estimate was fitted to. memory_retain_auto_fraction, default 0.5. "auto" used to keep jemalloc's freed pages whenever the retained estimate fit at all, which puts the fast path on a knife edge -- 14% of 576 VGP alignments came in above their estimate, and an under-estimate at the ceiling OOMs a job that has no checkpoint and has to redo the whole alignment. Retention is now only attempted when the retained figure for this job's core count fits inside a fraction of what the job can be given, so a miss still has somewhere to land. Setting it to 1 restores the old behaviour. On salamander Anc3 this turns a 929.8 Gi retained estimate on an 1843 Gi node into retention off and a 371.9 Gi request; that run peaked at 368.7 Gi. partialOrderAlignmentWindowBigGenome / ...Threshold, default 5000 above 10 Gb. Once sequence is long and repeat-rich the poa window is the only thing bounding the DP. Halving it on 22 Gb salamanders took bar's peak from 798.5 to 368.7 GiB and roughly doubled throughput, against 0.06 points of recall on evolver mammals -- measured with mafComparator against the cactus test truth set, where the curve is flat down to a window of 2000 and only degrades below that. The test is the largest ingroup, not the total: an outgroup adds alignment but is not what makes the flowers enormous. The largest per-sequence average across the VGP set is 6.25 GB, so the default leaves those alignments alone. The resolved window is written into the config the cons job is handed, by the same deep copy that already carries retain_pages, because an estimate made for one window and an alignment run at another is worse than either. The memory fit itself is deliberately untouched. Scored against the 576 VGP peaks it under-provisions 14.4% and is still short at 2x on 5, allocating 1.84x what was used; every variant tried trades those 5 for 21 or more. There is no honest improvement available without current-code ground truth at that scale. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- src/cactus/cactus_progressive_config.xml | 28 ++++++++++++++--- src/cactus/pipeline/cactus_workflow.py | 40 +++++++++++++++++++----- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/cactus/cactus_progressive_config.xml b/src/cactus/cactus_progressive_config.xml index 286c42917..380250b95 100644 --- a/src/cactus/cactus_progressive_config.xml +++ b/src/cactus/cactus_progressive_config.xml @@ -301,11 +301,21 @@ + 299 GiB). "1" always, - "0" never, "auto" keeps them unless the retained memory estimate is more than the job can - be given (the system memory on a single machine, or the maxMemory option), in which case the - estimate is divided by memory_retain_ratio and the pages are not kept. The memory fit - above was measured with retention on. The consRetainPages command line option overrides. --> + "0" never, "auto" decides per job: keep the pages when the retained estimate for this job's + core count fits inside memory_retain_auto_fraction of what a job can be given (the system + memory on a single machine, or the maxMemory option), otherwise divide the estimate by + memory_retain_ratio and do not keep them. The memory fit above was measured with retention + on. The consRetainPages command line option overrides. --> + + 2.2 (salamander Anc3, 769.0 -> 350.0 GiB), 3.3 (MammalsAnc0) --> 0 and big_threshold > 0: + biggest_ingroup = max([seq_id.size for seq_name, seq_id in seq_id_map.items() if seq_name not in outgroups] or [0]) + if biggest_ingroup >= big_threshold and big_window < poa_window: + RealtimeLogger.info('cactus_consolidated({}): largest ingroup is {}, at or above the {} threshold, so the poa window drops from {} to {}'.format( + name, bytes2human(biggest_ingroup), bytes2human(int(big_threshold)), poa_window, big_window)) + poa_window = big_window window_exp = getOptionalAttrib(cons_node, 'memory_poa_window_exponent', typeFn=float, default=0.43) if poa_window > 0 and poa_window != 10000: mem = int(mem * (poa_window / 10000.0) ** window_exp) @@ -134,12 +147,18 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma if retain_pages not in ['auto', '0', '1']: raise RuntimeError(' / --consRetainPages must be auto, 0 or 1, not {}'.format(retain_pages)) retain_ratio = getOptionalAttrib(cons_node, 'memory_retain_ratio', typeFn=float, default=2.5) + retain_fraction = getOptionalAttrib(cons_node, 'memory_retain_auto_fraction', typeFn=float, default=0.5) if retain_pages == 'auto': limits = [l for l in [max_system_memory, int(os.environ['CACTUS_MAX_MEMORY']) if 'CACTUS_MAX_MEMORY' in os.environ else None] if l] limit = min(limits) if limits else None - if limit and mem > limit: - RealtimeLogger.info('cactus_consolidated({}): the memory estimate of {} with jemalloc page retention exceeds the {} the job can be given, so the pages will not be retained'.format( - name, bytes2human(mem), bytes2human(limit))) + # Retention is only attempted with room to be wrong. mem here is the retained estimate for + # this job's core count, and 14% of VGP alignments came in over their estimate -- so asking + # it to fit in a fraction of what the job can be given means a miss still has somewhere to + # land. Without the fraction, any under-estimate at the ceiling OOMs the whole alignment. + budget = limit * retain_fraction if limit else None + if budget and mem > budget: + RealtimeLogger.info('cactus_consolidated({}): the memory estimate of {} with jemalloc page retention exceeds {:g} of the {} the job can be given, so the pages will not be retained'.format( + name, bytes2human(mem), retain_fraction, bytes2human(limit))) retain_pages = '0' else: retain_pages = '1' @@ -156,18 +175,25 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma cons_job = job.addChildJobFn(cactus_cons, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, intermediate_results_url=intermediate_results_url, chrom_name=chrom_name, cores = cons_cores, - memory=cactus_clamp_memory(mem), disk=disk, retain_pages=retain_pages) + memory=cactus_clamp_memory(mem), disk=disk, retain_pages=retain_pages, + poa_window=poa_window) return cons_job.rv() def cactus_cons(job, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, - intermediate_results_url = None, chrom_name = None, retain_pages = None): + intermediate_results_url = None, chrom_name = None, retain_pages = None, + poa_window = None): ''' run cactus_consolidated ''' # cactus_consolidated reads its settings from the config, so the resolved page retention # goes into the copy it is given (this job's copy of the node, so nothing else sees it) - if retain_pages is not None: + if retain_pages is not None or poa_window is not None: config_node = copy.deepcopy(config_node) - findRequiredNode(config_node, 'consolidated').set('retain_pages', str(retain_pages)) + if retain_pages is not None: + findRequiredNode(config_node, 'consolidated').set('retain_pages', str(retain_pages)) + # the estimator resolved the window (it may have been lowered for a giant genome), and the + # estimate it produced only holds if cactus_consolidated uses that same value + if poa_window is not None: + findRequiredNode(config_node, 'bar').find('poa').set('partialOrderAlignmentWindow', str(poa_window)) # Build up a genome -> fasta map. work_dir = job.fileStore.getLocalTempDir() From adfe6ab8e6932b45317bccd04daf7b6a2552eb67 Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 12:40:14 -0400 Subject: [PATCH 14/18] cut the bar progress line down to what is worth reading Seven fields became four. The line reported flowers and bases both as raw counts and as percentages, seconds for a phase that runs for days, megabytes for a peak in the hundreds of gibibytes, and an in-flight count that only meant something while debugging the tail. The flower percentage was the worst of them: the list is sorted largest-first, so it reads 0.00% while a third of the sequence is already aligned, and anyone watching a salamander saw a number that looked like nothing was happening for hours. Bases is the honest measure and is the one kept. Durations now pick their unit, because a bar phase runs from seconds on a test to 45.3h on a 22 Gb genome and "2718m elapsed" helps nobody. eta prints ? when no flower finished in the interval rather than inventing a figure from a zero rate. before: Bar progress: 69/18457342 flowers (0.00%), 38879597896/114143553856 bases (34.06%), 626 seconds in bar, eta 1211 seconds, peak memory 712204 MB, 63 in flight after: Bar progress: 34.1% of bases, 18.0h elapsed, eta 5.2h, peak 695.5 GiB Dropping those fields retires the flowersDone and flowersInFlight counters and their atomic updates in the flower loop, so this is less code and less synchronisation than before, not more. Evolver mammals is unchanged: same halStats, and the run was checked at the real 600s interval as well as at 1s to see the line actually render. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- bar/impl/bar.c | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/bar/impl/bar.c b/bar/impl/bar.c index c6dc446fa..2a9b8b213 100644 --- a/bar/impl/bar.c +++ b/bar/impl/bar.c @@ -20,6 +20,18 @@ #include #include +/* Minutes below an hour, hours above it: a bar phase runs from seconds on a test to two days + * on a 22 Gb genome, and "2418m elapsed" helps nobody. */ +static void bar_format_duration(char *buf, size_t n, int64_t seconds) { + if (seconds < 0) { + snprintf(buf, n, "?"); + } else if (seconds < 3600) { + snprintf(buf, n, "%" PRIi64 "m", (seconds + 30) / 60); + } else { + snprintf(buf, n, "%.1fh", seconds / 3600.0); + } +} + // How often to report progress through the flower list, in seconds. #define BAR_PROGRESS_INTERVAL 600 @@ -130,8 +142,7 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * const time_t barStartTime = time(NULL); time_t lastReportTime = barStartTime; int64_t lastReportBases = 0; - int64_t flowersDone = 0, basesDone = 0; - int64_t flowersInFlight = 0; // started but not finished: below the outer team size means the tail + int64_t basesDone = 0; #if defined(_OPENMP) #pragma omp parallel for schedule(dynamic, 1) @@ -144,10 +155,6 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * // Must be read before stCaf_finish, which adds block ends to the flower const int64_t flowerBases = reportProgress ? flower_getTotalBaseLength(flower) : 0; - if (reportProgress) { -#pragma omp atomic - ++flowersInFlight; - } // These are all variables used by the filter fns FilterArgs *fa = st_calloc(1, sizeof(FilterArgs)); @@ -220,11 +227,7 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * st_logDebug("Finished filling in the alignments for the flower\n"); if (reportProgress) { - int64_t done, bases, inFlight; -#pragma omp atomic capture - { --flowersInFlight; inFlight = flowersInFlight; } -#pragma omp atomic capture - done = ++flowersDone; + int64_t bases; #pragma omp atomic capture { basesDone += flowerBases; bases = basesDone; } @@ -263,13 +266,15 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * */ int64_t eta = (windowSeconds > 0 && windowBases > 0) ? (int64_t)((double)(totalBases - bases) * windowSeconds / windowBases) : -1; - st_logInfo("Bar progress: %" PRIi64 "/%" PRIi64 " flowers (%.2f%%), " - "%" PRIi64 "/%" PRIi64 " bases (%.2f%%), %" PRIi64 " seconds in bar, " - "eta %" PRIi64 " seconds, peak memory %" PRIi64 " MB, " - "%" PRIi64 " in flight\n", - done, flowerNumber, 100.0 * (double)done / (double)flowerNumber, - bases, totalBases, 100.0 * baseFraction, - elapsed, eta, peakMemMB, inFlight); + // one short line: the fraction that is actually informative (bases, not + // flowers -- the list is sorted largest-first, so 0.02% of flowers can be + // 40% of the sequence), how long it has taken, how long is left, and the + // peak. eta is omitted rather than guessed when nothing finished. + char etaBuf[32], elapsedBuf[32]; + bar_format_duration(etaBuf, sizeof(etaBuf), eta); + bar_format_duration(elapsedBuf, sizeof(elapsedBuf), elapsed); + st_logInfo("Bar progress: %.1f%% of bases, %s elapsed, eta %s, peak %.1f GiB\n", + 100.0 * baseFraction, elapsedBuf, etaBuf, peakMemMB / 1024.0); } } } From b46e3fc7f6d55c5d5ec591da07cba9a62520d25e Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 13:04:55 -0400 Subject: [PATCH 15/18] stop caf's profiling timers from drowning the log The caf-timing lines added in 9901d395 for profiling are emitted at info, which is the level a normal run uses. Four of them sit in stCaf_melt and stCaf_finish, and bar calls both once per flower -- so on salamander Anc3 they fire 18.4 million times. Measured on a run where they were not filtered out, and which was killed at 21.6% of its flowers: caf-timing lines 2,836,680 of 2,840,209 total = 99.88% of the log cactus-graph 727,544 convert 705,986 melt 703,690 finish graph 699,449 everything else 7 Every phase marker, progress line and error in that run is the remaining 0.12%. A completed run is the ~15 GB that barmem-salamander.sh greps away, where the filter was measured to be the bottleneck of the phase it was watching. Those four move to debug. The eight phase-level timers in caf.c stay at info: they fire once per run, and they are the view worth having. melting.c keeps two -- recoverable is per phase, and check-join only runs under the CACTUS_CAF_CHECK_JOIN hook. Nothing is lost that was readable. Profiling a single flower's melt among 2.8 million lines was never practical; --logDebug on a small input serves that far better. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- caf/impl/finishing.c | 4 ++-- caf/impl/melting.c | 4 ++-- caf/impl/pinchToCactus.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/caf/impl/finishing.c b/caf/impl/finishing.c index e311b7399..7e4cd4222 100644 --- a/caf/impl/finishing.c +++ b/caf/impl/finishing.c @@ -417,7 +417,7 @@ static void stCaf_convertCactusGraphToFlowers(stPinchThreadSet *threadSet, stCac double fillOutTime = stCaf_now() - t; t = stCaf_now(); stHash_destruct(cactusNodesToFlowers); - st_logInfo("caf-timing: convert endsHash %.3fs emptyFlowers %.3fs fillOut %.3fs cleanup %.3fs\n", + st_logDebug("caf-timing: convert endsHash %.3fs emptyFlowers %.3fs fillOut %.3fs cleanup %.3fs\n", endsHashTime, emptyFlowersTime, fillOutTime, stCaf_now() - t); } @@ -457,5 +457,5 @@ void stCaf_finish(Flower *flower, stPinchThreadSet *threadSet, int64_t minLength //Cleanup stCaf_destructCactusGraph(cactusGraph, threadSet); - st_logInfo("caf-timing: finish graph %.3fs convert %.3fs destruct %.3fs\n", graphTime, convertTime, stCaf_now() - t); + st_logDebug("caf-timing: finish graph %.3fs convert %.3fs destruct %.3fs\n", graphTime, convertTime, stCaf_now() - t); } diff --git a/caf/impl/melting.c b/caf/impl/melting.c index 555b7e7e3..a64830883 100644 --- a/caf/impl/melting.c +++ b/caf/impl/melting.c @@ -157,7 +157,7 @@ int64_t stCaf_melt(Flower *flower, stPinchThreadSet *threadSet, bool blockFilter } //Now heal up the trivial boundaries stCaf_joinTrivialBoundaries(threadSet); - st_logInfo("caf-timing: melt minChain=%" PRIi64 " trim %.3fs filter %.3fs graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 "\n", + st_logDebug("caf-timing: melt minChain=%" PRIi64 " trim %.3fs filter %.3fs graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 "\n", minimumChainLength, trimTime, filterTime, graphTime, scanTime, deleteTime, stCaf_now() - t, blocksDestroyed); return blocksDestroyed; } @@ -290,7 +290,7 @@ int64_t stCaf_meltChains(Flower *flower, stPinchThreadSet *threadSet, int64_t mi int64_t joins = joinTrivialBoundariesAtThreadEnds(threadSet); stCaf_ensureEndsAreDistinct(threadSet); double joinTime = stCaf_now() - t; - st_logInfo("caf-timing: melt minChain=%" PRIi64 " trim 0.000s filter 0.000s graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 " end-joins %" PRIi64 "\n", + st_logDebug("caf-timing: melt minChain=%" PRIi64 " trim 0.000s filter 0.000s graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 " end-joins %" PRIi64 "\n", minimumChainLength, graphTime, scanTime, deleteTime, joinTime, blocksDestroyed, joins); if (getenv("CACTUS_CAF_CHECK_JOIN") != NULL) { diff --git a/caf/impl/pinchToCactus.c b/caf/impl/pinchToCactus.c index 1ed13ae9a..0085894c3 100644 --- a/caf/impl/pinchToCactus.c +++ b/caf/impl/pinchToCactus.c @@ -543,7 +543,7 @@ stCactusGraph *stCaf_getCactusGraphForThreadSet(Flower *flower, stPinchThreadSet stCactusGraph *cactusGraph = stCaf_constructCactusGraph(threadSet, *deadEndComponent, adjacencyComponents, startCactusNode, breakChainsAtReverseTandems, maximumMedianSpacingBetweenLinkedEnds, &timing); - st_logInfo("caf-timing: cactus-graph ends=%" PRIi64 " adjacency %.3fs deadend %.3fs attach %.3fs build %.3fs collapse %.3fs bridges %.3fs tandems %.3fs median %.3fs nodes %" PRIi64 "->%" PRIi64 " tandem-calls %" PRIi64 " cached %" PRIi64 " block-reads %" PRIi64 "\n", + st_logDebug("caf-timing: cactus-graph ends=%" PRIi64 " adjacency %.3fs deadend %.3fs attach %.3fs build %.3fs collapse %.3fs bridges %.3fs tandems %.3fs median %.3fs nodes %" PRIi64 "->%" PRIi64 " tandem-calls %" PRIi64 " cached %" PRIi64 " block-reads %" PRIi64 "\n", timing.ends, timing.adjacency, timing.deadEnd, timing.attach, timing.build, timing.collapse, timing.bridges, timing.tandems, timing.median, timing.nodesBeforeCollapse, timing.nodesAfterCollapse, timing.tandemCalls, timing.tandemCached, timing.blockReads); stCaf_dumpCactusGraph(cactusGraph, *startCactusNode); From e6aa481f32e2f0e7f82ad631195f48ae8d8383d1 Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 13:15:31 -0400 Subject: [PATCH 16/18] give cactus_consolidated transparent huge pages by default jemalloc backs its extents with huge pages when MALLOC_CONF carries thp:always. Measured on salamander Anc3 without page retention, same processor model on both runs: no thp caf 6196s bar+reference 36465s total 44398s peak 347.0 GiB thp caf 5783s bar+reference 25579s total 33124s peak 350.0 GiB 1.34x over the whole job for 0.9% more memory, and it lands on the page retention *off* path -- which is the path a genome big enough to matter now takes, so the two compound. This has to go in the environment. jemalloc reads MALLOC_CONF before main and opt.thp is not writable through mallctl afterwards, so unlike page retention it cannot be set from inside cactus_consolidated. A MALLOC_CONF already in the environment is left alone, so --setEnv still overrides. It only does anything where the kernel has transparent huge pages in madvise mode, which is the usual setting: that is when jemalloc's thp:always is what issues the MADV_HUGEPAGE. Under never it is silently nothing, which is worth knowing before measuring. One pair of runs, and only with retention off -- thp with retention on is not measured. CAF moved 7% between the two hosts where bar+reference moved 43%, so the effect is well clear of the run-to-run noise, but it is still n=1. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- src/cactus/cactus_progressive_config.xml | 8 ++++++++ src/cactus/pipeline/cactus_workflow.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/src/cactus/cactus_progressive_config.xml b/src/cactus/cactus_progressive_config.xml index 380250b95..da584e4c9 100644 --- a/src/cactus/cactus_progressive_config.xml +++ b/src/cactus/cactus_progressive_config.xml @@ -605,6 +605,13 @@ memory on a single machine, or the maxMemory option), otherwise divide the estimate by memory_retain_ratio and do not keep them. The memory fit above was measured with retention on. The consRetainPages command line option overrides. --> + + # 33124 seconds for 0.9% more peak (347.0 -> 350.0 GiB). An existing MALLOC_CONF wins, so the + # setting stays overridable from the command line. + if getOptionalAttrib(findRequiredNode(config_node, 'consolidated'), 'transparent_huge_pages', typeFn=bool, default=True) \ + and 'MALLOC_CONF' not in os.environ: + os.environ['MALLOC_CONF'] = 'thp:always' + messages = cactus_call(check_output=True, returnStdErr=True, realtimeStderrPrefix=f'cactus_consolidated({chrom_name if chrom_name else ancestor_event})', parameters=["cactus_consolidated"] + args, From 2608aa525e595408e1b392848e5efdb6c7aa6f1b Mon Sep 17 00:00:00 2001 From: Glenn Hickey Date: Thu, 17 Sep 2026 14:29:48 -0400 Subject: [PATCH 17/18] fix what the review found An audit of the branch turned up five real problems and three claims of mine that were simply false. memory_retain_auto_fraction="0" meant "always retain", the exact inverse of what it reads as: a fraction of 0 makes the budget 0.0, which is falsy, so the test fell through to keeping the pages. Compared against None instead. rgfa-collapse's disk floor was set from a partial disk report. Reading all 25 instances in the sep16 HPRC log gives a peak of 6.57 GiB, not 2.9, and disk is flat in graph size exactly as memory is -- 6.50 to 6.57 GiB while the gfa spans 6000x. The floor goes to 16 GiB, matching memory's 2x headroom over its own observed peak. cactus_cons set the poa window without the poa_node guard the estimator uses three lines earlier, so a config with no child would die on an AttributeError rather than the error it deserves. Transparent huge pages went in through os.environ, which is the worker's own environment and is inherited by whatever job toil chains in next -- in this workflow that is export_hal. Prefixing `env` scopes it to the one process, and as a bonus it now reaches the container: dockerCommand passes no -e, so the exported variable never arrived under --binariesMode docker at all. Three --consRetainPages help strings still described the pre-K behaviour. And the false claims, all in comments or messages I wrote: "8*ref_size asked ~180 GiB" was wrong by about 8x. 180 GiB is what the minigraph formula gives (60*max_size + total/4); 8*ref_size is ~23 GiB for a whole reference, or the ~2 GiB actually seen for a chromosome slice. The mem_cap hoist was justified by saying a worker has no CACTUS_MAX_MEMORY. Workers do: toil pickles the leader's environment out to them, and line 400 of that same job function has always relied on it. The hoist also silently dropped cactus_clamp_memory's lower bound, so it is reverted and the clamp is back in the lambda where it belongs. "Those four move to debug" was five. The fifth, in stCaf_meltChains, is reached only from caf.c -- phase-level, the category that commit says stays at info. Restored. Two further findings needed no change. The window a subproblem was aligned at cannot be recorded in CACTUS_CONFIG, because one hal holds genomes aligned at different windows. And export_hal's 2x was said to be applied to a path the VGP fit does not cover; 40 of 40 sampled VGP logs ran halAppendCactusSubtree with --inMemory, the same as cactus-align, so it does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018ArwEg9b3ht5o6xnWh94Ui --- caf/impl/melting.c | 2 +- src/cactus/cactus_progressive_config.xml | 14 +++++------ src/cactus/pipeline/cactus_workflow.py | 25 ++++++++++++-------- src/cactus/progressive/cactus_progressive.py | 2 +- src/cactus/refmap/cactus_minigraph.py | 20 +++++++--------- src/cactus/refmap/cactus_pangenome.py | 2 +- src/cactus/setup/cactus_align.py | 2 +- 7 files changed, 35 insertions(+), 32 deletions(-) diff --git a/caf/impl/melting.c b/caf/impl/melting.c index a64830883..af931dde0 100644 --- a/caf/impl/melting.c +++ b/caf/impl/melting.c @@ -290,7 +290,7 @@ int64_t stCaf_meltChains(Flower *flower, stPinchThreadSet *threadSet, int64_t mi int64_t joins = joinTrivialBoundariesAtThreadEnds(threadSet); stCaf_ensureEndsAreDistinct(threadSet); double joinTime = stCaf_now() - t; - st_logDebug("caf-timing: melt minChain=%" PRIi64 " trim 0.000s filter 0.000s graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 " end-joins %" PRIi64 "\n", + st_logInfo("caf-timing: melt minChain=%" PRIi64 " trim 0.000s filter 0.000s graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 " end-joins %" PRIi64 "\n", minimumChainLength, graphTime, scanTime, deleteTime, joinTime, blocksDestroyed, joins); if (getenv("CACTUS_CAF_CHECK_JOIN") != NULL) { diff --git a/src/cactus/cactus_progressive_config.xml b/src/cactus/cactus_progressive_config.xml index da584e4c9..673acfd65 100644 --- a/src/cactus/cactus_progressive_config.xml +++ b/src/cactus/cactus_progressive_config.xml @@ -605,13 +605,13 @@ memory on a single machine, or the maxMemory option), otherwise divide the estimate by memory_retain_ratio and do not keep them. The memory fit above was measured with retention on. The consRetainPages command line option overrides. --> - +