diff --git a/bar/impl/bar.c b/bar/impl/bar.c index c6dc446fa..2a9b8b213 100644 --- a/bar/impl/bar.c +++ b/bar/impl/bar.c @@ -20,6 +20,18 @@ #include #include +/* Minutes below an hour, hours above it: a bar phase runs from seconds on a test to two days + * on a 22 Gb genome, and "2418m elapsed" helps nobody. */ +static void bar_format_duration(char *buf, size_t n, int64_t seconds) { + if (seconds < 0) { + snprintf(buf, n, "?"); + } else if (seconds < 3600) { + snprintf(buf, n, "%" PRIi64 "m", (seconds + 30) / 60); + } else { + snprintf(buf, n, "%.1fh", seconds / 3600.0); + } +} + // How often to report progress through the flower list, in seconds. #define BAR_PROGRESS_INTERVAL 600 @@ -130,8 +142,7 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * const time_t barStartTime = time(NULL); time_t lastReportTime = barStartTime; int64_t lastReportBases = 0; - int64_t flowersDone = 0, basesDone = 0; - int64_t flowersInFlight = 0; // started but not finished: below the outer team size means the tail + int64_t basesDone = 0; #if defined(_OPENMP) #pragma omp parallel for schedule(dynamic, 1) @@ -144,10 +155,6 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * // Must be read before stCaf_finish, which adds block ends to the flower const int64_t flowerBases = reportProgress ? flower_getTotalBaseLength(flower) : 0; - if (reportProgress) { -#pragma omp atomic - ++flowersInFlight; - } // These are all variables used by the filter fns FilterArgs *fa = st_calloc(1, sizeof(FilterArgs)); @@ -220,11 +227,7 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * st_logDebug("Finished filling in the alignments for the flower\n"); if (reportProgress) { - int64_t done, bases, inFlight; -#pragma omp atomic capture - { --flowersInFlight; inFlight = flowersInFlight; } -#pragma omp atomic capture - done = ++flowersDone; + int64_t bases; #pragma omp atomic capture { basesDone += flowerBases; bases = basesDone; } @@ -263,13 +266,15 @@ void bar(stList *flowers, CactusParams *params, CactusDisk *cactusDisk, stList * */ int64_t eta = (windowSeconds > 0 && windowBases > 0) ? (int64_t)((double)(totalBases - bases) * windowSeconds / windowBases) : -1; - st_logInfo("Bar progress: %" PRIi64 "/%" PRIi64 " flowers (%.2f%%), " - "%" PRIi64 "/%" PRIi64 " bases (%.2f%%), %" PRIi64 " seconds in bar, " - "eta %" PRIi64 " seconds, peak memory %" PRIi64 " MB, " - "%" PRIi64 " in flight\n", - done, flowerNumber, 100.0 * (double)done / (double)flowerNumber, - bases, totalBases, 100.0 * baseFraction, - elapsed, eta, peakMemMB, inFlight); + // one short line: the fraction that is actually informative (bases, not + // flowers -- the list is sorted largest-first, so 0.02% of flowers can be + // 40% of the sequence), how long it has taken, how long is left, and the + // peak. eta is omitted rather than guessed when nothing finished. + char etaBuf[32], elapsedBuf[32]; + bar_format_duration(etaBuf, sizeof(etaBuf), eta); + bar_format_duration(elapsedBuf, sizeof(elapsedBuf), elapsed); + st_logInfo("Bar progress: %.1f%% of bases, %s elapsed, eta %s, peak %.1f GiB\n", + 100.0 * baseFraction, elapsedBuf, etaBuf, peakMemMB / 1024.0); } } } diff --git a/caf/impl/finishing.c b/caf/impl/finishing.c index e311b7399..7e4cd4222 100644 --- a/caf/impl/finishing.c +++ b/caf/impl/finishing.c @@ -417,7 +417,7 @@ static void stCaf_convertCactusGraphToFlowers(stPinchThreadSet *threadSet, stCac double fillOutTime = stCaf_now() - t; t = stCaf_now(); stHash_destruct(cactusNodesToFlowers); - st_logInfo("caf-timing: convert endsHash %.3fs emptyFlowers %.3fs fillOut %.3fs cleanup %.3fs\n", + st_logDebug("caf-timing: convert endsHash %.3fs emptyFlowers %.3fs fillOut %.3fs cleanup %.3fs\n", endsHashTime, emptyFlowersTime, fillOutTime, stCaf_now() - t); } @@ -457,5 +457,5 @@ void stCaf_finish(Flower *flower, stPinchThreadSet *threadSet, int64_t minLength //Cleanup stCaf_destructCactusGraph(cactusGraph, threadSet); - st_logInfo("caf-timing: finish graph %.3fs convert %.3fs destruct %.3fs\n", graphTime, convertTime, stCaf_now() - t); + st_logDebug("caf-timing: finish graph %.3fs convert %.3fs destruct %.3fs\n", graphTime, convertTime, stCaf_now() - t); } diff --git a/caf/impl/melting.c b/caf/impl/melting.c index 555b7e7e3..af931dde0 100644 --- a/caf/impl/melting.c +++ b/caf/impl/melting.c @@ -157,7 +157,7 @@ int64_t stCaf_melt(Flower *flower, stPinchThreadSet *threadSet, bool blockFilter } //Now heal up the trivial boundaries stCaf_joinTrivialBoundaries(threadSet); - st_logInfo("caf-timing: melt minChain=%" PRIi64 " trim %.3fs filter %.3fs graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 "\n", + st_logDebug("caf-timing: melt minChain=%" PRIi64 " trim %.3fs filter %.3fs graph %.3fs scan %.3fs delete %.3fs join %.3fs destroyed %" PRIi64 "\n", minimumChainLength, trimTime, filterTime, graphTime, scanTime, deleteTime, stCaf_now() - t, blocksDestroyed); return blocksDestroyed; } diff --git a/caf/impl/pinchToCactus.c b/caf/impl/pinchToCactus.c index 1ed13ae9a..0085894c3 100644 --- a/caf/impl/pinchToCactus.c +++ b/caf/impl/pinchToCactus.c @@ -543,7 +543,7 @@ stCactusGraph *stCaf_getCactusGraphForThreadSet(Flower *flower, stPinchThreadSet stCactusGraph *cactusGraph = stCaf_constructCactusGraph(threadSet, *deadEndComponent, adjacencyComponents, startCactusNode, breakChainsAtReverseTandems, maximumMedianSpacingBetweenLinkedEnds, &timing); - st_logInfo("caf-timing: cactus-graph ends=%" PRIi64 " adjacency %.3fs deadend %.3fs attach %.3fs build %.3fs collapse %.3fs bridges %.3fs tandems %.3fs median %.3fs nodes %" PRIi64 "->%" PRIi64 " tandem-calls %" PRIi64 " cached %" PRIi64 " block-reads %" PRIi64 "\n", + st_logDebug("caf-timing: cactus-graph ends=%" PRIi64 " adjacency %.3fs deadend %.3fs attach %.3fs build %.3fs collapse %.3fs bridges %.3fs tandems %.3fs median %.3fs nodes %" PRIi64 "->%" PRIi64 " tandem-calls %" PRIi64 " cached %" PRIi64 " block-reads %" PRIi64 "\n", timing.ends, timing.adjacency, timing.deadEnd, timing.attach, timing.build, timing.collapse, timing.bridges, timing.tandems, timing.median, timing.nodesBeforeCollapse, timing.nodesAfterCollapse, timing.tandemCalls, timing.tandemCached, timing.blockReads); stCaf_dumpCactusGraph(cactusGraph, *startCactusNode); diff --git a/doc/pangenome.md b/doc/pangenome.md index cd932c088..c862e62ff 100644 --- a/doc/pangenome.md +++ b/doc/pangenome.md @@ -711,7 +711,7 @@ One thing to watch out for: **error intervals at contig tips interact with telom ``` cactus-panpatch ./js chromfile.txt --outDir patched --batch --requireTelomeres \ - --batchSystem slurm --slurmTime 10:00:00 --doubleMem true --maxMemory 1.5T --retryCount 10 \ + --batchSystem slurm --doubleMem true --maxMemory 1.5T --retryCount 10 \ --consCores 16 --mgCores 32 --mapCores 16 ``` diff --git a/doc/progressive.md b/doc/progressive.md index c3d71edd2..75e0f3670 100644 --- a/doc/progressive.md +++ b/doc/progressive.md @@ -232,7 +232,7 @@ cactus-hal2maf ./js evolverMammals.hal evolverMammals.maf.gz --refGenome simHuma Exporting a MAF for each reference in an 8-way [ape alignment](https://cglgenomics.ucsc.edu/february-2024-t2t-apes/) on UCSC Slurm cluster: ``` -for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16 --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --slurmTime 200:00:00 --slurmPartition long;done +for i in hs1 hg38 GCA_028858775.2 GCA_028885655.2 GCA_028885625.2 GCA_028878055.2 GCA_029281585.2 GCA_029289425.2; do cactus-hal2maf ./js_hal2maf8 ./8-t2t-apes-2023v2.hal ./8-t2t-apes-2023v2.${i}.maf.gz --filterGapCausingDupes --outType norm single --refGenome $i --chunkSize 500000 --batchCores 64 --noAncestors --batchCount 16 --batchSystem slurm --logFile ./8-t2t-apes-2023v2.${i}.gz.log --batchLogsDir batch-logs-8apes --slurmPartition long;done ``` Note that this invocation creates two MAFs per reference (as dictated by the `--outType` option). Notably, the `.single.maf.gz` files will be filtered so that each genome appears at most once per block, which is often required by browsers and other MAF-reading tools. @@ -343,7 +343,7 @@ cactus-phast ./js-vgp \ --geneAnnotation https://hgdownload.soe.ucsc.edu/goldenPath/hg38/database/ncbiRefSeq.txt.gz \ --bigwig --batchSystem slurm --chunkCores 32 --phyloFitCores 32 \ --substMod REV --modFreqs --precision HIGH \ - --slurmPartition medium --slurmTime 10:00:00 \ + --slurmPartition medium \ --doubleMem true ``` @@ -419,8 +419,15 @@ These are the most relevant options for running on a cluster On a cluster with partitions and/or time limits, make sure to use -* `--slurmTime` to specify the time for each job. Unfortunately cactus does not yet try to set this itself, so you need to give one value that will be applied to all jobs, ex `--slurmTime 200:00:00` -* `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own using the `--slurmTime` value along with whether or not the job needs GPU. But this option will allow you to override that. +Slurm chooses a job's partition from its time limit, so giving every job one big time (the old `--slurmTime 200:00:00` advice) puts even the thousands of jobs that finish in seconds into the slowest queue. **Cactus now estimates a walltime for each job itself**, from the size of that job's inputs, the same way it estimates memory. There is nothing you need to set for this: the defaults below are the knobs for when the estimates do not suit your cluster. + +* `--doubleTime true` is **on by default** (Cactus turns it on the way it raises `--retryCount` to 5). It is the time analogue of `--doubleMem`: if slurm kills a job for running past its walltime, it is retried with double the walltime. This is what makes tight per-job estimates safe -- an occasional underestimate costs one retry, not the run. Pass `--doubleTime false` to turn it off. +* `--walltimeFactor` is the safety multiplier applied to every estimate (default `2.5`). Raise it if your cluster is slower than the ones the estimates were measured on and jobs are being killed for running over. Pass `--walltimeFactor 0` to switch per-job walltimes off entirely, which puts every job back on `--defaultWalltime`. +* `--minWalltime` is the floor for any request, in seconds (default `1800`). A short partition is typically an hour, so every request below that routes the same way and there is nothing to gain by asking for less -- while a request that is short by a minute costs a whole `--doubleTime` retry. Half an hour leaves room for worker startup and for jobstore staging under heavy concurrency, which no size-based estimate can see. +* `--maxWalltime` is the ceiling, in seconds. By default, on Slurm, Cactus queries `sinfo` and caps every request at *half* the longest time limit of any partition your jobs could land on. This matters more than it looks: when Toil cannot find a partition that fits a job's walltime it raises rather than falling back to a default, and it does so on the batch system's own thread, taking the whole workflow down rather than just the job. Since `--doubleTime` retries a killed job with twice the walltime, an estimate sitting at the longest partition's limit would turn its own first retry into exactly that crash -- hence the half. A job needing more than half your longest partition still gets there, it just spends one attempt on the way. With `--doubleTime false` there is nothing to retry into and the full limit is used. +* `--defaultWalltime` is Toil's fallback for jobs Cactus did not estimate a time for. With the estimates on there is very little left for it to cover, so you can usually leave it alone. +* `--slurmPartition / --slurmGPUPartition` to specify the slurm partition where CPU / GPU jobs end up on. Cactus will try to figure this out on its own from each job's walltime along with whether or not the job needs GPU. But this option will allow you to override that. +* `--slurmTime` is a global override that forces a single time onto *every* job, ignoring the per-job estimates above (so it also disables the partition routing they exist for, and `--doubleTime` along with it). Reach for it only as an escape hatch, e.g. if a job's time estimate turns out too low and it keeps getting killed: `--slurmTime 200:00:00`. You can also use @@ -439,13 +446,13 @@ source /private/groups/cgl/cactus/venv-cactus-latest/bin/activate Some recommended options: ``` -cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true --slurmTime 200:00:00 +cactus ./js ./examples/evolverMammals.txt evolverMammals.hal --batchSystem slurm --batchLogsDir batch-logs --consCores 64 --maxMemory 1.4Ti --doubleMem true ``` To run the same command step by step, ``` -cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --slurmTime 200:00:00 --batchSystem slurm" --alignCores 64 --script > mammals.sh +cactus-prepare ./examples/evolverMammals.txt --outDir mammals-prepare --outHal mammals-prepare/evolverMammals.hal --cactusOptions "--maxMemory 1.4Ti --doubleMem true --batchSystem slurm" --alignCores 64 --script > mammals.sh chmod+x mammals.sh ./mammals.sh ``` diff --git a/pipeline/cactus_consolidated.c b/pipeline/cactus_consolidated.c index 82b8d029c..72b563360 100644 --- a/pipeline/cactus_consolidated.c +++ b/pipeline/cactus_consolidated.c @@ -110,6 +110,7 @@ static void cactus_jemalloc_retain_pages(CactusParams *params) { } const char *names[2] = { "dirty_decay_ms", "muzzy_decay_ms" }; + bool applied = true; for (int w = 0; w < 2; w++) { ssize_t v = -1; // never purge char key[64]; @@ -137,9 +138,24 @@ static void cactus_jemalloc_retain_pages(CactusParams *params) { size_t sz = sizeof(readback); snprintf(key, sizeof(key), "arenas.%s", names[w]); mallctl_fn(key, &readback, &sz, NULL, 0); - st_logInfo("jemalloc %s set to -1: default for new arenas rc=%i (reads back %" PRIi64 "), " - "%i of %u existing arenas set, %i declined\n", - names[w], rc_default, (int64_t)readback, set, narenas, declined); + // "declined" is the healthy case, not a failure: jemalloc pre-allocates arena slots but + // only initialises the ones in use, and an uninitialised slot refuses the write -- it + // inherits the default above when it is eventually created. Only the default taking + // matters, so that is what the one line reports; the rest is for --logDebug. + if (rc_default != 0 || readback != -1) { + applied = false; + } + st_logDebug("jemalloc %s: default rc=%i (reads back %" PRIi64 "), " + "%i of %u existing arenas set, %i declined\n", + names[w], rc_default, (int64_t)readback, set, narenas, declined); + } + // report by exception: the workflow has already logged which way it decided, so this only + // speaks up when the decision could not be carried out. + if (applied) { + st_logDebug("jemalloc page retention applied: dirty_decay_ms and muzzy_decay_ms set to -1\n"); + } else { + st_logInfo("jemalloc page retention was requested but could NOT be applied; " + "pages will be returned to the OS\n"); } } diff --git a/src/cactus/blast/cactus_blast.py b/src/cactus/blast/cactus_blast.py index 1a1047174..0c1abfd13 100644 --- a/src/cactus/blast/cactus_blast.py +++ b/src/cactus/blast/cactus_blast.py @@ -12,12 +12,12 @@ import timeit from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import getOptionalAttrib from cactus.shared.version import cactus_commit from cactus.progressive.cactus_prepare import human2bytesN @@ -34,6 +34,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help="Seq file") parser.add_argument("outputFile", type=str, help="Output pairwise alignment file") @@ -156,7 +157,7 @@ def runCactusBlastOnly(options): input_seq_id_map[genome] = toil.importFile(seq) paf_id = toil.start(Job.wrapJobFn(sanitize_then_make_paf_alignments, NXNewick().writeString(spanning_tree), - input_seq_id_map, options.root, config_node, options.outputFile)) + input_seq_id_map, options.root, config_node, options.outputFile, walltime=cactus_walltime())) # export the alignments toil.exportFile(paf_id, makeURL(options.outputFile)) diff --git a/src/cactus/cactus_progressive_config.xml b/src/cactus/cactus_progressive_config.xml index 286c42917..6b03bd959 100644 --- a/src/cactus/cactus_progressive_config.xml +++ b/src/cactus/cactus_progressive_config.xml @@ -135,6 +135,20 @@ five="--step=2 --ambiguous=iupac,100,100 --ydrop=4000 --hspthresh=2400 --gappedthresh=2600 --queryhspbest=100000" default="--step=1 --ambiguous=iupac,100,100 --ydrop=4000 --hspthresh=2200 --gappedthresh=2400 --queryhspbest=100000" /> + + + 299 GiB). "1" always, - "0" never, "auto" keeps them unless the retained memory estimate is more than the job can - be given (the system memory on a single machine, or the maxMemory option), in which case the - estimate is divided by memory_retain_ratio and the pages are not kept. The memory fit - above was measured with retention on. The consRetainPages command line option overrides. --> + "0" never, "auto" decides per job: keep the pages when the retained estimate for this job's + core count fits inside memory_retain_auto_fraction of what a job can be given (the system + memory on a single machine, or the maxMemory option), otherwise divide the estimate by + memory_retain_ratio and do not keep them. The memory fit above was measured with retention + on. The consRetainPages command line option overrides. --> + + + 2.2 (salamander Anc3, 769.0 -> 350.0 GiB), 3.3 (MammalsAnc0) --> diff --git a/src/cactus/maf/cactus_hal2chains.py b/src/cactus/maf/cactus_hal2chains.py index 4603f9f86..327144c02 100644 --- a/src/cactus/maf/cactus_hal2chains.py +++ b/src/cactus/maf/cactus_hal2chains.py @@ -14,12 +14,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.version import cactus_commit @@ -41,6 +41,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help = "HAL file to convert to MAF") parser.add_argument("outDir", help = "Output directory") @@ -154,7 +155,7 @@ def main(): config.substituteAllPredefinedConstantsWithLiterals(options) hal_id = toil.importFile(options.halFile) - chains_id_dict = toil.start(Job.wrapJobFn(hal2chains_workflow, config, options, hal_id)) + chains_id_dict = toil.start(Job.wrapJobFn(hal2chains_workflow, config, options, hal_id, walltime=cactus_walltime())) #export the chains for query_genome in chains_id_dict.keys(): @@ -176,15 +177,27 @@ def main(): def hal2chains_workflow(job, config, options, hal_id): - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) - check_tools_job = root_job.addChildJobFn(hal2chains_check_tools, options) + # three (five with --bigChain) no-arg tool probes: half a second with the binaries on the + # PATH. The estimate is for --binariesMode docker/singularity, where each probe is a + # container start and the first one may pull the image. + check_tools_job = root_job.addChildJobFn(hal2chains_check_tools, options, walltime=cactus_walltime(180)) + # halStats --tree is 0.01s on the 577-way and the all-pairs distance matrix is a few seconds + # of python on its ~1150 nodes; on a HAL of any size this job is a HAL copy and little else get_genomes_job = check_tools_job.addFollowOnJobFn(hal2chains_get_genomes, config, options, hal_id, - disk=int(hal_id.size * 1.2)) + disk=int(hal_id.size * 1.2), + walltime=cactus_walltime(300, io_bytes=hal_id.size)) leaf_genomes = get_genomes_job.rv(0) distance_matrix = get_genomes_job.rv(1) - chrom_info_job = get_genomes_job.addFollowOnJobFn(hal2chains_chrom_info_all, config, options, hal_id, leaf_genomes) - hal2chains_all_job = chrom_info_job.addFollowOnJobFn(hal2chains_all, config, options, hal_id, chrom_info_job.rv(), distance_matrix) + chrom_info_job = get_genomes_job.addFollowOnJobFn(hal2chains_chrom_info_all, config, options, hal_id, leaf_genomes, walltime=cactus_walltime()) + # not the coordination tier: what this job costs is toil job-graph construction, and that is + # |queryGenomes| x |targetGenomes|. It took 5s to issue the 576 pairs of the 577-way hg38 + # chains run (with --bigChain, so a follow-on per pair as well), and a default all-vs-all + # 577-way is 577x576 = 332352 pairs, i.e. the better part of an hour. Neither genome list is + # known here -- both come out of hal2chains_get_genomes -- so this has to be a constant. + hal2chains_all_job = chrom_info_job.addFollowOnJobFn(hal2chains_all, config, options, hal_id, chrom_info_job.rv(), distance_matrix, + walltime=cactus_walltime(1800)) return hal2chains_all_job.rv() def hal2chains_check_tools(job, options): @@ -282,6 +295,30 @@ def chain_pair_cost(q, t, chrom_info_dict, distance_matrix, epsilon=0.05): d = distance_matrix[q][t] return min(q_size, t_size) / (d + epsilon) +# Seconds of chain pipeline per unit of chain_pair_cost. Calibrated against the 1719 +# halLiftover|pslPosTarget|axtChain runs of the VGP 577-way: total time / total cost. The +# proxy is noisy per pair (log-log r=0.56, and the worst single pair is 26x the p50 rate), but +# a batch sums many pairs and the noise averages out -- replaying that LPT assignment, the +# predicted batch time is within 1.02x of the truth at 3 batches and 1.55x at 64. +CHAIN_SECS_PER_COST = 1.783e-05 + + +def estimate_batch_walltime(options, batch_pairs, chrom_info_dict, distance_matrix): + """ estimated seconds for one hal2chains batch: the batch's total pair cost spread over + the GNU parallel slots it will actually use """ + if not batch_pairs: + return 0.0 + total_cost = sum(chain_pair_cost(q, t, chrom_info_dict, distance_matrix) for q, t in batch_pairs) + slots = max(1, min(options.batchParallelHal2chains or 1, len(batch_pairs))) + # A batch can never finish faster than its single longest pair, and the mean rate is a poor + # predictor of one pair: across those 1719 runs p99/p50 was 7.3x and max/p50 19.3x. With a + # small --batchSize the sum is only a few pairs and that spread shows straight through, so + # floor the estimate at the worst pair in the batch charged at 5x the mean rate. + worst_cost = max(chain_pair_cost(q, t, chrom_info_dict, distance_matrix) for q, t in batch_pairs) + return max(CHAIN_SECS_PER_COST * total_cost / slots, + 5.0 * CHAIN_SECS_PER_COST * worst_cost) + + def estimate_batch_memory(options, hal_id, pair_2bit_max=0): """ pick a memory request for a hal2chains batch job. @@ -343,10 +380,16 @@ def hal2chains_chrom_info_all(job, config, options, hal_id, genomes): continue # disk: hal copy + headroom for all the 2bits/beds we'll generate (2bits ~ fasta size ~ hal/ngenomes) batch_disk = int(hal_id.size * 1.2) + int(hal_id.size * 1.5 * len(batch_genomes) / max(1, len(all_genomes))) + # hal2fasta|faToTwoBit was p50 20s / p99 82s per genome across the 577-way, run + # batchParallelHal2chains at a time -- but on a HAL that size the job spends most of + # its life copying the HAL, which is what io_bytes accounts for + batch_slots = max(1, min(options.batchParallelHal2chains or 1, len(batch_genomes))) + chrom_info_secs = 100.0 * len(batch_genomes) / batch_slots batch_job = job.addChildJobFn(hal2chains_chrom_info_batch, config, options, hal_id, batch_genomes, disk=batch_disk, cores=options.batchCores, - memory=batch_memory) + memory=batch_memory, + walltime=cactus_walltime(chrom_info_secs, io_bytes=hal_id.size)) for g in batch_genomes: chrom_info_dict[g] = batch_job.rv(g) return chrom_info_dict @@ -459,7 +502,10 @@ def hal2chains_all(job, config, options, hal_id, chrom_info_dict, distance_matri batch_pairs, batch_chrom_info, batch_distances, disk=batch_disk, cores=options.batchCores, - memory=batch_memory) + memory=batch_memory, + walltime=cactus_walltime( + estimate_batch_walltime(options, batch_pairs, chrom_info_dict, distance_matrix), + io_bytes=hal_id.size + total_2bit)) for q, t in batch_pairs: if q not in output_dict: @@ -470,10 +516,14 @@ def hal2chains_all(job, config, options, hal_id, chrom_info_dict, distance_matri # disk: chain file + intermediates (~10x target 2bit is generous). # memory: hgLoadChain peak is ~2x uncompressed chain size ≈ a few GiB for big mammals. t_2bit_size = chrom_info_dict[t]['2bit'].size + # every command here was under two minutes across the 577-way's 1719 bigChains + # (bedToBigBed p99 63s, hgLoadChain p99 15s); the chain it reads is the only + # thing that grows, and it is small next to the 2bit bigchains_job = batch_job.addFollowOnJobFn(chain2bigchain, options, q, t, chrom_info_dict[t], batch_job.rv(q, t), disk=max(20 * t_2bit_size, 1024**3), - memory=cactus_clamp_memory(max(5 * t_2bit_size, 2 * 1024**3))) + memory=cactus_clamp_memory(max(5 * t_2bit_size, 2 * 1024**3)), + walltime=cactus_walltime(300, io_bytes=2 * t_2bit_size)) output_dict[q][t]['bigChain'] = bigchains_job.rv(0) output_dict[q][t]['bigLink'] = bigchains_job.rv(1) diff --git a/src/cactus/maf/cactus_hal2maf.py b/src/cactus/maf/cactus_hal2maf.py index f40a75882..f9836b8a9 100644 --- a/src/cactus/maf/cactus_hal2maf.py +++ b/src/cactus/maf/cactus_hal2maf.py @@ -16,12 +16,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import clean_jobstore_files @@ -30,6 +30,7 @@ from cactus.progressive.cactus_prepare import human2bytesN from cactus.progressive.multiCactusTree import MultiCactusTree from cactus.maf.maf_chunk import parse_bed_ranges +from cactus.maf.maf_chunk import taffy_walltime_secs, TAFFY_SIDE_OUTPUT_FRACTION from toil.job import Job from toil.common import Toil @@ -45,6 +46,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help = "HAL file to convert to MAF") parser.add_argument("outputMAF", help = "Output MAF (will be gzipped if ends in .gz). Suffix with .taf or .taf.gz if you want TAF output") @@ -221,12 +223,37 @@ def main(): bed_id = toil.importFile(options.bedRanges) if options.bedRanges else None hal_id = toil.importFile(options.halFile) - toil.start(Job.wrapJobFn(hal2maf_workflow, hal_id, bed_id, options, config)) + toil.start(Job.wrapJobFn(hal2maf_workflow, hal_id, bed_id, options, config, walltime=cactus_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time logger.info("cactus-hal2maf has finished after {} seconds".format(run_time)) +# Core-seconds of hal2maf-plus-taffy per chunk per genome in the alignment. Fitted to the 824 +# batch jobs of the VGP 577-way MAF export (31 references, 209k chunks, 22153 CPU-hours): the +# work in a chunk is set by how many genomes align there, not by how many bases the chunk spans +# -- a fit against chunk length comes out with a negative slope. +HAL2MAF_CORE_SECS_PER_CHUNK_PER_GENOME = 1.2 + +# A batch cannot finish faster than its slowest single chunk, and chunk cost varies enormously +# (per-chunk hal2maf was p50 71s against p99 1570s on that alignment). Dividing the total work +# by the core count assumes a balance that does not exist when a batch holds only a few chunks +# per core, which is where every badly-underestimated batch in the fit came from. This term is +# one slow chunk's worth, again per genome. +HAL2MAF_SLOW_CHUNK_CORE_SECS_PER_GENOME = 2.0 + + +def hal2maf_batch_walltime(batch_chunks, genome_list, options): + """ estimated seconds for one hal2maf batch: its total work spread over the cores it has, + plus the one slow chunk that sets the tail. Covers 98% of the 824 VGP batches once + cactus_walltime()'s factor is applied; the rest are what --doubleTime is for. """ + n_chunks = len(batch_chunks) + n_genomes = max(1, len(genome_list)) + slots = max(1, min(options.batchCores or 1, n_chunks)) + return (HAL2MAF_CORE_SECS_PER_CHUNK_PER_GENOME * n_genomes * n_chunks / slots + + HAL2MAF_SLOW_CHUNK_CORE_SECS_PER_GENOME * n_genomes) + + def export_file(job, file_id, out_path): """ run toil export in its own job in order to a) do it right away but b) in a separate job (in case it fails) """ job.fileStore.exportFile(file_id, makeURL(out_path)) @@ -234,11 +261,13 @@ def export_file(job, file_id, out_path): def hal2maf_workflow(job, hal_id, bed_id, options, config): - hal2maf_ranges_job = job.addChildJobFn(hal2maf_ranges, hal_id, bed_id, options, cores=1, disk=hal_id.size) + # halStats itself is seconds; on a HAL of any size this job is a HAL copy and little else + hal2maf_ranges_job = job.addChildJobFn(hal2maf_ranges, hal_id, bed_id, options, cores=1, disk=hal_id.size, + walltime=cactus_walltime(300, io_bytes=hal_id.size)) chunks, genome_list = hal2maf_ranges_job.rv(0), hal2maf_ranges_job.rv(1) - hal2maf_all_job = hal2maf_ranges_job.addFollowOnJobFn(hal2maf_all, hal_id, chunks, genome_list, options, config) + hal2maf_all_job = hal2maf_ranges_job.addFollowOnJobFn(hal2maf_all, hal_id, chunks, genome_list, options, config, walltime=cactus_walltime()) hal2maf_merge_job = hal2maf_all_job.addFollowOnJobFn(hal2maf_merge_all, hal2maf_all_job.rv(), options, genome_list, - disk=hal_id.size) + disk=hal_id.size, walltime=cactus_walltime()) hal2maf_ranges_job.addFollowOn(hal2maf_merge_job) # note: merge job also handles exporting (and some cleanup), indexing and coverage @@ -365,10 +394,14 @@ def hal2maf_all(job, hal_id, chunks, genome_list, options, config): cur_chunk = i * batch_size cur_batch_size = min(chunks_left, batch_size) if cur_batch_size: - batch_results.append(job.addChildJobFn(hal2maf_batch, hal_id, chunks[cur_chunk:cur_chunk+cur_batch_size], + batch_chunks = chunks[cur_chunk:cur_chunk+cur_batch_size] + batch_results.append(job.addChildJobFn(hal2maf_batch, hal_id, batch_chunks, genome_list, options, config, disk=math.ceil((1 + 1.5 / num_batches)*hal_id.size), cores=options.batchCores, - memory=batch_memory).rv()) + memory=batch_memory, + walltime=cactus_walltime( + hal2maf_batch_walltime(batch_chunks, genome_list, options), + io_bytes=hal_id.size)).rv()) chunks_left -= cur_batch_size assert chunks_left == 0 @@ -673,8 +706,12 @@ def hal2maf_merge_all(job, output_dicts, options, genome_list): for out_type in options.outType: maf_ids = [out_dict[out_type] for out_dict in output_dicts] maf_size = sum([maf_id.size for maf_id in maf_ids]) + # a cat of every chunk MAF, so it is pure I/O -- unless the output is TAF, which adds a + # taffy view pass over the merged file + merge_secs = taffy_walltime_secs(maf_size) if options.outputMAF.endswith(('.taf', '.taf.gz')) else 0 merge_job = job.addChildJobFn(hal2maf_merge, maf_ids, options, - disk=int(3 * maf_size)) + disk=int(3 * maf_size), + walltime=cactus_walltime(merge_secs, io_bytes=2 * maf_size)) # we export ASAP output_name, output_ext = os.path.splitext(options.outputMAF) if output_ext == '.gz': @@ -682,21 +719,30 @@ def hal2maf_merge_all(job, output_dicts, options, genome_list): output_ext += '.gz' if out_type != 'norm' and len(options.outType) > 1: output_name += '.{}'.format(out_type) - export_job = merge_job.addFollowOnJobFn(export_file, merge_job.rv(), output_name + output_ext) - merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=maf_ids) + # the merged MAF is a promise, so it has no size here; it is a concatenation of the + # chunk MAFs, so their total is the right stand-in (an upper bound once it is gzipped) + export_job = merge_job.addFollowOnJobFn(export_file, merge_job.rv(), output_name + output_ext, + walltime=cactus_walltime(0, io_bytes=2 * maf_size)) + merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=maf_ids, walltime=cactus_walltime()) if options.index: index_job = merge_job.addFollowOnJobFn(taffy_index, merge_job.rv(), output_name + output_ext, disk=int(1.1 * maf_size), - memory=cactus_clamp_memory(maf_size / 10)) - index_job.addFollowOnJobFn(export_file, index_job.rv(), output_name + output_ext + '.tai') + memory=cactus_clamp_memory(maf_size / 10), + walltime=cactus_walltime(taffy_walltime_secs(maf_size), + io_bytes=maf_size)) + index_job.addFollowOnJobFn(export_file, index_job.rv(), output_name + output_ext + '.tai', + walltime=cactus_walltime(0, io_bytes=2 * TAFFY_SIDE_OUTPUT_FRACTION * maf_size)) if options.coverage: coverage_job = merge_job.addFollowOnJobFn(taffy_coverage, merge_job.rv(), output_name + output_ext, genome_list, options, disk=int(1.1 * maf_size), - memory=cactus_clamp_memory(maf_size / 10)) - coverage_job.addFollowOnJobFn(export_file, coverage_job.rv(), output_name + output_ext + '.cov.tsv') + memory=cactus_clamp_memory(maf_size / 10), + walltime=cactus_walltime(taffy_walltime_secs(maf_size), + io_bytes=maf_size)) + coverage_job.addFollowOnJobFn(export_file, coverage_job.rv(), output_name + output_ext + '.cov.tsv', + walltime=cactus_walltime(0, io_bytes=2 * TAFFY_SIDE_OUTPUT_FRACTION * maf_size)) return export_job.rv() diff --git a/src/cactus/maf/cactus_maf2bigmaf.py b/src/cactus/maf/cactus_maf2bigmaf.py index 4ea7a16e4..28e0f2a68 100644 --- a/src/cactus/maf/cactus_maf2bigmaf.py +++ b/src/cactus/maf/cactus_maf2bigmaf.py @@ -14,12 +14,12 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import cactus_clamp_memory @@ -36,6 +36,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("mafFile", help = "MAF file to convert to BigMaf (can be gzipped)") parser.add_argument("outFile", help = "Output bigMaf file (.bb)") @@ -93,7 +94,7 @@ def main(): if options.halFile: hal_id = toil.importFile(options.halFile) - bigmaf_id_dict = toil.start(Job.wrapJobFn(maf2bigmaf_workflow, config, options, maf_id, hal_id)) + bigmaf_id_dict = toil.start(Job.wrapJobFn(maf2bigmaf_workflow, config, options, maf_id, hal_id, walltime=cactus_walltime())) #export the big maf out_bm_path = makeURL(options.outFile) @@ -109,27 +110,62 @@ def main(): logger.info("cactus-maf2bigmaf has finished after {} seconds".format(run_time)) +# How much bigger a MAF gets when it is unzipped. Not measured directly, but it is what the +# disk model below has always implied: a gzipped MAF asks for 7x its own size of local disk to +# hold the compressed input plus the uncompressed stream. +MAF_GZIP_RATIO = 6 + +# Seconds per GB of *uncompressed* MAF for one run of the UCSC/mafTools chain +# (mafDuplicateFilter | mafFilter | mafToBigMaf | sort | bedToBigBed). cactus-maf2bigmaf +# appears in none of the mined cluster logs, so this is anchored on the one measured sequential +# pass over a MAF of this kind -- taffy, at 200s per GB of *gzipped* MAF (see +# maf_chunk.TAFFY_SECS_PER_GB), which is ~35s per GB uncompressed -- and scaled up 3x for +# mafTools being slower per byte than taffy and for the external sort, which is a genuine extra +# pass over the whole file rather than another stage of the pipeline. (cactus_call builds the +# mafTools stages as one shell pipeline, so they overlap rather than running end to end.) The +# sort is the superlinear step: if these jobs start retrying, raise this rather than widening +# --walltimeFactor for the whole run. +MAF2BIGMAF_SECS_PER_UNCOMPRESSED_GB = 100 + +# hgLoadMafSummary is the heavier of the pair -- the job's own memory model says so, asking +# mem_mult 1.1 of the MAF against maf2bigmaf's maf_id.size/20. +MAF2BIGMAF_SUMMARY_SECS_PER_UNCOMPRESSED_GB = 130 + + def maf2bigmaf_workflow(job, config, options, maf_id, hal_id): - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) - check_tools_job = root_job.addChildJobFn(maf2bigmaf_check_tools) + check_tools_job = root_job.addChildJobFn(maf2bigmaf_check_tools, walltime=cactus_walltime()) genomes_list = None + # halStats is a metadata read and takes seconds; on a HAL of any size this job is a HAL copy + # and little else (same shape as hal2maf_ranges) chrom_sizes_job = check_tools_job.addFollowOnJobFn(maf2bigmaf_chrom_sizes, options, hal_id, - disk=hal_id.size) + disk=hal_id.size, + walltime=cactus_walltime(300, io_bytes=hal_id.size)) chrom_sizes_id = chrom_sizes_job.rv(0) genomes_list = chrom_sizes_job.rv(1) if options.mafFile.endswith('.gz'): disk_mult = 7 mem_mult = 1.1 + gzip_ratio = MAF_GZIP_RATIO else: disk_mult = 3 mem_mult = 0.3 + gzip_ratio = 1 + # the two jobs below stream the whole MAF through uncompressed, so that is the volume that + # sets their runtime. Not disk_mult, which is peak local disk -- input plus intermediates + # plus output -- and so counts the same bytes several times over. + uncompressed_gb = gzip_ratio * maf_id.size / 1e9 bigmaf_job = chrom_sizes_job.addFollowOnJobFn(maf2bigmaf, maf_id, chrom_sizes_id, genomes_list, options, disk=disk_mult * maf_id.size, - memory=cactus_clamp_memory(maf_id.size / 20)) + memory=cactus_clamp_memory(maf_id.size / 20), + walltime=cactus_walltime(MAF2BIGMAF_SECS_PER_UNCOMPRESSED_GB * uncompressed_gb, + io_bytes=2 * maf_id.size)) bigmaf_summary_job = chrom_sizes_job.addFollowOnJobFn(maf2bigmaf_summary, maf_id, chrom_sizes_id, genomes_list, options, disk=disk_mult * maf_id.size, - memory=cactus_clamp_memory(maf_id.size * mem_mult)) + memory=cactus_clamp_memory(maf_id.size * mem_mult), + walltime=cactus_walltime(MAF2BIGMAF_SUMMARY_SECS_PER_UNCOMPRESSED_GB * uncompressed_gb, + io_bytes=2 * maf_id.size)) return { 'bb' : bigmaf_job.rv(), 'summary.bb' : bigmaf_summary_job.rv() } diff --git a/src/cactus/maf/cactus_phast.py b/src/cactus/maf/cactus_phast.py index 51219ba5f..c3d44a65c 100644 --- a/src/cactus/maf/cactus_phast.py +++ b/src/cactus/maf/cactus_phast.py @@ -23,19 +23,20 @@ import timeit import xml.etree.ElementTree as ET -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory from cactus.shared.common import cactus_cpu_count from cactus.shared.version import cactus_commit from cactus.progressive.cactus_prepare import human2bytesN -from cactus.maf.maf_chunk import (taffy_index_job, get_ref_sequence_lengths, +from cactus.maf.maf_chunk import (taffy_walltime_secs, TAFFY_SIDE_OUTPUT_FRACTION, + taffy_index_job, get_ref_sequence_lengths, get_aligned_ref_contigs, plan_chunks, plan_chunks_in_regions, parse_bed_ranges, filter_chunks_to_indexed, @@ -49,8 +50,36 @@ from toil.lib.humanize import bytes2human +# Core-seconds of chunker_job per GB of source MAF. The chunker runs one `taffy view` per +# chunk under GNU parallel; across 18 cactus-phast runs of the VGP 577-way MAF (28 to 208 GB, +# every one of them at --chunkCores 32) it ran at 10.7 to 23.5 seconds per GB of wall time, +# i.e. 340 to 750 core-s/GB. Only 32-core runs were measured, so dividing by the core count is +# an assumption -- a conservative one at 32, and at --chunkCores 1 it correctly asks for the +# ~40 hours a single-threaded pass over a 208 GB MAF really takes. +CHUNKER_CORE_SECS_PER_GB = 700 + +# Seconds of 4d-site extraction per GB of (compressed) chunk in the group. `msa_view --4d` was +# p50 39.7s / p99 147.4s / max 193.2s across the 5024 group jobs of those 18 runs, whose groups +# ran up to ~700 MB of compressed MAF -- so this is a little above the worst rate seen. +EXTRACT_4D_SECS_PER_GB = 500 + +# Core-seconds of phyloFit per species in the tree. The 18 fits of the 577-way 4d SS took 1917 +# to 15143 s at --phyloFitCores 64, i.e. up to 969000 core-s over 577 species = 1680 core-s per +# species. EM cost is really n_species x n_4d_sites x iterations, but the SS is a promise where +# this is needed and the site count is set by the annotation rather than the tree, so the +# species count carries the scaling. +PHYLOFIT_CORE_SECS_PER_SPECIES = 2000 + +# Seconds of phyloP per MB of the (bgzipped) chunk it scores. Back-solving each run's issued +# per-chunk disk against its phyloP times over 42234 chunk runs put 15 of the 18 runs at 60-81 +# s/MB at p50/p90/p99 (hg38 73/81/77), and the single worst chunk in the corpus at 216 s/MB -- +# which cactus_walltime()'s factor covers from here. +PHYLOP_SECS_PER_MB = 100 + + def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("inMaf", help="Input alignment as produced by cactus-hal2maf. " "Accepts .maf, .maf.gz, .taf, or .taf.gz (auto-detected " @@ -415,7 +444,7 @@ def main(): toil.start(Job.wrapJobFn(phast_workflow, config, options, maf_id, tai_id, tai_built, hal_id, ann_id, model_id, - bed_id)) + bed_id, walltime=cactus_walltime())) end_time = timeit.default_timer() logger.info("cactus-phast finished in {} seconds".format(end_time - start_time)) @@ -786,7 +815,10 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # fail fast if any required external binary is missing rather than crashing # mid-workflow on a remote worker after hours of localization - check_job = job.addChildJobFn(phast_check_tools, options) + # 13 no-arg tool probes: 0.54s for all of them with the binaries on the PATH. The estimate + # is for --binariesMode docker/singularity, where each probe is a container start and the + # first one may pull the image. + check_job = job.addChildJobFn(phast_check_tools, options, walltime=cactus_walltime(300)) # Normalize the annotation up front: convert to genePred, filter to # CDS-bearing transcripts, single-cover. Only matters when we'll train. @@ -796,15 +828,26 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ if need_train and ann_id is not None: # Annotations are tiny relative to MAF/HAL; default memory is plenty # for a human-scale ncbiRefSeq.txt.gz (~10 MB compressed). + # + # gtfToGenePred was p50 7.4s / max 14.3s on a human GTF across the 18 phast runs and + # genePredSingleCover under a second; the rest is a python decompress, filter and sort of + # the same file, and the io term is the compressed annotation in and the genePred out. prep_job = check_job.addFollowOnJobFn(prep_annotation, options, ann_id, - disk=max(2 * 1024**3, int(ann_id.size * 20))) + disk=max(2 * 1024**3, int(ann_id.size * 20)), + walltime=cactus_walltime(120, io_bytes=10 * ann_id.size)) ann_id = prep_job.rv() setup_parent = prep_job else: setup_parent = check_job + # halStats is seconds even on the 577-way (--genomes 5.6s, everything else under 0.1s), so + # what this job costs is the HAL copy plus halAlignedExtract, which scans the reference's + # top segments and streams a BED of them through awk. That scan tracks the size of the + # REFERENCE, which the HAL's own size says nothing about (a two-genome HAL can have the same + # 3 Gb reference as the 577-way), so it goes in the constant rather than in io_bytes. setup_job = setup_parent.addFollowOnJobFn(phast_setup, options, hal_id, - disk=int(hal_id.size * 1.1)) + disk=int(hal_id.size * 1.1), + walltime=cactus_walltime(1800, io_bytes=hal_id.size)) species_list = setup_job.rv(0) tree_str = setup_job.rv(1) ref_seq_lengths = setup_job.rv(2) @@ -814,20 +857,23 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # Convenience sed script: maps phast canonical names back to HAL genome # names. Only relevant when at least one HAL genome name contains '.'; # the job no-ops otherwise. - setup_job.addFollowOnJobFn(export_name_map_job, options, species_list) + setup_job.addFollowOnJobFn(export_name_map_job, options, species_list, walltime=cactus_walltime()) # build .tai if missing, else just chain through the existing one if tai_built: idx_job = setup_job.addFollowOnJobFn(taffy_index_job, maf_id, os.path.basename(options.inMaf), - disk=int(maf_id.size * 1.1)) + disk=int(maf_id.size * 1.1), + walltime=cactus_walltime(taffy_walltime_secs(maf_id.size), + io_bytes=maf_id.size)) tai_id = idx_job.rv() - idx_job.addFollowOnJobFn(export_file, tai_id, tai_export_path(options)) + idx_job.addFollowOnJobFn(export_file, tai_id, tai_export_path(options), + walltime=cactus_walltime(0, io_bytes=2 * TAFFY_SIDE_OUTPUT_FRACTION * maf_id.size)) plan_parent = idx_job else: plan_parent = setup_job plan_job = plan_parent.addFollowOnJobFn(plan_chunks_job, options, ref_seq_lengths, tai_id, - bed_id, aligned_contigs) + bed_id, aligned_contigs, walltime=cactus_walltime()) chunk_specs = plan_job.rv() # The single multi-core chunker job: localizes the source MAF once, @@ -856,18 +902,23 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ # lands pre-filtered for downstream phast tools (msa_view --4d and phyloP both # need the same filtered MAF, no point doing it twice per chunk). filter_cmd = '{strip} | mafDuplicateFilter -k -m -'.format(strip=make_strip_perl_cmd()) + # io_bytes charges the source MAF in and every chunk back out again -- neither shows up in + # the parallel command's runtime, and on the 208 GB MAF that is as much time as the chunking. + chunker_secs = CHUNKER_CORE_SECS_PER_GB * (maf_id.size / 1e9) / max(1, options.chunkCores or 1) chunk_job = plan_job.addFollowOnJobFn(chunker_job, options, maf_id, tai_id, os.path.basename(options.inMaf), chunk_specs, filter_cmd, disk=chunker_disk, memory=chunker_memory, - cores=options.chunkCores) + cores=options.chunkCores, + walltime=cactus_walltime(chunker_secs, + io_bytes=2 * maf_id.size)) chunks = chunk_job.rv() # ----- phyloFit branch (also runs in phyloP mode if no model was given) ----- if need_train: train_job = chunk_job.addFollowOnJobFn(train_workflow, options, chunks, - species_list, tree_str, ann_id) + species_list, tree_str, ann_id, walltime=cactus_walltime()) # train_workflow returns the model file id trained_model_id = train_job.rv() if options.mode == 'phyloFit': @@ -890,7 +941,7 @@ def phast_workflow(job, config, options, maf_id, tai_id, tai_built, hal_id, ann_ for sub in track_inputs: score_job = score_parent.addFollowOnJobFn(phyloP_workflow, options, chunks, model_id, ref_seq_lengths, species_list, - sub, effective_root_name) + sub, effective_root_name, walltime=cactus_walltime()) track_rvs.append(score_job.rv()) return track_rvs @@ -1301,20 +1352,30 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): # projection); with --root, it's restricted to clade leaves. leaves_csv = ','.join(phast_name(g) for g in newick_leaves(tree_str)) extract_job = job.addChildJobFn(extract_4d_all, options, chunks, ann_id, - species_list, leaves_csv) + species_list, leaves_csv, walltime=cactus_walltime()) ss_results = extract_job.rv() # flat list of ss file ids (one per chunk-group) # 4d-site SS files are tiny even at 447-way (~300 MB aggregated). 4 GiB # disk covers concat scratch comfortably. + # msa_view --aggregate itself was p50 3.3s / max 8.5s over the 18 phast runs; the job's real + # cost is staging the 135-2414 per-group SS files. ss_results is a promise here, so that + # goes in as a constant -- the 4 GiB disk above is the bound on how much there can be. aggregate_job = extract_job.addFollowOnJobFn(aggregate_4d, options, ss_results, species_list, tree_str, - disk=4 * 1024**3) + disk=4 * 1024**3, + walltime=cactus_walltime(300, io_bytes=2 * 1024**3)) aggregate_id = aggregate_job.rv() - aggregate_job.addFollowOnJobFn(export_file, aggregate_id, ss_export_path(options)) + aggregate_job.addFollowOnJobFn(export_file, aggregate_id, ss_export_path(options), walltime=cactus_walltime()) # phyloFit on 577-way × 1.4 GB SS peaked at ~300 MiB; default memory is # fine unless the user overrides via --phyloFitMemory. - fit_kwargs = dict(disk=4 * 1024**3, cores=options.phyloFitCores) + # + # The thread divisor is capped at 64 because that is the only setting the 18 runs used, and + # phast's scaling above it is unmeasured. + fit_secs = (PHYLOFIT_CORE_SECS_PER_SPECIES * max(1, len(species_list)) + / max(1, min(options.phyloFitCores or 1, 64))) + fit_kwargs = dict(disk=4 * 1024**3, cores=options.phyloFitCores, + walltime=cactus_walltime(fit_secs)) if options.phyloFitMemory: fit_kwargs['memory'] = cactus_clamp_memory(options.phyloFitMemory) fit_job = aggregate_job.addFollowOnJobFn(phyloFit_job, options, aggregate_id, @@ -1322,14 +1383,16 @@ def train_workflow(job, options, chunks, species_list, tree_str, ann_id): model_id = fit_job.rv() if options.modFreqs: + # modFreqs was 17 ms at worst; a phast .mod is 30 KB even at 577 species mf_job = fit_job.addFollowOnJobFn(mod_freqs_job, options, model_id, - disk=2 * 1024**3) + disk=2 * 1024**3, + walltime=cactus_walltime()) model_id = mf_job.rv() export_parent = mf_job else: export_parent = fit_job - export_parent.addFollowOnJobFn(export_file, model_id, model_export_path(options)) + export_parent.addFollowOnJobFn(export_file, model_id, model_export_path(options), walltime=cactus_walltime()) return model_id @@ -1356,9 +1419,15 @@ def extract_4d_all(job, options, chunks, ann_id, species_list, leaves_csv): # tracks the aligned-column count over the (small) CDS subset, which # is bounded by the reference-coordinate window per group. Observed # ~700 MiB on a 577-way ~10 Mb chunk-group; default memory is fine. + # + # Walltime: the 120s on top of the rate is the python gunzip+cat of the group's chunks, + # which no command timing sees. rvs.append(job.addChildJobFn(extract_4d_chunk_group, options, group, ann_id, leaves_csv, - disk=per_disk, cores=1).rv()) + disk=per_disk, cores=1, + walltime=cactus_walltime( + EXTRACT_4D_SECS_PER_GB * (group_compressed / 1e9) + 120, + io_bytes=group_compressed + ann_id.size)).rv()) return rvs @@ -1571,7 +1640,7 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis else: RealtimeLogger.info('phyloP track: global conservation (no --subtree)') score_job = job.addChildJobFn(phyloP_all, options, chunks, model_id, species_list, - track_subtree) + track_subtree, walltime=cactus_walltime()) per_chunk_wigs = score_job.rv() # flat list of (contig, start, wig_id_or_None) per chunk # disk estimates: per-base wig text is ~10 bytes/ref_bp uncompressed; bgzip @@ -1582,11 +1651,21 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis # bgzip on the concatenated wig is CPU-bound and parallelizable. 8 cores is # enough to be much faster than single-threaded without monopolizing a node. merge_cores = min(options.chunkCores or 8, 8) + # bgzip of the merged wig took 129s on hg38's 3.1 Gb reference at 8 threads (p50 7.4s, max + # 162.7s over the 18 runs); the rest of the job is catFiles pulling the 1000-4800 per-chunk + # wigs in and writing the merged one back, which io_bytes charges at ~10 bytes of wig text + # per reference base each way. + merge_secs = 5e-8 * total_ref_bp * 8.0 / merge_cores merge_job = score_job.addFollowOnJobFn(phyloP_merge, options, per_chunk_wigs, disk=merge_disk, - cores=merge_cores) + cores=merge_cores, + walltime=cactus_walltime(merge_secs, + io_bytes=20 * total_ref_bp)) wig_id = merge_job.rv() - merge_job.addFollowOnJobFn(export_file, wig_id, wig_export_path(options, track_subtree)) + # the merged wig is a promise, but its size follows the reference: ~10 bytes/bp of wig text, + # which bgzip takes down to roughly a byte a base (see the disk estimate above) + merge_job.addFollowOnJobFn(export_file, wig_id, wig_export_path(options, track_subtree), + walltime=cactus_walltime(0, io_bytes=2 * total_ref_bp)) if options.bigwig: # decompressed wig + .bw output + bbiFile scratch @@ -1597,12 +1676,18 @@ def phyloP_workflow(job, options, chunks, model_id, ref_seq_lengths, species_lis # leaves comfortable headroom; floor at 8 GiB so smaller tests don't # over-request. bw_mem = max(8 * 1024**3, 24 * total_ref_bp) + # wigToBigWig was 1461.5s on hg38's 3.1 Gb reference (p50 534s over the 18 runs), i.e. + # 4.7e-7 s per reference base; the wig it reads is ~10 bytes/ref_bp and the .bw it writes + # back is only megabytes. bw_job = merge_job.addFollowOnJobFn(wig_to_bigwig_job, options, wig_id, ref_seq_lengths, disk=bw_disk, memory=cactus_clamp_memory(bw_mem), - cores=merge_cores) + cores=merge_cores, + walltime=cactus_walltime(6e-7 * total_ref_bp, + io_bytes=10 * total_ref_bp)) bw_job.addFollowOnJobFn(export_file, bw_job.rv(), - bigwig_export_path(options, track_subtree)) + bigwig_export_path(options, track_subtree), + walltime=cactus_walltime(0, io_bytes=2 * total_ref_bp)) return wig_id @@ -1620,7 +1705,10 @@ def phyloP_all(job, options, chunks, model_id, species_list, subtree): # memory is fine. rvs.append(job.addChildJobFn(phyloP_chunk, options, chunk_spec, model_id, subtree, - disk=per_disk, cores=1).rv()) + disk=per_disk, cores=1, + walltime=cactus_walltime( + PHYLOP_SECS_PER_MB * (chunk_id.size / 1e6), + io_bytes=chunk_id.size)).rv()) return rvs diff --git a/src/cactus/maf/maf_chunk.py b/src/cactus/maf/maf_chunk.py index 301fcc85a..1366f0fe7 100644 --- a/src/cactus/maf/maf_chunk.py +++ b/src/cactus/maf/maf_chunk.py @@ -32,6 +32,33 @@ from cactus.shared.common import cactus_call +# One sequential taffy pass over a MAF, as seconds plus seconds per GB. Fitted to the 94 +# taffy index and taffy coverage jobs of the VGP 577-way MAF export, whose MAFs ran from 4.5 to +# 351 GB and whose runtimes ran from 16 minutes to 19 hours. The intercept is what makes this +# work: cost per GB is far higher on a small MAF (169 s/GB at the median against 608 at the +# worst), so a slope-only model tuned to cover the small ones asks three times too much of the +# big ones. These values leave every one of the 94 inside its request once cactus_walltime()'s +# factor is applied. taffy view gets the same treatment -- it is the same single pass. +TAFFY_SECS_BASE = 3000 +TAFFY_SECS_PER_GB = 200 + +# Smallest MAF in that fit. Below it the intercept is pure extrapolation and would hand an +# evolver-sized test MAF the better part of an hour, so it is tapered to zero. +TAFFY_RAMP_GB = 4.5 + + +def taffy_walltime_secs(maf_bytes): + """ estimated seconds for one sequential taffy pass (index, coverage, view) over a MAF """ + maf_gb = maf_bytes / 1e9 + ramp = min(1.0, maf_gb / TAFFY_RAMP_GB) if TAFFY_RAMP_GB > 0 else 1.0 + return TAFFY_SECS_BASE * ramp + TAFFY_SECS_PER_GB * maf_gb + +# The .tai index and .cov.tsv that taffy writes beside a MAF are small next to the MAF itself: +# the largest coverage table of that export was 560 MB against a 386 GB MAF. Jobs that only +# move one of those get a size in this proportion rather than the MAF's own. +TAFFY_SIDE_OUTPUT_FRACTION = 0.01 + + def taffy_index_job(job, maf_id, maf_basename): """ Build a .tai for the given source MAF. Single sequential pass. """ work_dir = job.fileStore.getLocalTempDir() diff --git a/src/cactus/paf/local_alignment.py b/src/cactus/paf/local_alignment.py index aa0d540e7..088811f41 100755 --- a/src/cactus/paf/local_alignment.py +++ b/src/cactus/paf/local_alignment.py @@ -20,12 +20,53 @@ import copy from Bio import SeqIO from cactus.paf.paf import get_event_pairs, get_leaves, get_node, get_distances -from cactus.shared.common import cactus_call, getOptionalAttrib, zip_gz +from cactus.shared.common import cactus_call, getOptionalAttrib, zip_gz, cactus_walltime from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.preprocessor.unmasking import unmask_contigs_all from cactus.preprocessor.cactus_preprocessor import clean_if_different from cactus.shared.common import cactus_clamp_memory +def get_divergence_class(distance, params): + """ The bucket a pair at the given distance falls in ("one" through + "five", or "default" for anything more diverged). It selects the lastz parameters, and with + them most of the runtime, so the walltime estimate keys off the same bucket. """ + divergences = params.find("constants").find("divergences") + if getOptionalAttrib(divergences, 'useDefault', typeFn=bool, default=False): + return "default" + for i in "one", "two", "three", "four", "five": + if distance <= float(divergences.attrib[i]): + return i + return "default" + + +def get_lastz_walltime(distance, params, chunk_a, chunk_b): + """ Estimated seconds for one lastz/kegalign job on this pair of chunks. + + holds the per-divergence-class time for a pair of full-sized chunks; + we scale it by how big these two chunks actually are, so a small genome (whose whole + sequence is one short chunk) gets a short time rather than the 577-way figure. + + The table was measured on lastz, and is used for whichever mapper is configured: minimap2 + and FastGA are both faster, so it is conservative for them rather than wrong. """ + lastz_params_node = params.find("blast") + walltime_node = lastz_params_node.find("lastzWalltime") + if walltime_node is None: + return None + lastz_class = get_divergence_class(distance, params) + base = getOptionalAttrib(walltime_node, lastz_class, typeFn=float, default=None) + if base is None: + return None + gpu = getOptionalAttrib(lastz_params_node, 'gpu', typeFn=int, default=0) + fastga = getOptionalAttrib(lastz_params_node, 'mapper', typeFn=str) == 'fastga' + chunk_attr = 'bigChunkSize' if gpu or fastga else 'chunkSize' + chunk_size = getOptionalAttrib(lastz_params_node, chunk_attr, typeFn=float, default=None) + if chunk_size and chunk_size > 0: + # the table was measured on a pair of full chunks; lastz time is roughly linear in the + # sequence it is handed, so a pair that is half that size gets half the time + base *= (chunk_a.size + chunk_b.size) / (2.0 * chunk_size) + return base + + def run_lastz(job, name_A, genome_A, name_B, genome_B, distance, params): # Create a local temporary file to put the alignments in. work_dir = job.fileStore.getLocalTempDir() @@ -43,12 +84,8 @@ def run_lastz(job, name_A, genome_A, name_B, genome_B, distance, params): cpu = getOptionalAttrib(lastz_params_node, 'cpu', typeFn=int, default=None) lastz_divergence_node = lastz_params_node.find("kegalignArguments" if gpu else "lastzArguments") divergences = params.find("constants").find("divergences") - lastz_params = lastz_divergence_node.attrib["default"] + lastz_params = lastz_divergence_node.attrib[get_divergence_class(distance, params)] if not getOptionalAttrib(divergences, 'useDefault', typeFn=bool, default=False): - for i in "one", "two", "three", "four", "five": - if distance <= float(divergences.attrib[i]): - lastz_params = lastz_divergence_node.attrib[i] - break logger.info("For distance {} for genomes {}, {} using {} lastz parameters".format(distance, genome_A, genome_B, lastz_params)) if gpu: @@ -217,7 +254,7 @@ def run_fastga(job, name_A, genome_A, name_B, genome_B, distance, params): outfile=unaligned_fasta_b_file, job_memory=job.memory) # flip back to the lastz aligner - root = Job() + root = Job(walltime=cactus_walltime()) job.addChild(root) params_lastz = copy.deepcopy(params) params_lastz.find('blast').attrib['mapper'] = 'lastz' @@ -227,17 +264,32 @@ def run_fastga(job, name_A, genome_A, name_B, genome_B, distance, params): lastz_job = root.addChildJobFn(make_chunked_alignments, name_A, unaligned_fasta_a_id, name_B, unaligned_fasta_b_id, distance, params_lastz, memory=chunked_alignment_memory(unaligned_fasta_a_id, unaligned_fasta_b_id, - params_lastz)) + params_lastz), + walltime=chunked_alignment_walltime(unaligned_fasta_a_id, + unaligned_fasta_b_id)) # run paffy dechunk a second time, since they contigs were chunked both by extract and chunk - dechunk_job = root.addFollowOnJobFn(combine_chunks, [lastz_job.rv()], 1) + # this is one dechunk of a whole paf rather than of a chunk of one, and the worst such call + # in the 577-way blast logs took 600s. The paf is a promise, so the bytes it stages are + # proxied by the unaligned sequence that went into it. + unaligned_size = unaligned_fasta_a_id.size + unaligned_fasta_b_id.size + dechunk_job = root.addFollowOnJobFn(combine_chunks, [lastz_job.rv()], 1, + walltime=cactus_walltime(300, io_bytes=4 * unaligned_size)) # merge the fastga and lastz alignments together - merge_job = dechunk_job.addFollowOnJobFn(merge_alignments, job.fileStore.writeGlobalFile(alignment_file), dechunk_job.rv()) + # (merge_alignments without has_resources only re-schedules itself, so it is coordination) + merge_job = dechunk_job.addFollowOnJobFn(merge_alignments, job.fileStore.writeGlobalFile(alignment_file), dechunk_job.rv(), + walltime=cactus_walltime()) if getOptionalAttrib(params.find('blast'), 'fastga_stats', typeFn=bool, default=False): merge_job.addFollowOnJobFn(log_paf_stats, merge_job.rv(), name_A, genome_A, name_B, genome_B, distance, 'after-lastz', memory=cactus_clamp_memory(2 * (genome_A.size + genome_B.size)), - disk=2*(genome_A.size + genome_B.size)) + disk=2*(genome_A.size + genome_B.size), + # paffy view streams the whole paf against both genomes, the + # same shape as paffy to_bed, which ran at 99 s/GB of paf at + # p99 in the 577-way blast logs. The paf is a promise here, + # so the estimate scales off the genomes instead. + walltime=cactus_walltime(120 * (genome_A.size + genome_B.size) / 1e9, + io_bytes=3 * (genome_A.size + genome_B.size))) return merge_job.rv() @@ -337,18 +389,32 @@ def log_paf_stats(job, alignment, name_A, genome_A, name_B, genome_B, distance, stats = "EMPTY" RealtimeLogger.info('paf-stats\t{}\t{}\t{}\t{}\t{}'.format(tag, name_B, name_A, distance, '\t'.join(stats.strip().split()))) +# Seconds for one paffy dechunk call, of which combine_chunks makes one per chunk paf it is +# handed. The 577-way blast logs have 49536 of them averaging 0.5s (p90 0.5s, p99 7.8s), so a +# second a chunk is twice the mean before --walltimeFactor. What makes these jobs slow is +# downloading the chunks, not dechunking them, and that is the io_bytes term. +DECHUNK_SECS_PER_CHUNK = 1 + + def combine_chunks(job, chunked_alignment_files, batch_size): if len(chunked_alignment_files) >= 2 * batch_size: # run combine_chunks in batches - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) batch_results = [] for chunk_idx in range(math.ceil(len(chunked_alignment_files) / batch_size)): batch = chunked_alignment_files[chunk_idx * batch_size : chunk_idx * batch_size + batch_size] + batch_bytes = sum([f.size for f in batch]) batch_results.append(root_job.addChildJobFn(combine_chunks, batch, batch_size, - disk=sum([f.size for f in batch])).rv()) + disk=batch_bytes, + walltime=cactus_walltime(DECHUNK_SECS_PER_CHUNK * len(batch), + io_bytes=2 * batch_bytes)).rv()) + # merge_combined_chunks runs no command at all -- it is a copyfileobj of each batch result -- + # so its time is the staging, which was 23 GB at p90 in the 577-way logs + total_bytes = sum([f.size for f in chunked_alignment_files]) return root_job.addFollowOnJobFn(merge_combined_chunks, batch_results, - disk=2*sum([f.size for f in chunked_alignment_files])).rv() + disk=2*total_bytes, + walltime=cactus_walltime(30, io_bytes=2 * total_bytes)).rv() else: # Make combined alignments file alignment_file = job.fileStore.getLocalTempFile() @@ -393,6 +459,21 @@ def chunked_alignment_memory(genome_a, genome_b, params): genome_a.size, genome_b.size)) +# Seconds of faffy chunk per GB of input fasta. The 577-way blast logs have 8040 chunk calls +# (two per make_chunked_alignments) topping out at 174s, against genomes of up to 10 Gb -- about +# 17 s/Gb at the worst. +CHUNK_SECS_PER_GB = 20 + + +def chunked_alignment_walltime(genome_a, genome_b): + """ Walltime for a make_chunked_alignments job: one faffy chunk pass over each genome, then a + writeGlobalFile per chunk. Both terms scale with the two genomes, which are real FileIDs at + every call site -- where unmasking has replaced them with promises, pass the pre-unmask sizes, + which are the same bytes since unmasking only changes case. """ + seq_size = genome_a.size + genome_b.size + return cactus_walltime(CHUNK_SECS_PER_GB * seq_size / 1e9, io_bytes=2 * seq_size) + + def make_chunked_alignments(job, event_a, genome_a, event_b, genome_b, distance, params): lastz_params_node = params.find("blast") gpu = getOptionalAttrib(lastz_params_node, 'gpu', typeFn=int, default=0) @@ -432,10 +513,17 @@ def make_chunks(genome): cores=lastz_cores, disk=max(4*(chunk_a.size+chunk_b.size), memory), memory=cactus_clamp_memory(memory), - accelerators=accelerators).rv()) + accelerators=accelerators, + walltime=cactus_walltime(get_lastz_walltime(distance, params, chunk_a, chunk_b))).rv()) dechunk_batch_size = getOptionalAttrib(lastz_params_node, 'dechunkBatchSize', typeFn=int, default=1e9) - return job.addFollowOnJobFn(combine_chunks, chunked_alignment_files, dechunk_batch_size).rv() # Combine the chunked alignment files + # combine_chunks either dechunks the chunk pafs itself or fans them out in batches, so it never + # runs more than a batch's worth of dechunks. The chunk pafs are promises here, so the bytes it + # stages are proxied by the sequence they came from (the paf is about 2x the sequence at p90). + dechunks = min(len(chunked_alignment_files), 2 * dechunk_batch_size) + return job.addFollowOnJobFn(combine_chunks, chunked_alignment_files, dechunk_batch_size, + walltime=cactus_walltime(DECHUNK_SECS_PER_CHUNK * dechunks, + io_bytes=8 * (genome_a.size + genome_b.size))).rv() # Combine the chunked alignment files def invert_alignments(job, alignment_file): @@ -451,21 +539,27 @@ def invert_alignments(job, alignment_file): def make_ingroup_to_outgroup_alignments_0(job, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # Generate the alignments fle alignment_file = job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, - event_names_to_sequences, distances, params).rv() + event_names_to_sequences, distances, params, walltime=cactus_walltime()).rv() # Invert the final alignment so that the query is the outgroup and the target is the ingroup. # paffy invert holds the alignment and its inverse on local disk at once. The alignment's size # is a promise here, so estimate it from the sequence that went into it: on a repeat-rich # genome the ingroup-to-outgroups paf runs to several times the sequence, and this job # otherwise falls back on the 2 Gi default and overruns it by orders of magnitude. - alignment_disk = 10 * (event_names_to_sequences[ingroup_event.iD].size + - sum(event_names_to_sequences[outgroup.iD].size for outgroup in outgroup_events)) - return job.addFollowOnJobFn(invert_alignments, alignment_file, disk=alignment_disk).rv() + alignment_seq_size = (event_names_to_sequences[ingroup_event.iD].size + + sum(event_names_to_sequences[outgroup.iD].size for outgroup in outgroup_events)) + alignment_disk = 10 * alignment_seq_size + # The walltime uses the same proxy for the same reason: paffy invert ran at 65 s/GB of paf at + # p90 across the 1726 inverts in the 577-way blast logs, and the paf is about 2x its sequence + # at p90, so ~150 s/GB of sequence. + return job.addFollowOnJobFn(invert_alignments, alignment_file, disk=alignment_disk, + walltime=cactus_walltime(150 * alignment_seq_size / 1e9, + io_bytes=4 * alignment_seq_size)).rv() def make_ingroup_to_outgroup_alignments_1(job, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # a job should never set its own follow-on, so we hang everything off root_job here to encapsulate - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # align ingroup to first outgroup to produce paf alignments @@ -476,7 +570,9 @@ def make_ingroup_to_outgroup_alignments_1(job, ingroup_event, outgroup_events, e ingroup_event.iD, event_names_to_sequences[ingroup_event.iD], distances[ingroup_event, outgroup], params, memory=chunked_alignment_memory(event_names_to_sequences[outgroup.iD], event_names_to_sequences[ingroup_event.iD], params), - disk=4*(event_names_to_sequences[ingroup_event.iD].size+event_names_to_sequences[outgroup.iD].size)).rv() + disk=4*(event_names_to_sequences[ingroup_event.iD].size+event_names_to_sequences[outgroup.iD].size), + walltime=chunked_alignment_walltime(event_names_to_sequences[outgroup.iD], + event_names_to_sequences[ingroup_event.iD])).rv() # post process the alignments and recursively generate alignments to remaining outgroups # Memory: paffy to_bed creates SequenceCountArray (2 bytes per base) for query (ingroup) sequences @@ -485,14 +581,20 @@ def make_ingroup_to_outgroup_alignments_1(job, ingroup_event, outgroup_events, e return root_job.addFollowOnJobFn(make_ingroup_to_outgroup_alignments_2, alignment, ingroup_event, outgroup_events[1:], event_names_to_sequences, distances, params, disk=4*(ingroup_size + outgroup_size), - memory=cactus_clamp_memory(4*ingroup_size + 2*outgroup_size)).rv() if len(outgroup_events) > 1 else alignment + memory=cactus_clamp_memory(4*ingroup_size + 2*outgroup_size), + # _2 runs paffy to_bed over the ingroup-to-outgroup paf and then + # faffy extract over the ingroup fasta: 43 s/GB of (ingroup + + # outgroup) sequence at p99 in the 577-way blast logs, on top of + # staging the paf and the fasta. The paf is a promise here. + walltime=cactus_walltime(100 * (ingroup_size + outgroup_size) / 1e9, + io_bytes=3 * (ingroup_size + outgroup_size))).rv() if len(outgroup_events) > 1 else alignment def make_ingroup_to_outgroup_alignments_2(job, alignments, ingroup_event, outgroup_events, event_names_to_sequences, distances, params): # a job should never set its own follow-on, so we hang everything off root_job here to encapsulate - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # identify all ingroup sub-sequences that remain unaligned longer than a threshold as follows: @@ -524,10 +626,12 @@ def make_ingroup_to_outgroup_alignments_2(job, alignments, ingroup_event, outgro # recursively make alignments with the remaining outgroups alignments2 = root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_1, ingroup_event, outgroup_events, - event_names_to_sequences, distances, params).rv() + event_names_to_sequences, distances, params, walltime=cactus_walltime()).rv() + # _3 with has_resources unset only re-schedules itself once the two pafs have resolved, so it + # is coordination; the instance that does the merging is sized below return root_job.addFollowOnJobFn(make_ingroup_to_outgroup_alignments_3, ingroup_event, event_names_to_sequences[ingroup_event.iD], - alignments, alignments2).rv() + alignments, alignments2, walltime=cactus_walltime()).rv() def make_ingroup_to_outgroup_alignments_3(job, ingroup_event, ingroup_seq_file, alignments, alignments2, has_resources=False): @@ -535,8 +639,12 @@ def make_ingroup_to_outgroup_alignments_3(job, ingroup_event, ingroup_seq_file, if not has_resources: # unpack promises for disk requirement + paf_size = alignments.size + alignments2.size + # a paffy dechunk of alignments2 then a cat of both. cat measured 10 s/GB of paf at p99 in + # the 577-way blast logs, so staging the two pafs in and the merged one out is the bigger half return job.addChildJobFn(make_ingroup_to_outgroup_alignments_3, ingroup_event, ingroup_seq_file, alignments, - alignments2, has_resources=True, disk=3*(alignments.size + alignments2.size)).rv() + alignments2, has_resources=True, disk=3*paf_size, + walltime=cactus_walltime(30 * paf_size / 1e9, io_bytes=3 * paf_size)).rv() alignments = job.fileStore.readGlobalFile(alignments) # Copy the global alignment files locally alignments2 = job.fileStore.readGlobalFile(alignments2) @@ -561,8 +669,12 @@ def merge_alignments(job, alignment_file1, alignment_file2, has_resources=False) """" Merge together two alignment files """ if not has_resources: # unpack promises for disk requirement + paf_size = alignment_file1.size + alignment_file2.size + # one cat, so this is almost all staging: cat itself was 10 s/GB of paf at p99 in the + # 577-way blast logs against 30 s/GB for reading both pafs and writing the merge return job.addChildJobFn(merge_alignments, alignment_file1, alignment_file2, has_resources=True, - disk = 2 * (alignment_file1.size + alignment_file2.size)).rv() + disk = 2 * paf_size, + walltime=cactus_walltime(20 * paf_size / 1e9, io_bytes=3 * paf_size)).rv() # Get a temporary directory to work in work_dir = job.fileStore.getLocalTempDir() @@ -606,7 +718,14 @@ def chain_alignments_splitting_ingroups_and_outgroups(job, ingroup_alignment_fil total_sequence_size=total_sequence_size, disk=6 * ingroup_size, memory=cactus_clamp_memory(2 * min(ingroup_size, - chain_split_min_size))).rv() + chain_split_min_size)), + # concat every input paf, paffy invert it (65 s/GB + # of paf at p99 in the 577-way blast logs) and + # paffy split_file the result (~69 s/GB, taken + # from the measured paffy filter, which is the + # closest thing to it we have timings for) + walltime=cactus_walltime(200 * ingroup_size / 1e9, + io_bytes=4 * ingroup_size)).rv() # Separately pick the primary of the outgroups to the ingroups. By setting include_inverted_alignments=False # we only get outgroup-to-ingroup alignments and not imgroup-to-ouygroup alignments and therefore primary @@ -617,13 +736,20 @@ def chain_alignments_splitting_ingroups_and_outgroups(job, ingroup_alignment_fil total_sequence_size=total_sequence_size, disk=6 * outgroup_size, memory=cactus_clamp_memory(2 * min(outgroup_size, - chain_split_min_size))).rv() + chain_split_min_size)), + # same pipeline as the ingroup side above, minus + # the paffy invert + walltime=cactus_walltime(120 * outgroup_size / 1e9, + io_bytes=3 * outgroup_size)).rv() # Calculate approximately total alignment file size total_file_size = sum(alignment_file.size for alignment_file in ingroup_alignment_files + outgroup_alignment_files) # Merge the resulting two alignment files into a single set of alignments - return job.addFollowOnJobFn(merge_alignments, chained_ingroup_alignments, chained_outgroup_alignments).rv() + # (both inputs are promises, so this is merge_alignments' coordination instance -- it schedules + # the sized one, which is where the merging happens) + return job.addFollowOnJobFn(merge_alignments, chained_ingroup_alignments, chained_outgroup_alignments, + walltime=cactus_walltime()).rv() def concat_global_files(job, file_ids, output_path): """Download and concatenate file_ids into output_path, deleting each from the job store after use.""" @@ -635,6 +761,13 @@ def concat_global_files(job, file_ids, output_path): job.fileStore.deleteGlobalFile(file_id) +# Seconds per GB of input paf for the whole chain -> tile -> trim -> filter -> rechain -> filter +# pipeline of chain_tile_trim_filter_one_contig. Pairing per-command seconds against paf bytes in +# 444 of the 577-way blast logs gives 240 s/GB at p90, 351 at p99 and 722 at the worst -- and that +# worst case is a 1.7 GB input, i.e. it is the fixed cost showing through rather than a slow rate. +CHAIN_SECS_PER_GB = 400 + + def chain_alignments(job, alignment_files, alignment_names, reference_event_name, params, include_inverted_alignments=True, total_sequence_size=0): """Merge all input PAFs, split by query contig, run the full chain/tile/trim/filter @@ -663,7 +796,8 @@ def chain_alignments(job, alignment_files, alignment_names, reference_event_name return job.addChildJobFn( chain_tile_trim_filter_one_contig, merged_file_id, reference_event_name, params, disk=4 * merged_size, - memory=cactus_clamp_memory(4 * merged_size) + memory=cactus_clamp_memory(4 * merged_size), + walltime=cactus_walltime(CHAIN_SECS_PER_GB * merged_size / 1e9, io_bytes=2 * merged_size) ).rv() # Large input: split by query contig and run per-contig jobs in parallel @@ -682,10 +816,13 @@ def chain_alignments(job, alignment_files, alignment_names, reference_event_name processed_rvs.append( job.addChildJobFn(chain_tile_trim_filter_one_contig, split_file_id, reference_event_name, params, disk=4 * split_size, - memory=cactus_clamp_memory(4 * split_size)).rv() + memory=cactus_clamp_memory(4 * split_size), + walltime=cactus_walltime(CHAIN_SECS_PER_GB * split_size / 1e9, + io_bytes=2 * split_size)).rv() ) - return job.addFollowOnJobFn(merge_processed_alignments, processed_rvs, disk=2 * merged_size).rv() + return job.addFollowOnJobFn(merge_processed_alignments, processed_rvs, disk=2 * merged_size, + walltime=cactus_walltime(0, io_bytes=2 * merged_size)).rv() def chain_tile_trim_filter_one_contig(job, split_file_id, reference_event_name, params): @@ -769,19 +906,23 @@ def merge_processed_alignments(job, processed_file_ids): def sanitize_then_make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancestor_event_string, params, output_path=None): - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, event_names_to_sequences) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, event_names_to_sequences, walltime=cactus_walltime()) paf_job = sanitize_job.addFollowOnJobFn(make_paf_alignments, event_tree_string, sanitize_job.rv(), - ancestor_event_string, params) + ancestor_event_string, params, walltime=cactus_walltime()) # gzip the output if requested if output_path and output_path.endswith('.gz'): - gzip_job = paf_job.addFollowOnJobFn(zip_gz, output_path, paf_job.rv()) + # gzipping the final paf: one per run in the 577-way blast logs, p50 165s, p99 575s, max + # 788s. A constant rather than a formula because the only input here is paf_job.rv(), + # a promise, which has no size until it resolves. + gzip_job = paf_job.addFollowOnJobFn(zip_gz, output_path, paf_job.rv(), + walltime=cactus_walltime(900)) return gzip_job.rv() return paf_job.rv() def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancestor_event_string, params): # a job should never set its own follow-on, so we hang everything off the root_job here to encapsulate - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) logger.info("Parsing species tree: {}".format(event_tree_string)) @@ -819,10 +960,10 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest if getOptionalAttrib(lastz_params_node.find("unmask"), 'action', typeFn=str, default='none') != 'none': ingroups = [ingroup.iD for ingroup in ingroup_events] # Pass a copy of event_names_to_sequences to unmask_job to avoid circular reference - unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params) + unmask_job = root_job.addChildJobFn(unmask_contigs_all, input_sequence_map, ingroups, params, walltime=cactus_walltime()) for i,ingroup in enumerate(ingroups): event_names_to_sequences[ingroup] = unmask_job.rv(i) - new_root_job = Job() + new_root_job = Job(walltime=cactus_walltime()) root_job.addFollowOn(new_root_job) root_job = new_root_job @@ -841,7 +982,9 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest # pre-unmask size is the right number anyway. memory=chunked_alignment_memory(input_sequence_map[ingroup.iD], input_sequence_map[ingroup2.iD], params), - disk=2*total_sequence_size).rv()) + disk=2*total_sequence_size, + walltime=chunked_alignment_walltime(input_sequence_map[ingroup.iD], + input_sequence_map[ingroup2.iD])).rv()) ingroup_alignment_names.append('{}-{}_vs_{}'.format(ancestor_event_string, ingroup.iD, ingroup2.iD)) distances = get_distances(event_tree) # Distances between all pairs of nodes @@ -854,7 +997,7 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest # for each ingroup make alignments to the outgroups if int(params.find("blast").attrib["trimIngroups"]): # Trim the ingroup sequences outgroup_alignments = [root_job.addChildJobFn(make_ingroup_to_outgroup_alignments_0, ingroup, outgroup_events, - dict(event_names_to_sequences), distances, params).rv() + dict(event_names_to_sequences), distances, params, walltime=cactus_walltime()).rv() for ingroup in ingroup_events] if len(outgroup_events) > 0 else [] else: outgroup_alignments = [root_job.addChildJobFn(make_chunked_alignments, @@ -864,7 +1007,9 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest distances[ingroup, outgroup], params, memory=chunked_alignment_memory(input_sequence_map[ingroup.iD], input_sequence_map[outgroup.iD], params), - disk=2*total_sequence_size).rv() + disk=2*total_sequence_size, + walltime=chunked_alignment_walltime(input_sequence_map[ingroup.iD], + input_sequence_map[outgroup.iD])).rv() for ingroup in ingroup_events for outgroup in outgroup_events] # for better logs outgroup_alignment_names = ['{}-og_{}'.format(ancestor_event_string, i) for i in range(len(outgroup_alignments))] @@ -877,18 +1022,23 @@ def make_paf_alignments(job, event_tree_string, event_names_to_sequences, ancest ingroup_alignments, ingroup_alignment_names, outgroup_alignments, outgroup_alignment_names, ancestor_event_string, params, - total_sequence_size=total_sequence_size).rv() + total_sequence_size=total_sequence_size, walltime=cactus_walltime()).rv() # Delete the unmasked fastas (todo: should we do the unmasking somewhere further upstream?) for ingroup in ingroup_events: - root_job.addFollowOnJobFn(clean_if_different, event_names_to_sequences[ingroup.iD], input_sequence_map[ingroup.iD]) + root_job.addFollowOnJobFn(clean_if_different, event_names_to_sequences[ingroup.iD], input_sequence_map[ingroup.iD], walltime=cactus_walltime()) return root_job.addFollowOnJobFn(chain_alignments, ingroup_alignments + outgroup_alignments, ingroup_alignment_names + outgroup_alignment_names, ancestor_event_string, params, total_sequence_size=total_sequence_size, disk=6 * total_sequence_size, - memory=cactus_clamp_memory(2 * total_sequence_size)).rv() + memory=cactus_clamp_memory(2 * total_sequence_size), + # every alignment file here is a promise, so the sequence is the + # only size in scope: 600 s/GB of it covers the concat, invert + # and split_file at the p99 paf-to-sequence ratio of about 7 + walltime=cactus_walltime(600 * total_sequence_size / 1e9, + io_bytes=8 * total_sequence_size)).rv() def trim_unaligned_sequences(job, sequences, alignments, params, has_resources=False): @@ -897,9 +1047,14 @@ def trim_unaligned_sequences(job, sequences, alignments, params, has_resources=F if not has_resources: seq_size = sum([seq.size for seq in sequences]) + # paffy to_bed --includeInverted over the paf, which is the heavier both-ends variant of the + # 99 s/GB-of-paf p99 measured in the 577-way blast logs, then one faffy extract per sequence + # and a paffy upconvert, both of which stream at a few s/GB return job.addChildJobFn(trim_unaligned_sequences, sequences, alignments, params, has_resources=True, disk=4*seq_size + 2*alignments.size, - memory=cactus_clamp_memory(4*seq_size + 2*alignments.size)).rv() + memory=cactus_clamp_memory(4*seq_size + 2*alignments.size), + walltime=cactus_walltime(150 * alignments.size / 1e9 + 10 * seq_size / 1e9, + io_bytes=2 * seq_size + 2 * alignments.size)).rv() work_dir = job.fileStore.getLocalTempDir() alignments_file = os.path.join(work_dir, 'alignments.paf') diff --git a/src/cactus/pipeline/cactus_workflow.py b/src/cactus/pipeline/cactus_workflow.py index 3e4827c22..494199dd0 100644 --- a/src/cactus/pipeline/cactus_workflow.py +++ b/src/cactus/pipeline/cactus_workflow.py @@ -20,6 +20,7 @@ from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import findRequiredNode, getOptionalAttrib from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime ############################################################ ############################################################ @@ -31,6 +32,20 @@ ############################################################ ############################################################ +# bar is 63.5% of cactus_consolidated's time across the 576 VGP alignments, caf 20.3%, +# reference 15.4%. The parallel part stops improving somewhere around 24 cores -- which is why +# --consCores above that buys memory rather than speed -- so a job given fewer than that, and +# only then, takes proportionally longer. The fits themselves all ran at 64 cores, on the +# plateau. +CONS_CORE_BASELINE = 24 +CONS_PARALLEL_FRACTION = 0.64 + +def cons_core_scale(cores, baseline=CONS_CORE_BASELINE, parallel=CONS_PARALLEL_FRACTION): + """ how much longer cactus_consolidated takes when given `cores` rather than a full node """ + if cores and 0 < cores < baseline: + return (1.0 - parallel) + parallel * (float(baseline) / cores) + return 1.0 + def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, cons_cores = None, cons_memory = None, intermediate_results_url = None, chrom_name = None, cons_retain_pages = None): @@ -85,6 +100,19 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma # the fit ran with it disabled -- so enabling it can only make the estimate conservative. poa_node = findRequiredNode(config_node, 'bar').find('poa') poa_window = getOptionalAttrib(poa_node, 'partialOrderAlignmentWindow', typeFn=int, default=10000) if poa_node is not None else 10000 + + # Giant genomes get a smaller poa window. The window is the only bound on the DP once the + # sequence is long and repeat-rich, and on 22 Gb salamanders halving it took bar's peak from + # 798.5 to 368.7 GiB and doubled throughput, for 0.06 points of recall on evolver mammals. + # Ingroups only: an outgroup contributes alignment but is not what makes the flowers huge. + big_window = getOptionalAttrib(poa_node, 'partialOrderAlignmentWindowBigGenome', typeFn=int, default=0) if poa_node is not None else 0 + big_threshold = getOptionalAttrib(poa_node, 'partialOrderAlignmentWindowBigGenomeThreshold', typeFn=float, default=0) if poa_node is not None else 0 + if big_window > 0 and big_threshold > 0: + biggest_ingroup = max([seq_id.size for seq_name, seq_id in seq_id_map.items() if seq_name not in outgroups] or [0]) + if biggest_ingroup >= big_threshold and big_window < poa_window: + RealtimeLogger.info('cactus_consolidated({}): largest ingroup is {}, at or above the {} threshold, so the poa window drops from {} to {}'.format( + name, bytes2human(biggest_ingroup), bytes2human(int(big_threshold)), poa_window, big_window)) + poa_window = big_window window_exp = getOptionalAttrib(cons_node, 'memory_poa_window_exponent', typeFn=float, default=0.43) if poa_window > 0 and poa_window != 10000: mem = int(mem * (poa_window / 10000.0) ** window_exp) @@ -134,12 +162,20 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma if retain_pages not in ['auto', '0', '1']: raise RuntimeError(' / --consRetainPages must be auto, 0 or 1, not {}'.format(retain_pages)) retain_ratio = getOptionalAttrib(cons_node, 'memory_retain_ratio', typeFn=float, default=2.5) + retain_fraction = getOptionalAttrib(cons_node, 'memory_retain_auto_fraction', typeFn=float, default=0.5) if retain_pages == 'auto': limits = [l for l in [max_system_memory, int(os.environ['CACTUS_MAX_MEMORY']) if 'CACTUS_MAX_MEMORY' in os.environ else None] if l] limit = min(limits) if limits else None - if limit and mem > limit: - RealtimeLogger.info('cactus_consolidated({}): the memory estimate of {} with jemalloc page retention exceeds the {} the job can be given, so the pages will not be retained'.format( - name, bytes2human(mem), bytes2human(limit))) + # Retention is only attempted with room to be wrong. mem here is the retained estimate for + # this job's core count, and 14% of VGP alignments came in over their estimate -- so asking + # it to fit in a fraction of what the job can be given means a miss still has somewhere to + # land. Without the fraction, any under-estimate at the ceiling OOMs the whole alignment. + # `budget is not None`, not `budget`: a fraction of 0 means never retain, and a bare + # truthiness test would make 0.0 falsy and fall through to retaining every time. + budget = limit * retain_fraction if limit is not None else None + if budget is not None and mem > budget: + RealtimeLogger.info('cactus_consolidated({}): the memory estimate of {} with jemalloc page retention exceeds {:g} of the {} the job can be given, so the pages will not be retained'.format( + name, bytes2human(mem), retain_fraction, bytes2human(limit))) retain_pages = '0' else: retain_pages = '1' @@ -154,20 +190,79 @@ def cactus_cons_with_resources(job, tree, ancestor_event, config_node, seq_id_ma name, bytes2human(mem), bytes2human(max_system_memory))) mem = max_system_memory + # Runtime has two regimes, and which one you are in is decided by (what + # --maxLen sets). They are different in shape, not just in scale: + # + # unbanded, 1 Mb (progressive) secs = 214 * disk_gb**0.906 r = 0.64 + # banded, 10 kb (pangenome) secs = 2522 * disk_gb**0.251 r = 0.26 + # + # fitted to 576 VGP 577-way alignments and to the 50 chromosome alignments of two HPRC + # pangenomes. Banding to 10 kb bounds bar's work per column, so the cost follows the number + # of reference columns rather than the sequence volume -- which is why the pangenome + # exponent is nearly flat, and why adding haplotypes barely moves it even though it moves + # `disk` a great deal. Applying the progressive fit to a pangenome chromosome overshoots by + # 13-17x at the median: HPRC chr5 takes 4.1 h and would have been given 312. + # + # The coefficients below are the fits scaled so that cactus_walltime()'s factor covers the + # worst residual, then divided by 4: the 2x cactus_consolidated speedup that has already + # landed since both sets of logs, and a further 2x that is expected but NOT yet measured + # here. If that second 2x underdelivers the cost is small and bounded -- replaying the + # fits against today's times, 5 of 576 progressive and 3 of 50 pangenome alignments would + # run over, none by more than 1.5x, so a single --doubleTime retry rescues every one. The + # gain is not small: it takes the median progressive request from 15.6 h to 7.8 h and the + # number of them over 24 h from 181 to 7. Raise these two attributes if that turns out to + # be optimistic. `disk` is the size term because it already combines the sequence and paf + # sizes in the proportions that drive the work (5:2). + # + # Only the two measured banding values are in real use, so this selects between them rather + # than interpolating a curve through data that does not exist. Anything between them takes + # the unbanded model, which is the conservative side. + banding_limit = getOptionalAttrib(findRequiredNode(config_node, 'bar'), 'bandingLimit', typeFn=int, default=0) + banding_threshold = getOptionalAttrib(cons_node, 'walltime_banding_threshold', typeFn=int, default=100000) + if banding_limit and banding_limit < banding_threshold: + wt_coef = getOptionalAttrib(cons_node, 'walltime_banded_coefficient_secs', typeFn=float, default=550.0) + wt_exp = getOptionalAttrib(cons_node, 'walltime_banded_exponent', typeFn=float, default=0.35) + else: + # the progressive fit's exponent was 0.906 and rounded to 0.95 here, which is near enough + # to linear that carrying it as a tunable was not worth the knob: over sizes from 0.1 to + # 50 GB and core counts from 8 to 64, dropping it moves 3 of 36 jobs across a partition + # boundary, and always upward. The banded exponent below is a different matter -- 0.35 is + # genuinely concave, and linearising it either under-provisions the middle of the range by + # 1.7x or over-provisions the top by 4.4x -- so that one stays. + wt_coef = getOptionalAttrib(cons_node, 'walltime_coefficient_secs', typeFn=float, default=400.0) + wt_exp = 1.0 + walltime_secs = wt_coef * ((disk / 1e9) ** wt_exp) if disk > 0 else 0 + # bar is 63.5% of consolidated's time across those 576 alignments, caf 20.3%, reference + # 15.4%; the parallel part of that stops improving somewhere around 24 cores (which is why + # --consCores above ~24 buys memory, not speed), so scale up only when a job is given fewer + # than that. The fit's own jobs all ran at 64 cores, i.e. already on the plateau. + # CONS_CORE_BASELINE and CONS_PARALLEL_FRACTION rather than config attributes: they describe + # how cactus_consolidated parallelises, not anything about a particular alignment, and the + # three other callers of cons_core_scale already take them from the module. + walltime_secs *= cons_core_scale(cons_cores) + cons_job = job.addChildJobFn(cactus_cons, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, intermediate_results_url=intermediate_results_url, chrom_name=chrom_name, cores = cons_cores, - memory=cactus_clamp_memory(mem), disk=disk, retain_pages=retain_pages) + memory=cactus_clamp_memory(mem), disk=disk, retain_pages=retain_pages, + poa_window=poa_window, walltime=cactus_walltime(walltime_secs)) return cons_job.rv() def cactus_cons(job, tree, ancestor_event, config_node, seq_id_map, og_map, paf_id, - intermediate_results_url = None, chrom_name = None, retain_pages = None): + intermediate_results_url = None, chrom_name = None, retain_pages = None, + poa_window = None): ''' run cactus_consolidated ''' # cactus_consolidated reads its settings from the config, so the resolved page retention # goes into the copy it is given (this job's copy of the node, so nothing else sees it) - if retain_pages is not None: + if retain_pages is not None or poa_window is not None: config_node = copy.deepcopy(config_node) - findRequiredNode(config_node, 'consolidated').set('retain_pages', str(retain_pages)) + if retain_pages is not None: + findRequiredNode(config_node, 'consolidated').set('retain_pages', str(retain_pages)) + # the estimator resolved the window (it may have been lowered for a giant genome), and the + # estimate it produced only holds if cactus_consolidated uses that same value + poa_node = findRequiredNode(config_node, 'bar').find('poa') + if poa_window is not None and poa_node is not None: + poa_node.set('partialOrderAlignmentWindow', str(poa_window)) # Build up a genome -> fasta map. work_dir = job.fileStore.getLocalTempDir() @@ -225,9 +320,20 @@ def cactus_cons(job, tree, ancestor_event, config_node, seq_id_map, og_map, paf_ if use_secondary_alignments: # Optionally add the secondary alignments args += ["--secondaryAlignments", secondary_alignment_file] + # jemalloc reads MALLOC_CONF once, before main, so transparent huge pages cannot be switched on + # through mallctl the way page retention is. Prefixing `env` puts it in front of this one + # process rather than in the worker's own environment, which a follow-on job would inherit -- + # and it travels into the container, which an exported variable does not, since dockerCommand + # passes no -e. Measured on salamander Anc3 without page retention, same processor model, + # 44398 -> 33124 seconds for 0.9% more peak (347.0 -> 350.0 GiB). An existing MALLOC_CONF wins. + thp_prefix = [] + if getOptionalAttrib(findRequiredNode(config_node, 'consolidated'), 'transparent_huge_pages', typeFn=bool, default=True) \ + and 'MALLOC_CONF' not in os.environ: + thp_prefix = ['env', 'MALLOC_CONF=thp:always'] + messages = cactus_call(check_output=True, returnStdErr=True, realtimeStderrPrefix=f'cactus_consolidated({chrom_name if chrom_name else ancestor_event})', - parameters=["cactus_consolidated"] + args, + parameters=thp_prefix + ["cactus_consolidated"] + args, work_dir=work_dir, job_memory=job.memory)[1] # Get just the standard error output diff --git a/src/cactus/preprocessor/cactus_preprocessor.py b/src/cactus/preprocessor/cactus_preprocessor.py index 70663ece1..15edbe3a6 100644 --- a/src/cactus/preprocessor/cactus_preprocessor.py +++ b/src/cactus/preprocessor/cactus_preprocessor.py @@ -19,7 +19,7 @@ from sonLib.bioio import getTempDirectory from toil.common import Toil from toil.job import Job -from cactus.shared.common import cactus_call +from cactus.shared.common import cactus_call, cactus_walltime, WALLTIME_COORDINATION from cactus.shared.common import RoundedJob from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import runGetChunks @@ -36,7 +36,7 @@ from toil.statsAndLogging import set_logging_from_options from toil.realtimeLogger import RealtimeLogger -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.preprocessor.checkUniqueHeaders import checkUniqueHeaders from cactus.preprocessor.lastzRepeatMasking.cactus_lastzRepeatMask import LastzRepeatMaskJob from cactus.preprocessor.lastzRepeatMasking.cactus_lastzRepeatMask import RepeatMaskOptions @@ -50,6 +50,30 @@ from cactus.preprocessor.checkPreprocessedSequence import preprocessed_fasta_id from cactus.progressive.cactus_prepare import human2bytesN +# Seconds per GB of fasta for the jobs here that make one linear pass over a whole genome. +# Micro-benchmarked on a 210 MB fasta, and unchanged between a 52-scaffold and a 10309-contig +# version of it: the Bio.SeqIO parse-and-rewrite of CheckUniqueHeaders 12 s/GB, the unmaskFasta +# uppercase loop 9 s/GB, checkPreprocessedSequence's crc32 digest 1.5 s/GB per file. The one +# tool in the group, `faffy merge`, is covered too: faffy chunk and faffy extract managed +# ~33 s/GB across the VGP 577-way (n=8040, p99 67 s on ~2 GB genomes). 40 is the slowest of +# those with margin for a contended cluster worker; cactus_walltime's factor takes the +# effective allowance to 100 s/GB. +FASTA_PASS_SECS_PER_GB = 40 + +# Seconds per GB of a --maskFile. See the call site in maskJobOverride below for provenance. +MASK_FILE_SECS_PER_GB = 300 + +def fasta_pass_walltime(size, passes=1, io_multiple=2): + """ Walltime for a job that makes `passes` linear passes over a fasta of `size` bytes and + stages `io_multiple` times that size through the jobstore. + + size is None where the input is still a promise, which has no size at scheduling time; that + gives no estimate at all rather than a wrong one, leaving the job on --defaultWalltime. """ + if not size: + return cactus_walltime(None) + return cactus_walltime(passes * FASTA_PASS_SECS_PER_GB * size / 1e9, + io_bytes=io_multiple * size) + class PreprocessorOptions: def __init__(self, chunkSize, memory, cpu, check, proportionToSample, unmask, preprocessJob, checkAssemblyHub=None, lastzOptions=None, minPeriod=None, @@ -95,7 +119,8 @@ def __init__(self, prepOptions, inChunkID): disk = 2*inChunkID.size memory = max(prepOptions.memory, inChunkID.size) RoundedJob.__init__(self, memory=memory, cores=prepOptions.cpu, disk=disk, - preemptable=True) + preemptable=True, + walltime=fasta_pass_walltime(inChunkID.size)) self.prepOptions = prepOptions self.inChunkID = inChunkID @@ -111,7 +136,7 @@ def run(self, fileStore): class MergeChunks(RoundedJob): def __init__(self, prepOptions, chunkIDList): - RoundedJob.__init__(self, preemptable=True) + RoundedJob.__init__(self, preemptable=True, walltime=cactus_walltime()) self.prepOptions = prepOptions self.chunkIDList = chunkIDList @@ -121,9 +146,11 @@ def run(self, fileStore): class MergeChunks2(RoundedJob): """merge a list of chunks into a fasta file""" def __init__(self, prepOptions, chunkIDList): - disk = 2*sum([chunkID.size for chunkID in chunkIDList]) + total_size = sum([chunkID.size for chunkID in chunkIDList]) + disk = 2*total_size RoundedJob.__init__(self, cores=prepOptions.cpu, memory=prepOptions.memory, disk=disk, - preemptable=True) + preemptable=True, + walltime=fasta_pass_walltime(total_size)) self.prepOptions = prepOptions self.chunkIDList = chunkIDList @@ -143,9 +170,16 @@ class PreprocessSequence(RoundedJob): """Cut a sequence into chunks, process, then merge """ def __init__(self, prepOptions, inSequenceID, chunksToCompute=None): - disk = 3*inSequenceID.size if hasattr(inSequenceID, "size") else None + in_size = inSequenceID.size if hasattr(inSequenceID, "size") else None + disk = 3*in_size if in_size else None + # in the default unchunked config this job is only a genome-sized jobstore round trip, + # but Toil chains its single mask-job child into it whenever the child's memory, cores + # and disk fit: the VGP 577-way preprocess log demanded 577 CheckUniqueHeaders and only + # ever issued 259 of them as batch jobs, so the other ~318 ran inside their parent. So + # it has to carry that child's fasta pass and its output as well as our own staging. RoundedJob.__init__(self, memory=prepOptions.memory, disk=disk, - preemptable=True) + preemptable=True, + walltime=fasta_pass_walltime(in_size, passes=2, io_multiple=3)) self.prepOptions = prepOptions self.inSequenceID = inSequenceID self.chunksToCompute = chunksToCompute @@ -268,11 +302,17 @@ def unmaskFasta(inFasta, outFasta): out.write(line) class BatchPreprocessor(RoundedJob): - def __init__(self, prepXmlElems, inSequenceID, iteration = 0): + def __init__(self, prepXmlElems, inSequenceID, iteration = 0, inSequenceSize=None): self.prepXmlElems = prepXmlElems self.inSequenceID = inSequenceID self.iteration = iteration - RoundedJob.__init__(self, preemptable=True) + if inSequenceSize is None and hasattr(inSequenceID, "size"): + inSequenceSize = inSequenceID.size + # not the coordination tier: a preprocessor with unmask="1" -- dna-brnn in the default + # config, and whichever masker unmasking.py picks in remask mode -- reads the whole + # genome out of the jobstore, uppercases it in python and writes it back + RoundedJob.__init__(self, preemptable=True, + walltime=fasta_pass_walltime(inSequenceSize)) def run(self, fileStore): # Parse the "preprocessor" config xml element @@ -282,6 +322,7 @@ def run(self, fileStore): prepNode = self.prepXmlElems[self.iteration] checkJob = None + inSize = self.inSequenceID.size if hasattr(self.inSequenceID, "size") else None if getOptionalAttrib(prepNode, "active", typeFn = bool, default=True): prepOptions = PreprocessorOptions(chunkSize = int(prepNode.get("chunkSize", default="-1")), preprocessJob=prepNode.attrib["preprocessJob"], @@ -319,12 +360,15 @@ def run(self, fileStore): outSeqID = ppJob.rv() # make sure the step only masked/renamed, and did not corrupt the sequence. # this has to run before clean_if_different, which drops the input file. - inSize = self.inSequenceID.size if hasattr(self.inSequenceID, "size") else None + # it pulls both whole genomes out of the jobstore before either crc32 pass starts, + # so it is an I/O job rather than a coordination one checkJob = self.addFollowOnJobFn(check_preprocessed_sequence, self.inSequenceID, outSeqID, prepOptions.preprocessJob, prepOptions.eventName, prepOptions.dnabrnnAction, - disk=3*inSize if inSize else None) - checkJob.addFollowOnJobFn(clean_if_different, self.inSequenceID, outSeqID) + disk=3*inSize if inSize else None, + walltime=fasta_pass_walltime(inSize)) + checkJob.addFollowOnJobFn(clean_if_different, self.inSequenceID, outSeqID, + walltime=cactus_walltime()) else: logger.info("Skipping inactive preprocessor {}".format(prepNode.attrib["preprocessJob"])) outSeqID = self.inSequenceID @@ -334,7 +378,10 @@ def run(self, fileStore): # starts by deleting its input, which is the very file the check reads as # its output, and follow-ons of the same job would otherwise race previous = checkJob if checkJob is not None else self - return previous.addFollowOn(BatchPreprocessor(self.prepXmlElems, outSeqID, self.iteration + 1)).rv() + # outSeqID is a promise, so the next iteration cannot size itself off it; no + # preprocessor changes the sequence length except by clipping, so our input stands in + return previous.addFollowOn(BatchPreprocessor(self.prepXmlElems, outSeqID, self.iteration + 1, + inSequenceSize=inSize)).rv() else: return outSeqID @@ -374,7 +421,22 @@ class CactusPreprocessor(RoundedJob): """Modifies the input genomes, doing things like masking/checking, etc. """ def __init__(self, inputSequenceIDs, configNode, eventNames=[]): - RoundedJob.__init__(self, disk=sum([id.size for id in inputSequenceIDs if hasattr(id, 'size')]), preemptable=True) + in_bytes = sum([id.size for id in inputSequenceIDs if hasattr(id, 'size')]) + # run() only deepcopies the config and adds one child per genome -- it stages nothing. + # In the VGP 577-way preprocess log this job was issued at 20:58:28 and its 577 children + # at 21:00:05: 97 s covering the Slurm queue, worker startup and 577 child + # serializations, so ~0.1 s per child. 0.25 s per child is twice that, and below ~1900 + # genomes it is indistinguishable from the bare coordination tier. + # + # The exception is a one-genome fan-out (unmasking.py's remask, cactus_progressive.py's + # per-ingroup preprocess): Toil chains a lone successor into the same worker, so there + # the parent has to cover the child's pass over the fasta as well. A real fan-out does + # not, and must not price in every genome it schedules. + chained_bytes = in_bytes if len(inputSequenceIDs) == 1 else 0 + RoundedJob.__init__(self, disk=in_bytes, preemptable=True, + walltime=cactus_walltime(WALLTIME_COORDINATION + 0.25 * len(inputSequenceIDs) + + FASTA_PASS_SECS_PER_GB * chained_bytes / 1e9, + io_bytes=2 * chained_bytes)) self.inputSequenceIDs = inputSequenceIDs self.configNode = configNode self.eventNames = eventNames @@ -406,7 +468,12 @@ def getOutputSequenceFiles(inputSequences, outputSequenceDir): class CactusPreprocessor2(RoundedJob): def __init__(self, inputSequenceID, configNode): - RoundedJob.__init__(self, preemptable=True) + # the body only adds one BatchPreprocessor child, but Toil chains a single successor + # into its predecessor when the successor's memory, cores and disk fit, and these two + # both take the defaults -- so this has to cover a BatchPreprocessor iteration, which + # reads and rewrites the whole genome when the preprocessor it runs has unmask="1" + in_size = inputSequenceID.size if hasattr(inputSequenceID, "size") else None + RoundedJob.__init__(self, preemptable=True, walltime=fasta_pass_walltime(in_size)) self.inputSequenceID = inputSequenceID self.configNode = configNode @@ -461,7 +528,7 @@ def stageWorkflow(outputSequenceDir, configNode, inputSequences, toil, restart=F inputSequenceIDs.append(toil.importFile(makeURL(seq))) maskFileID = toil.importFile(makeURL(maskFile)) if maskFile else None unzip_job = Job.wrapJobFn(unzip_then_pp, configNode, inputSequences, inputSequenceIDs, inputEventNames, - maskFile, maskFileID, maskAction, minLength) + maskFile, maskFileID, maskAction, minLength, walltime=cactus_walltime()) outputSequenceIDs = toil.start(unzip_job) else: outputSequenceIDs = toil.restart() @@ -477,13 +544,20 @@ def stageWorkflow(outputSequenceDir, configNode, inputSequences, toil, restart=F def unzip_then_pp(job, config_node, input_fa_paths, input_fa_ids, input_event_names, mask_file_path, mask_file_id, mask_file_action, min_length): """ unzip then preprocess """ - unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids) + unzip_job = job.addChildJobFn(unzip_gzs, input_fa_paths, input_fa_ids, walltime=cactus_walltime()) if mask_file_id is not None: - mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id]) + mask_unzip_job = unzip_job.addChildJobFn(unzip_gzs, [mask_file_path], [mask_file_id], walltime=cactus_walltime()) + # a PAF mask file gets one pafcoverage scan of the whole thing; the nearest measured + # analogue is filter-paf-deletions over a whole-pangenome PAF (HPRC v2.1, n=26, p99 + # 701 s), hence MASK_FILE_SECS_PER_GB. mask_file_id.size is the *compressed* size when + # the mask file is gzipped, which is why the rate is this high and why disk is 20x. For + # a plain BED the job is pure XML rewriting and this collapses to the walltime floor. config_node = mask_unzip_job.addFollowOnJobFn(maskJobOverride, config_node, mask_file_path, mask_unzip_job.rv(0), mask_file_action, min_length, - disk=mask_file_id.size*20).rv() + disk=mask_file_id.size*20, + walltime=cactus_walltime(MASK_FILE_SECS_PER_GB * mask_file_id.size / 1e9, + io_bytes=4 * mask_file_id.size)).rv() pp_job = unzip_job.addFollowOn(CactusPreprocessor([unzip_job.rv(i) for i in range(len(input_fa_ids))], config_node, eventNames=input_event_names)) - zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0]) + zip_job = pp_job.addFollowOnJobFn(zip_gzs, input_fa_paths, pp_job.rv(), list_elems = [0], walltime=cactus_walltime()) return zip_job.rv() def runCactusPreprocessor(outputSequenceDir, configFile, inputSequences, toilDir): @@ -495,6 +569,7 @@ def runCactusPreprocessor(outputSequenceDir, configFile, inputSequences, toilDir def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("inSeqFile", type=str, nargs='?', default=None, help = "Input Seq file") parser.add_argument("outSeqFile", type=str, nargs='?', default=None, help = "Output Seq file (ex generated with cactus-prepare)") parser.add_argument("--configFile", default=os.path.join(cactusRootPath(), "cactus_progressive_config.xml")) diff --git a/src/cactus/preprocessor/checkUniqueHeaders.py b/src/cactus/preprocessor/checkUniqueHeaders.py index 6d8825a7f..55e97786b 100644 --- a/src/cactus/preprocessor/checkUniqueHeaders.py +++ b/src/cactus/preprocessor/checkUniqueHeaders.py @@ -6,6 +6,16 @@ import os from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime + +# Seconds per GB of *input* fasta for sanitize_fasta_header, the busiest job this module +# defines. Rank-pairing its command time against its input size gives 42-49 s/GB at the median +# in three independent runs (VGP 577-way n=5748, HPRC v2.1 n=22221, HPRC v2.0 n=11385), but the +# tail under heavy concurrency is what a walltime has to survive: in HPRC v2.0 the two commands +# this job runs peaked at 367 s and 223 s over ~0.98 GB gzipped inputs. Those maxima are from +# different instances, so their sum is an upper bound rather than one job's time; 250 s/GB is +# aimed at that bound rather than at the median. +SANITIZE_SECS_PER_GB = 250 def checkUniqueHeaders(inputFile, outputFile, eventName, checkAlphaNumeric=False, checkUCSC=False, checkAssemblyHub=True): """Check that headers are unique and meet certain requirements.""" @@ -39,9 +49,13 @@ def sanitize_fasta_headers(job, fasta_id_map, pangenome=False, log_stats=True): """ input must be map of event -> fasta id""" out_fasta_id_map = {} for event, fasta_id in fasta_id_map.items(): + # fasta_id.size is the compressed size when the input is gzipped, while the job writes + # the fasta back out uncompressed -- the same reason disk is 7x -- so io_bytes is 4x out_fasta_id_map[event] = job.addChildJobFn(sanitize_fasta_header, fasta_id, event, pangenome, log_stats, memory=cactus_clamp_memory(fasta_id.size * 4), - disk=fasta_id.size*7).rv() + disk=fasta_id.size*7, + walltime=cactus_walltime(SANITIZE_SECS_PER_GB * fasta_id.size / 1e9, + io_bytes=4 * fasta_id.size)).rv() return out_fasta_id_map def sanitize_fasta_header(job, fasta_id, event, pangenome, log_stats): diff --git a/src/cactus/preprocessor/cutHeaders.py b/src/cactus/preprocessor/cutHeaders.py index 6a7a58040..90b57e64a 100644 --- a/src/cactus/preprocessor/cutHeaders.py +++ b/src/cactus/preprocessor/cutHeaders.py @@ -12,13 +12,24 @@ from Bio.SeqRecord import SeqRecord from cactus.shared.common import RoundedJob +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Seconds per GB of input fasta for the Bio.SeqIO parse-and-rewrite this job does. +# Micro-benchmarked at 8-9 s/GB on a 210 MB fasta, and the same on a 52-scaffold and on a +# 10309-contig version of it, so the record count does not matter. Rounded up to 12 for the +# benchmark being one file on one machine; the cluster margin is --walltimeFactor's job. This +# one really does run on every genome of every --pangenome run, where the VGP 577-way issued +# 635 of them over inputs with a p99 of 9.2 GB. +CUT_HEADERS_SECS_PER_GB = 12 + class CutHeadersJob(RoundedJob): def __init__(self, fastaID, cutBefore, cutBeforeOcc, cutAfter): disk = 2*(fastaID.size) memory = fastaID.size - RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True, + walltime=cactus_walltime(CUT_HEADERS_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=2 * fastaID.size)) self.fastaID = fastaID self.cutBefore = cutBefore self.cutBeforeOcc = cutBeforeOcc diff --git a/src/cactus/preprocessor/dnabrnnMasking.py b/src/cactus/preprocessor/dnabrnnMasking.py index 615e82152..c379ff459 100644 --- a/src/cactus/preprocessor/dnabrnnMasking.py +++ b/src/cactus/preprocessor/dnabrnnMasking.py @@ -17,9 +17,20 @@ from cactus.shared.common import getOptionalAttrib from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Seconds per GB of input fasta for dna-brnn. No direct evidence: dna-brnn is active="0" in the +# default config (--maskMode brnn turns it on) and did not run in any of the mined logs. This is +# from first principles -- dna-brnn is a recurrent-network base classifier that streams the +# genome, and the job asks for a flat 4 GiB and 2 cores, so it is CPU-bound and roughly linear in +# genome size. With cactus_walltime's factor the effective allowance is 1500 s/GB, ~1.3 hours +# for a 3 Gbp genome and ~4.4 hours for the largest VGP one. The bedtools sort/merge/subtract +# and cactus_fasta_softmask_intervals.py tail that follows is a rounding error next to that: the +# latter was p99 181 s over genomes up to 10.5 GB in the VGP 577-way. +DNABRNN_SECS_PER_GB = 600 + def loadDnaBrnnModel(toil, configNode, maskAlpha = False): """ store the model in a toil file id so it can be used in any workflow """ for prepXml in configNode.findall("preprocessor"): @@ -37,7 +48,9 @@ def __init__(self, fastaID, dnabrnnOpts, cpu, minLength=None, action=None, input memory = 4*1024*1024*1024 disk = 2*(fastaID.size) cores = min(cactus_cpu_count(), cpu) - RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, preemptable=True, + walltime=cactus_walltime(DNABRNN_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=3 * fastaID.size)) self.fastaID = fastaID self.minLength = minLength self.action = action diff --git a/src/cactus/preprocessor/fastanMasking.py b/src/cactus/preprocessor/fastanMasking.py index bfb90cce1..161614bde 100644 --- a/src/cactus/preprocessor/fastanMasking.py +++ b/src/cactus/preprocessor/fastanMasking.py @@ -18,17 +18,30 @@ from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime from cactus.preprocessor.maskingCommon import prefilter_cmd, masked_base_count from cactus.preprocessor.maskingCommon import soft_mask_intervals, log_masking_delta +from cactus.preprocessor.redMasking import RED_SECS_PER_GB, RED_SPEEDUP from toil.realtimeLogger import RealtimeLogger +# FasTAN is active="0" by default and has never run in a logged workflow, so there is no rate +# for it. This job is RedMaskJob's twin -- same prefilter, same bed extraction, same interval +# application, one different repeat masker in the middle -- so Red's measured rate stands in. +# RED_SECS_PER_GB has Red's 3x speedup divided out of it, and FasTAN has no claim to that +# discount, so it is multiplied back: this is Red's rate as measured, before Red got faster. +# --doubleTime buys the retry if FasTAN turns out slower still. +FASTAN_SECS_PER_GB = RED_SECS_PER_GB * RED_SPEEDUP + + class FasTANMaskJob(RoundedJob): def __init__(self, fastaID, fastanOpts, fastanPrefilterOpts, eventName=None, unmask=False): memory = cactus_clamp_memory(12*fastaID.size) disk = 5*(fastaID.size) - RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True, + walltime=cactus_walltime(FASTAN_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=2 * fastaID.size)) self.fastaID = fastaID self.fastanOpts = fastanOpts self.fastanPrefilterOpts = fastanPrefilterOpts diff --git a/src/cactus/preprocessor/fileMasking.py b/src/cactus/preprocessor/fileMasking.py index 0a8f182bd..d999fb16e 100644 --- a/src/cactus/preprocessor/fileMasking.py +++ b/src/cactus/preprocessor/fileMasking.py @@ -21,14 +21,27 @@ from cactus.shared.common import getOptionalAttrib from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Seconds per GB of input fasta for FileMaskingJob. The --maskFile path did not run in any of +# the mined logs, but its two dominant commands did, in the VGP 577-way preprocess log: +# cactus_softmask2hardmask n=601 p99 51 s over genomes up to 10.5 GB (~10 s/GB) and +# cactus_fasta_softmask_intervals.py n=577 p99 181 s (~30 s/GB). 80 doubles that sum to cover +# the bed merge loop, which is pure python and has been observed in no log; the margin beyond +# that is --walltimeFactor's job. Caveat for whoever tunes this: the input BED/PAF arrives as an XML +# attribute, so its size is not in scope here at all -- a pathological mask file is invisible to +# this formula and only --doubleTime would save the job. +FILE_MASKING_SECS_PER_GB = 80 + class FileMaskingJob(RoundedJob): def __init__(self, fastaID, minLength=None, action=None, inputBedID=None, eventName=None): disk = 2*(fastaID.size) memory = fastaID.size - RoundedJob.__init__(self, disk=disk, memory=memory, preemptable=True) + RoundedJob.__init__(self, disk=disk, memory=memory, preemptable=True, + walltime=cactus_walltime(FILE_MASKING_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=3 * fastaID.size)) self.fastaID = fastaID self.minLength = minLength self.action = action diff --git a/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py b/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py index 4707d041a..243d223e0 100644 --- a/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py +++ b/src/cactus/preprocessor/lastzRepeatMasking/cactus_lastzRepeatMask.py @@ -13,8 +13,21 @@ from cactus.shared.common import cactus_call from cactus.shared.common import RoundedJob +from cactus.shared.common import cactus_walltime from toil.realtimeLogger import RealtimeLogger +# Estimated seconds for one repeat-masking lastz. This preprocessor is deprecated and +# active="0", so it never ran in any mined log; the stand-in is the blast phase's lastz, whose +# p99 across the VGP 577-way (n=826872) was 7469 s, or ~3700 s for the 2x-faster lastz we ship +# now. That is generous, which is the point: this job aligns one chunk (10 MB from the config, +# forced to 6 GB in gpu mode) against proportionToSample of the genome, a smaller job than a +# blast-phase chunk pair. Scaled by what this job is actually handed, the same way +# get_lastz_walltime scales the blast phase's own table: the p99 above was measured on a pair of +# full 90 MB chunks, so a repeat-mask job aligning the config's 10 MB chunk against a sampled +# target set is a fraction of it. +LASTZ_REPEAT_MASK_SECS = 3700 +LASTZ_REPEAT_MASK_REFERENCE_BYTES = 2 * 90e6 + class RepeatMaskOptions: def __init__(self, fragment=200, @@ -59,7 +72,10 @@ def __init__(self, repeatMaskOptions, queryID, targetIDs): disk = max(4*(queryID.size + targetsSize), memory) cores = repeatMaskOptions.cpu accelerators = ['cuda:{}'.format(repeatMaskOptions.gpu)] if repeatMaskOptions.gpu else None - RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, accelerators=accelerators, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, cores=cores, accelerators=accelerators, preemptable=True, + walltime=cactus_walltime(LASTZ_REPEAT_MASK_SECS * + (queryID.size + targetsSize) / LASTZ_REPEAT_MASK_REFERENCE_BYTES, + io_bytes=2 * (queryID.size + targetsSize))) self.repeatMaskOptions = repeatMaskOptions self.queryID = queryID self.targetIDs = targetIDs diff --git a/src/cactus/preprocessor/redMasking.py b/src/cactus/preprocessor/redMasking.py index 8942cb52c..5771e31f5 100644 --- a/src/cactus/preprocessor/redMasking.py +++ b/src/cactus/preprocessor/redMasking.py @@ -19,6 +19,7 @@ from cactus.shared.common import makeURL from cactus.shared.common import get_faidx_subpath_rename_cmd from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime from cactus.preprocessor.checkPreprocessedSequence import check_sequence_preserved from cactus.preprocessor.maskingCommon import prefilter_cmd, masked_base_count from cactus.preprocessor.maskingCommon import extract_masking_bed, soft_mask_intervals @@ -65,6 +66,18 @@ def red_memory_estimate(fasta_size, longest_record_bytes): return int(1.25 * (table_bytes + 8 * longest_record_bytes)) +# How much faster Red is than it was when the VGP 577-way logs were made. Named because +# FasTAN borrows Red's rate and has to multiply this back out -- it got no such speedup. +RED_SPEEDUP = 3.0 + +# Seconds of Red per GB of input fasta. Across the 625 Red runs of the VGP 577-way (0.13 to +# 10 Gb of genome) the p99 was 2799 s/Gb and the worst 5147 s/Gb. Everything else this job +# runs -- the prefilter, the softmask/hardmask conversions, extracting and applying the +# intervals -- came to well under 100 s each even on the largest genome, and is covered by +# cactus_walltime()'s safety factor. +RED_SECS_PER_GB = 2799 / RED_SPEEDUP + + class RedMaskJob(RoundedJob): def __init__(self, fastaID, redOpts, redPrefilterOpts, eventName=None, unmask=False, longestRecordSize=None): @@ -74,7 +87,9 @@ def __init__(self, fastaID, redOpts, redPrefilterOpts, eventName=None, unmask=Fa longestRecordSize = fastaID.size memory = cactus_clamp_memory(red_memory_estimate(fastaID.size, longestRecordSize)) disk = 5*(fastaID.size) - RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True) + RoundedJob.__init__(self, memory=memory, disk=disk, preemptable=True, + walltime=cactus_walltime(RED_SECS_PER_GB * fastaID.size / 1e9, + io_bytes=2 * fastaID.size)) self.fastaID = fastaID self.redOpts = redOpts self.redPrefilterOpts = redPrefilterOpts diff --git a/src/cactus/preprocessor/unmasking.py b/src/cactus/preprocessor/unmasking.py index e2e42d208..1577f4c88 100644 --- a/src/cactus/preprocessor/unmasking.py +++ b/src/cactus/preprocessor/unmasking.py @@ -5,20 +5,39 @@ from Bio.SeqRecord import SeqRecord import os from toil.job import Job -from cactus.shared.common import cactus_clamp_memory, catFiles, clean_jobstore_files +from cactus.shared.common import cactus_clamp_memory, catFiles, clean_jobstore_files, cactus_walltime from cactus.shared.common import cactus_call, getOptionalAttrib from toil.realtimeLogger import RealtimeLogger from cactus.preprocessor.cactus_preprocessor import CactusPreprocessor +# Seconds per GB of input fasta for the unmask scan. The inner loop below walks every base of +# every contig in python (`for c in seq_str: ... c.isupper()`), micro-benchmarked at 131 s/GB on +# a 210 MB fasta -- by far the slowest per-byte operation in the preprocessor, ~15x the +# Bio.SeqIO parse-and-rewrite rate on the same file. Rounded up to 150 because the benchmark +# was one file on one machine; the margin for a contended worker is --walltimeFactor's job, not +# this constant's. Two mitigations are deliberately not priced in: the loop stops early once +# threshold unmasked bases are seen, and contigs under min_length are skipped -- but the +# heavily-masked contigs this job exists to find are exactly the ones scanned to the end. +UNMASK_SCAN_SECS_PER_GB = 150 + +# Seconds per GB of fasta for merge_fa: two Bio.SeqIO parse-and-write passes (8-12 s/GB each, +# micro-benchmarked) plus two whole-genome cactus_analyseAssembly runs, which dominate and whose +# tail under heavy concurrency is the worst part of the picture -- n=11392, p99 270 s, max 367 s +# in the HPRC v2.0 pangenome. +MERGE_FA_SECS_PER_GB = 120 + def unmask_contigs_all(job, event_names_to_sequences, ingroup_events, params): """ unmask the given events and return and updated fasta ids for each ingroup as list """ - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) output_ids = [] for event in ingroup_events: + event_size = event_names_to_sequences[event].size unmask_job = root_job.addChildJobFn(unmask_contigs_one, event, event_names_to_sequences[event], params, - memory=event_names_to_sequences[event].size * 2, - disk=event_names_to_sequences[event].size * 5) + memory=event_size * 2, + disk=event_size * 5, + walltime=cactus_walltime(UNMASK_SCAN_SECS_PER_GB * event_size / 1e9, + io_bytes=3 * event_size)) output_ids.append(unmask_job.rv()) return output_ids @@ -94,11 +113,15 @@ def unmask_contigs_one(job, event, fasta_id, params): pp_job = job.addChild(CactusPreprocessor([copy_fasta_id], params, eventNames=[event])) pp_id = pp_job.rv(0) # mix in the masking for the to-mask contigs with our original file + # pp_id is a promise, but it is a preprocessed copy of the same genome, so + # fasta_id.size sizes both: two reads in and one write out, plus margin fa_merge_job = pp_job.addFollowOnJobFn(merge_fa, event, fasta_id, pp_id, set(unmasked_contigs.keys()), - disk=fasta_id.size * 4) + disk=fasta_id.size * 4, + walltime=cactus_walltime(MERGE_FA_SECS_PER_GB * fasta_id.size / 1e9, + io_bytes=4 * fasta_id.size)) fasta_id = fa_merge_job.rv() # delete the pp_id - fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id]) + fa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[pp_id], walltime=cactus_walltime()) else: RealtimeLogger.warning('Remasking activated but no maskers active in preprocessor: doing nothing') diff --git a/src/cactus/progressive/cactus_prepare.py b/src/cactus/progressive/cactus_prepare.py index f9f257798..42c671443 100644 --- a/src/cactus/progressive/cactus_prepare.py +++ b/src/cactus/progressive/cactus_prepare.py @@ -32,8 +32,10 @@ from cactus.shared.common import findRequiredNode from cactus.shared.common import makeURL, cactus_call, RoundedJob from cactus.shared.common import write_s3, has_s3, get_aws_region -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options +from cactus.pipeline.cactus_workflow import cons_core_scale from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import cactus_walltime from toil.job import Job from toil.common import Toil @@ -44,6 +46,56 @@ from toil.lib.conversions import human2bytes, bytes2human from toil.lib.accelerators import count_nvidia_gpus +# --toil mode runs each cactus-* command as a nested single-node Toil workflow, so these are +# estimates for a whole command rather than for one of the jobs inside it. All of them come +# from the VGP 577-way, which was run through cactus-prepare's script mode: 576 cactus-blast +# runs, 576 cactus-align runs, one cactus-preprocess run over all 577 genomes. Those nested +# workflows were given 64 cores. + +# In toil mode this parser is Toil's own, so --defaultCores is Toil's option and defaults to 1. +# main() copies it into blastCores/alignCores/preprocessCores when they are unset, so the +# default cactus-prepare-toil invocation runs these nested workflows at a single core -- which +# is what the estimates below have to assume unless the user says otherwise. + +def prepare_cores(cores): + """ cores for a nested cactus-* workflow, as a number the estimates can divide by """ + try: + return max(1, int(cores or 1)) + except (TypeError, ValueError): + return 1 + +# Per-ancestor lastz CPU-seconds for cactus-blast: the p50 of the summed command time of those +# 576 blast logs, 99.7% of it lastz, halved for the 2x lastz speedup since. lastz is the one +# command here that is embarrassingly parallel (827k independent jobs in that run), so dividing +# by the cores the nested workflow gets is the right model. The p50 rather than the p99 +# because cactus_walltime multiplies by 2.5 on top and --doubleTime takes the tail: at the p99 +# (12.1M CPU-seconds) the single-core default asks for 173 days, which no partition can run. +BLAST_CPU_SECS = 4.6e5 + +# cactus-align is cactus_consolidated and little else: it ran p50 10,014 s, p90 34,771 and p99 +# 120,138 over those 576 ancestors -- halve for the 2x consolidated speedup -- while everything +# else in the phase (the paf ops, export_hal, cactus-hal2fasta) summed to under 1,500 s even at +# the p99. The p90 is the anchor, on the same reasoning as blast. Scaled by cores with the +# same model cactus_cons itself uses, because those 576 jobs ran at 64 cores while the default +# here is one. +ALIGN_SECS = 17400 + +# One genome through cactus-preprocess. Grouping the 577-way preprocess log by genome gives +# p50 2,835 s, p90 7,172, p99 11,420; taking the 3x Red speedup out of the Red component of +# each (Red is 88% of the total) leaves p50 1,218, p90 2,859, p99 4,776. +# Red is 88% of that and is multithreaded, so it takes the same core scaling as consolidated. +PREPROCESS_SECS = 4800 + +# Seconds per halAppendSubtree. The 12 cactus-halAppendSubtrees runs of the 577-way did 578 +# appends in 346,706 s of command time -- 600 s each, and flat across batches of 25 to 53. +HAL_APPEND_SECS_PER_SUBTREE = 600 + +# Extra seconds per append for staging, for the call site where the sizes are promises and so +# cannot go in io_bytes. Those same 12 runs took 414,562 s of wall against 346,706 s of +# command time: 117 s per append of pulling the target HAL and the subtree HALs down and +# pushing the grown target back up. +HAL_APPEND_STAGING_SECS_PER_SUBTREE = 120 + def main_toil(): return main(toil_mode=True) @@ -51,6 +103,7 @@ def main(toil_mode=False): parser = ArgumentParser() if toil_mode: parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("--latest", dest="latest", action="store_true", help="Use the latest version of the docker container " "rather than pulling one matching this version of cactus") @@ -456,6 +509,14 @@ def cactusPrepare(options): # write the instructions if options.toil: + # this is the one Toil-launching path in the module that never did this, and the plan + # below now depends on it: cactus_override_toil_options is what exports + # CACTUS_MAX_WALLTIME, without which nothing clamps a walltime to what the cluster's + # partitions will actually accept -- and Toil raises, on the batch system's own thread, + # when it cannot find one that fits. It is also what makes --walltimeFactor, + # --minWalltime and --maxWalltime (added to this parser by add_cactus_toil_options) + # mean anything here at all. + cactus_override_toil_options(options) with Toil(options) as toil: if options.restart: toil.restart() @@ -480,7 +541,7 @@ def get_plan(options, inSeqFile, outSeqFile, configWrapper, toil): # kick things off with an empty job which we will hook subsequent jobs onto # (using RoundedJob because root job must be sublcass of Job, # https://github.com/ComparativeGenomicsToolkit/cactus/pull/284#issuecomment-684125478) - start_job = RoundedJob() + start_job = RoundedJob(walltime=cactus_walltime()) parent_job = start_job job_idx = {} @@ -500,7 +561,8 @@ def get_plan(options, inSeqFile, outSeqFile, configWrapper, toil): job_idx[("preprocess", leaves[i])] = parent_job.addChildJobFn(toil_call_preprocess, options, inSeqFile, outSeqFile, leaves[i], cores=options.preprocessCores, memory=options.preprocessMemory, - disk=options.preprocessDisk) + disk=options.preprocessDisk, + walltime=cactus_walltime(PREPROCESS_SECS * cons_core_scale(prepare_cores(options.preprocessCores)))) else: plan += 'cactus-preprocess {} {} {} --inputNames {} {} {}{}{}{}{}{}\n'.format( get_jobstore(options), options.seqFile, options.outSeqFile, ' '.join(pre_batch), @@ -586,7 +648,7 @@ def cigarPath(event): if options.toil and sub_idx == 0: # advance toil phase (only once per original round) # todo: recapitulate exact dependencies - parent_job = parent_job.addFollowOn(Job()) + parent_job = parent_job.addFollowOn(Job(walltime=cactus_walltime())) if options.script: plan += 'pids=()\n' for event in sub_group: @@ -608,7 +670,8 @@ def cigarPath(event): *fa_promises, cores=options.blastCores, memory=options.blastMemory, - disk=options.preprocessDisk) + disk=options.preprocessDisk, + walltime=cactus_walltime(BLAST_CPU_SECS / prepare_cores(options.blastCores))) job_idx[("align", event)] = job_idx[("blast", event)].addFollowOnJobFn(toil_call_align, options, outSeqFile, mc_tree, @@ -621,7 +684,8 @@ def cigarPath(event): leaf_deps + anc_deps, *fa_promises, cores=options.alignCores, memory=options.alignMemory, - disk=options.alignDisk) + disk=options.alignDisk, + walltime=cactus_walltime(ALIGN_SECS * cons_core_scale(prepare_cores(options.alignCores)))) else: # todo: support cactus interface (it's easy enough here, but cactus_progressive.py needs changes to handle) cactus_options = options.cactusOptions @@ -658,7 +722,7 @@ def cigarPath(event): # advance toil phase if options.toil: - parent_job = parent_job.addFollowOn(Job()) + parent_job = parent_job.addFollowOn(Job(walltime=cactus_walltime())) # stitch together the final tree plan += '\n## HAL merging\n' @@ -710,7 +774,9 @@ def cigarPath(event): *[job_idx[('align', e)].rv(1) for e in event_list], cores=1, memory=options.alignMemory, - disk=options.halAppendDisk) + disk=options.halAppendDisk, + walltime=cactus_walltime((HAL_APPEND_SECS_PER_SUBTREE + + HAL_APPEND_STAGING_SECS_PER_SUBTREE) * len(event_list))) if options.wdl: prev_event = mc_tree.getRootName() @@ -1272,6 +1338,7 @@ def main_hal2fasta(): """ cli wrapper for hal2fasta so that cactus-perpare can send it to slurm consistently with other cactus commands """ parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help="input HAL file") parser.add_argument("genome", help="Genome to convert to FASTA") @@ -1310,9 +1377,14 @@ def main_hal2fasta(): fa_id = toil.restart() else: hal_id = toil.importFile(options.halFile) + # hal2fasta itself is trivial -- 576 runs on the VGP 577-way were p99 120 s, max + # 167 s -- so the job is really the HAL coming down and the (bgzipped) fasta going + # back up. cactus-prepare emits one of these per ancestor, so it is worth keeping + # them out of a long partition. fa_id = toil.start(Job.wrapJobFn(hal2fasta, hal_id, options.halFile, options.genome, options.outputFastaFile, memory=cactus_clamp_memory(3000000000), - disk=int(hal_id.size * 1.3))) + disk=int(hal_id.size * 1.3), + walltime=cactus_walltime(200, io_bytes=int(1.5 * hal_id.size)))) # export the alignments toil.exportFile(fa_id, makeURL(options.outputFastaFile)) @@ -1339,6 +1411,7 @@ def hal2fasta(job, hal_id, hal_path, genome, fa_path): def main_hal_append_subtrees(): """ toil wrapper for halAppendSubtree command(s) """ parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("tgtFile", help="input HAL file to append to") parser.add_argument("subFiles", nargs='+', help="subtree HAL files to append") @@ -1378,9 +1451,16 @@ def main_hal_append_subtrees(): else: hal_id = toil.importFile(options.tgtFile) sub_hal_ids = [toil.importFile(sub_hal) for sub_hal in options.subFiles] + # halAppendSubtree averaged 600 s over the 578 appends of the 577-way, and this job + # runs one per subtree in a serial loop; io_bytes is the target HAL coming down + # mutable, every subtree HAL coming down, and the grown target going back up, which + # is what dominates once the target is a few hundred GiB. + sub_hal_bytes = sum([f.size for f in sub_hal_ids]) out_hal_id = toil.start(Job.wrapJobFn(hal_append_subtrees, hal_id, sub_hal_ids, options, memory=cactus_clamp_memory(10 * max([f.size for f in sub_hal_ids])), - disk=2 * (hal_id.size + sum([f.size for f in sub_hal_ids])))) + disk=2 * (hal_id.size + sub_hal_bytes), + walltime=cactus_walltime(HAL_APPEND_SECS_PER_SUBTREE * len(sub_hal_ids), + io_bytes=2 * (hal_id.size + sub_hal_bytes)))) # export the alignments toil.exportFile(out_hal_id, makeURL(options.outHalFile)) diff --git a/src/cactus/progressive/cactus_progressive.py b/src/cactus/progressive/cactus_progressive.py index 3ccbb2589..44b97ace0 100755 --- a/src/cactus/progressive/cactus_progressive.py +++ b/src/cactus/progressive/cactus_progressive.py @@ -18,7 +18,7 @@ from base64 import b64encode from toil.lib.bioio import getTempFile -from cactus.shared.common import cactus_cpu_count +from cactus.shared.common import cactus_cpu_count, cactus_walltime from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options from toil.realtimeLogger import RealtimeLogger @@ -33,7 +33,7 @@ from cactus.shared.version import cactus_commit from cactus.shared.common import cactusRootPath from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import write_s3 from cactus.shared.common import cactus_clamp_memory from cactus.shared.common import clean_jobstore_files @@ -51,6 +51,14 @@ from sonLib.nxnewick import NXNewick from sonLib.bioio import getTempDirectory +# Seconds of halAppendCactusSubtree per GiB of the c2h + fasta it reads. The cost per GiB is +# steady across two very different data sets: the VGP 577-way spent 64.0 h appending ~7.0 TiB +# in 576 subtrees (33 s/GiB on average, p90 41), and the HPRC v2.1 pangenome's 25 chromosome +# subtrees ran p50 2570 s on ~90 GiB apiece (29 s/GiB, p99 61). export_hal appends every +# subtree of its tree in one serial loop, so it is the average, not the per-append tail, that +# sets a whole-tree export's runtime. +HAL_APPEND_SECS_PER_GIB = 45 + def logAssemblyStats(job, message, name, sequenceID, preemptable=True): sequenceFile = job.fileStore.readGlobalFile(sequenceID) analysisString = cactus_call(parameters=["cactus_analyseAssembly", sequenceFile], check_output=True) @@ -58,7 +66,7 @@ def logAssemblyStats(job, message, name, sequenceID, preemptable=True): def preprocess_all(job, options, config_node, input_seq_id_map): ''' run prepreprocessor on every input sequence ''' - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) events = list(input_seq_id_map.keys()) seq_ids = list(input_seq_id_map.values()) @@ -67,8 +75,12 @@ def preprocess_all(job, options, config_node, input_seq_id_map): for i, event in enumerate(events): pp_seq_ids[event] = preprocessor_job.rv(i) - # do the logging and checkpointing - root_job.addFollowOnJobFn(save_preprocessed_files, options, config_node, pp_seq_ids) + # do the logging and checkpointing. with --intermediateResultsUrl this copies every + # preprocessed sequence out of the jobstore, so it is an I/O job, not a coordination one; + # masking does not change a sequence's length, so the inputs are the right size for it + input_seq_bytes = sum(seq_id.size for seq_id in input_seq_id_map.values()) + root_job.addFollowOnJobFn(save_preprocessed_files, options, config_node, pp_seq_ids, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) return pp_seq_ids @@ -81,12 +93,15 @@ def save_preprocessed_files(job, options, config_node, seq_id_map): # Log the stats for the preprocessed assemblies for name, sequence in list(seq_id_map.items()): - job.addChildJobFn(logAssemblyStats, "After preprocessing", name, sequence) + # cactus_analyseAssembly never took more than 121 s in 27,938 runs across the VGP + # 577-way and the HPRC v2.1 pangenome; downloading the sequence is the rest of the job + job.addChildJobFn(logAssemblyStats, "After preprocessing", name, sequence, + walltime=cactus_walltime(120, io_bytes=sequence.size)) def progressive_schedule(job, options, config_node, seq_id_map, tree, og_map, root_event): ''' create job for every internal node, use child dependencies to make tree (+ outgroups)''' - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) config_wrapper = ConfigWrapper(config_node) @@ -137,7 +152,7 @@ def progressive_schedule(job, options, config_node, seq_id_map, tree, og_map, ro # to be consistent with pre-refactor (and work with updating tests), we include the root when its id is input if event in seq_id_map and seq_id_map[event]: event_id_map[event] = seq_id_map[event] - event_job = Job.wrapJobFn(progressive_step, options, config_node, event_id_map, tree, og_map, event) + event_job = Job.wrapJobFn(progressive_step, options, config_node, event_id_map, tree, og_map, event, walltime=cactus_walltime()) job_table[event] = event_job for dep in dep_table[event]: if dep in job_table: @@ -174,22 +189,23 @@ def progressive_step(job, options, config_node, seq_id_map, tree, og_map, event) # do the blast paf_job = job.addChildJobFn(make_paf_alignments, NXNewick().writeString(spanning_tree), - subtree_eventmap, event, config_node).encapsulate() + subtree_eventmap, event, config_node, walltime=cactus_walltime()).encapsulate() outgroups = og_map[event] if event in og_map else [] # trim the outgroups if outgroups and int(config_node.find("blast").attrib["trimOutgroups"]): # Trim the outgroup sequences trim_sequences = paf_job.addChildJobFn(trim_unaligned_sequences, - [subtree_eventmap[i] for i in outgroups], paf_job.rv(), config_node) + [subtree_eventmap[i] for i in outgroups], paf_job.rv(), config_node, + walltime=cactus_walltime()) cons_job = paf_job.addFollowOnJobFn(progressive_step_2, trim_sequences.rv(), options, config_node, subtree_eventmap, - spanning_tree, og_map, event) + spanning_tree, og_map, event, walltime=cactus_walltime()) else: # Without outgroup trimming (or if there are no outgroups to trim) cons_job = paf_job.addChildJobFn(cactus_cons_with_resources, spanning_tree, event, config_node, subtree_eventmap, og_map, paf_job.rv(), cons_cores=options.consCores, cons_memory=options.consMemory, cons_retain_pages=getattr(options, 'consRetainPages', None), - intermediate_results_url=options.intermediateResultsUrl) + intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_walltime()) # erase the paf since its now longer needed - cons_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[paf_job.rv()]) + cons_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[paf_job.rv()], walltime=cactus_walltime()) return cons_job.rv() @@ -203,12 +219,12 @@ def progressive_step_2(job, trimmed_outgroups_and_alignments, options, config_no # now do consolidated return job.addChildJobFn(cactus_cons_with_resources, spanning_tree, event, config_node, subtree_eventmap, og_map, pafs, cons_cores=options.consCores, cons_memory=options.consMemory, cons_retain_pages=getattr(options, 'consRetainPages', None), - intermediate_results_url=options.intermediateResultsUrl).rv() + intermediate_results_url=options.intermediateResultsUrl, walltime=cactus_walltime()).rv() def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=None, cacheBytes=None, cacheMDC=None, cacheRDC=None, cacheW0=None, chunk=None, inMemory=False, - checkpointInfo=None, acyclicEvent=None, has_resources=False, memory_override=None): + checkpointInfo=None, acyclicEvent=None, has_resources=False): # todo: going through list nonsense because (i think) it helps with promises, should at least clean up work_dir = job.fileStore.getLocalTempDir() @@ -271,16 +287,23 @@ def export_hal(job, mc_tree, config_node, seq_id_map, og_map, results, event=Non cactus_call(parameters=args, job_memory=job.memory) if not has_resources: - disk = 3 * sum([file_id.size for file_id in fa_file_ids + c2h_file_ids]) - mem = cactus_clamp_memory(5 * (max([file_id.size for file_id in fa_file_ids]) + max([file_id.size for file_id in c2h_file_ids]))) - # allows pass-through of memory override from --consMemory - if memory_override: - mem = memory_override + total_size = sum([file_id.size for file_id in fa_file_ids + c2h_file_ids]) + disk = 3 * total_size + # 2x the largest subtree's inputs: halAppendCactusSubtree peaked at 1.15x of that across + # 576 VGP alignments, and never used more than 23% of the old 5x. --doubleMem covers a + # workload outside that envelope. + mem = cactus_clamp_memory(2 * (max([file_id.size for file_id in fa_file_ids]) + max([file_id.size for file_id in c2h_file_ids]))) + # this is the pass that actually runs halAppendCactusSubtree, once per subtree root, so + # its cost is the whole c2h+fa volume rather than any one subtree's. io_bytes reads all + # of that back out of the jobstore and writes the merged HAL, which came to ~13% of the + # input on the 577-way. return job.addChildJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, results, event=event, cacheBytes=cacheBytes, cacheMDC=cacheMDC, cacheRDC=cacheRDC, cacheW0=cacheW0, chunk=chunk, inMemory=inMemory, checkpointInfo=checkpointInfo, acyclicEvent=acyclicEvent, has_resources=True, - disk=disk, memory=mem).rv() + disk=disk, memory=mem, + walltime=cactus_walltime(HAL_APPEND_SECS_PER_GIB * total_size / 2**30, + io_bytes=int(1.2 * total_size))).rv() cactus_call(parameters=["halSetMetadata", hal_path, "CACTUS_COMMIT", cactus_commit]) config_path = os.path.join(work_dir, 'config.xml') @@ -300,11 +323,11 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i ''' run the entire progressive workflow ''' # run the usual unzip / rename, even before preprocessing - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, walltime=cactus_walltime()) # start with the preprocessor if not options.skipPreprocessor: - pp_job = sanitize_job.addFollowOnJobFn(preprocess_all, options, config_node, sanitize_job.rv()) + pp_job = sanitize_job.addFollowOnJobFn(preprocess_all, options, config_node, sanitize_job.rv(), walltime=cactus_walltime()) seq_id_map = pp_job.rv() sanitize_job = pp_job else: @@ -322,16 +345,24 @@ def progressive_workflow(job, options, config_node, mc_tree, og_map, input_seq_i scaled_tree = get_ancestor_scaled_tree(mc_tree, root_event, max_div, branch_scale=options.branchScale, upweight_ancestors=upweight_ancestors) - progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event) + # progressive_schedule builds one job per ancestor, and the get_subtree() it does for each + # is O(tree size): 31 ms per ancestor measured on the 577-way's 1153-node tree, so ~18 s for + # the whole of it, plus Toil writing out the 576 job descriptions. Keying off the ancestor + # count keeps a tree much bigger than that from being clipped; anything ordinary floors. + progressive_job = sanitize_job.addFollowOnJobFn(progressive_schedule, options, config_node, seq_id_map, scaled_tree, og_map, root_event, + walltime=cactus_walltime(0.2 * len(scaled_tree.getSubtreeRootNames()))) # then do the hal export + # the first export_hal pass stages nothing: it resolves the c2h/fasta promises into sizes and + # re-dispatches itself with them, and that resourced pass sizes its own walltime hal_export_job = progressive_job.addFollowOnJobFn(export_hal, mc_tree, config_node, seq_id_map, og_map, - progressive_job.rv(), event=root_event, memory_override=options.consMemory) + progressive_job.rv(), event=root_event, walltime=cactus_walltime()) return hal_export_job.rv() def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file") parser.add_argument("outputHal", type=str, help = "Output HAL file") @@ -368,7 +399,7 @@ def main(): "Standard suffixes like K, Ki, M, Mi, G or Gi are supported (default=bytes))", default=None) parser.add_argument("--consRetainPages", choices=['auto', '0', '1'], default=None, help="Whether cactus_consolidated keeps the memory pages jemalloc frees, which is much faster but takes 2-3x the peak memory. " - "auto (the default, from in the config) keeps them unless the memory estimate exceeds what the job can be given") + "auto (the default, from in the config) keeps them only when the estimate fits in memory_retain_auto_fraction of what the job can be given") parser.add_argument("--intermediateResultsUrl", help="URL prefix to save intermediate results like DB dumps to (e.g. " "prefix-dump-caf, prefix-dump-avg, etc.)", default=None) @@ -496,7 +527,7 @@ def main(): loadDnaBrnnModel(toil, config_node) # run the whole workflow - hal_id = toil.start(Job.wrapJobFn(progressive_workflow, options, config_node, mc_tree, og_map, input_seq_id_map)) + hal_id = toil.start(Job.wrapJobFn(progressive_workflow, options, config_node, mc_tree, og_map, input_seq_id_map, walltime=cactus_walltime())) toil.exportFile(hal_id, makeURL(options.outputHal)) diff --git a/src/cactus/refmap/cactus_graphmap.py b/src/cactus/refmap/cactus_graphmap.py index d53e1207b..80918d785 100644 --- a/src/cactus/refmap/cactus_graphmap.py +++ b/src/cactus/refmap/cactus_graphmap.py @@ -18,12 +18,13 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime +from cactus.shared.common import GZIP_COMPRESS_BYTES_PER_SEC from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import unzip_gz, zip_gz @@ -39,11 +40,13 @@ from cactus.shared.common import cactus_clamp_memory from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set from cactus.refmap.cactus_minigraph import check_sample_names, minigraph_gfa_from_pansn, read_chromfile +from cactus.refmap.cactus_minigraph import GFA_RENAME_SECS_PER_GB, RAW_BYTES_PER_GZ_BYTE from sonLib.nxnewick import NXNewick from sonLib.bioio import getTempDirectory, getTempFile def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (will be modified if necessary to include graph Fasta sequence) (or chromfile with --batch)") parser.add_argument("minigraphGFA", nargs='?', default='', type=str, @@ -254,7 +257,7 @@ def graph_map(options): # run the workflow # output_dict is chrom -> paf_id, gfa_fa_id, gaf_id, unfiltered_paf_id, paf_filter_log, paf_was_filtered output_dict = toil.start(Job.wrapJobFn(minigraph_batch_separate_workflow, options, config_wrapper, input_dict, graph_event, True, - in_gaf_id=in_gaf_id)) + in_gaf_id=in_gaf_id, walltime=cactus_walltime())) export_graphmap_output(options, config_node, input_map, output_dict, toil) @@ -330,7 +333,8 @@ def minigraph_batch_workflow(job, options, config, input_dict, graph_event, sani else: chrom_options = options mgwf_job = job.addChildJobFn(minigraph_workflow, chrom_options, config, seq_id_map, gfa_id, graph_event, - sanitize, ref_collapse_paf_id, pansn_gfa_input, in_gaf_id=in_gaf_id) + sanitize, ref_collapse_paf_id, pansn_gfa_input, in_gaf_id=in_gaf_id, + walltime=cactus_walltime()) output_dict[chrom] = mgwf_job.rv() return output_dict @@ -344,15 +348,46 @@ def add_separate_ref_contigs_job(batch_job, options, config, input_dict): reference = options.reference[0] if type(options.reference) is list else options.reference return batch_job.addFollowOnJobFn(separate_ref_contigs_batch, config, input_dict, batch_job.rv(), reference, getattr(options, 'permissiveContigFilter', None), - whole_genome_ref=getattr(options, 'mgSplitWholeGenomeRef', False)) + whole_genome_ref=getattr(options, 'mgSplitWholeGenomeRef', False), + walltime=cactus_walltime()) def minigraph_batch_separate_workflow(job, options, config, input_dict, graph_event, sanitize, pansn_gfa_input=True, in_gaf_id=None): """ minigraph_batch_workflow followed by the separation pass, for callers that just want the final result and add nothing after it """ batch_job = job.addChildJobFn(minigraph_batch_workflow, options, config, input_dict, graph_event, sanitize, - pansn_gfa_input, in_gaf_id=in_gaf_id) + pansn_gfa_input, in_gaf_id=in_gaf_id, walltime=cactus_walltime()) return add_separate_ref_contigs_job(batch_job, options, config, input_dict).rv() +# Walltime estimates for the graphmap jobs. Everything below was measured on the two HPRC +# pangenome runs, the biggest graphmaps we have logs for: the whole-panel pass of v2.0 (one 8.3 +# GiB raw GFA, a 34.4 GiB merged PAF) and the per-chromosome pass of v2.1 (~0.6 GiB of GFA and +# ~2.2 GiB of PAF each). + +# The merged minigraph PAF comes out at about 4x the raw GFA it was mapped against: 34.4 GiB +# against 8.3 GiB whole-panel, 2.2 GiB against 0.6 GiB per chromosome. The PAF is only a promise +# while the workflow is being built, so this is how its size reaches the walltimes of the jobs +# that make it, read it and copy it. +PAF_BYTES_PER_GFA_BYTE = 4 + +# Seconds per GB of raw GFA for rgfa2paf: 402s on the 8.3 GiB whole-panel GFA against a 112s worst +# case on the 0.6 GiB per-chromosome ones, ie ~40 s/GB on top of a ~90s fixed cost. +RGFA2PAF_SECS_PER_GB = 40 + +# Seconds per GB of raw GFA for filter_paf_deletions: 4337s for filter-paf-deletions plus ~500s +# for the vg convert that precedes it on the whole-panel GFA, against 802s worst case on the +# per-chromosome ones. +# +# This looks far too high under --mgSplitWholeGenomeRef, where the whole job came to 563 s over +# 50 jobs against a graph term of 4386 s. Do not cut the rate to close that gap: the rate is +# right and the size it is applied to is wrong. gfa_id_size is the compressed GFA expanded by +# the hardcoded 10 above, which the whole-panel measurement supports (0.83 GiB gz to 8.3 GiB raw) +# but these runs do not -- their 27 unzip_gz jobs report 3.12 GB of raw GFA from a 757 MB input, +# a ratio of 4.12. Cutting the rate to 120 fits the split runs and leaves the whole-panel case +# asking 1669 s for work that was measured at 4837 s, which is the wrong trade. The expansion +# ratio is the thing to fix, once it is known why the two graphs differ by 2.4x. +FILTER_PAF_DELETIONS_SECS_PER_GB = 500 + + def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sanitize, ref_collapse_paf_id, pansn_gfa_input=True, in_gaf_id=None): """ Overall workflow takes command line options and returns (paf-id, (optional) fa-id) """ @@ -364,28 +399,34 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa if type(options.reference) is list: options.reference = options.reference[0] - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) mg_cores = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "cpu", typeFn=int, default=1) # enforce unique prefixes and unzip fastas if sanitize: - sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() # add unique prefixes to the input PAF if ref_collapse_paf_id: + # one awk pass over the PAF, then staging it in and out. --collapseRefPAF was not used in + # any of the runs we have logs for, so the ~50 MB/s awk rate is inferred, not measured ref_collapse_paf_id = root_job.addChildJobFn(add_paf_prefixes, ref_collapse_paf_id, options.reference, - disk=2*ref_collapse_paf_id.size).rv() + disk=2*ref_collapse_paf_id.size, + walltime=cactus_walltime(20 * ref_collapse_paf_id.size / 1e9, + io_bytes=2*ref_collapse_paf_id.size)).rv() # convert the GFA from PanSN to Cactus names if pansn_gfa_input: # the renaming pass decompresses the GFA before bgzipping it back up, so it needs room for # the raw copy (reckoned at 10x, as elsewhere) on top of the compressed input and output rename_gfa_job = root_job.addChildJobFn(minigraph_gfa_from_pansn, genome_names, options.minigraphGFA, gfa_id, - disk=gfa_id.size*12) - new_root_job = Job() + disk=gfa_id.size*12, + walltime=cactus_walltime(GFA_RENAME_SECS_PER_GB * gfa_id.size / 1e9, + io_bytes=2*gfa_id.size)) + new_root_job = Job(walltime=cactus_walltime()) root_job.addFollowOn(new_root_job) root_job = new_root_job gfa_id = rename_gfa_job.rv(0) @@ -394,24 +435,36 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa # job just as its mapping would have been in_gaf_map = None if in_gaf_id: + # one pass over the reused GAF, splitting it per genome: I/O rather than compute split_gaf_job = root_job.addChildJobFn(split_gaf_by_event, in_gaf_id, genome_names, options.inGAF, - disk=12*in_gaf_id.size) + disk=12*in_gaf_id.size, + walltime=cactus_walltime(0, io_bytes=RAW_BYTES_PER_GZ_BYTE * in_gaf_id.size)) in_gaf_map = split_gaf_job.rv() zipped_gfa = options.minigraphGFA.endswith('.gz') if options.outputFasta: # convert GFA to fasta scale = 5 if zipped_gfa else 1 + # gfatools barely scales with GFA size -- 199s on the whole-panel GFA against a 239s worst + # case per chromosome -- so a flat compute term plus the staging is the honest shape fa_job = root_job.addChildJobFn(make_minigraph_fasta, gfa_id, options.outputFasta, graph_event, - disk=scale*2*gfa_id_size, memory=cactus_clamp_memory(2*scale*gfa_id_size)) + disk=scale*2*gfa_id_size, memory=cactus_clamp_memory(2*scale*gfa_id_size), + walltime=cactus_walltime(300, io_bytes=2*gfa_id_size)) fa_id = fa_job.rv() if zipped_gfa: # gaf2paf needs unzipped gfa, so we take care of that upfront - gfa_unzip_job = root_job.addChildJobFn(unzip_gz, options.minigraphGFA, gfa_id, delete_original=False, disk=5*gfa_id_size) + # gunzip itself is fast (27s for the 0.83 GiB compressed whole-panel GFA); what costs is + # writing the ~10x bigger raw GFA back to the jobstore + gfa_unzip_job = root_job.addChildJobFn(unzip_gz, options.minigraphGFA, gfa_id, delete_original=False, disk=5*gfa_id_size, + walltime=cactus_walltime(60, io_bytes=(1 + RAW_BYTES_PER_GZ_BYTE) * gfa_id_size)) gfa_id = gfa_unzip_job.rv() gfa_id_size *= 10 options.minigraphGFA = options.minigraphGFA[:-3] + + # size of the merged PAF every job below either makes, reads or copies + paf_bytes = PAF_BYTES_PER_GFA_BYTE * gfa_id_size + if in_gaf_id: # resolving a reused GAF is the same work for every genome, so a GAF that does not belong # to this graph fails identically in all of them -- once per genome, after the fan-out, and @@ -421,9 +474,11 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa check_parent = gfa_unzip_job if zipped_gfa else root_job check_parent.addFollowOnJobFn(check_reusable_gaf, config, in_gaf_id, gfa_id, genome_names, options.inGAF, options.minigraphGFA, - disk=4*gfa_id_size, memory=cactus_clamp_memory(2*gfa_id_size)) + disk=4*gfa_id_size, memory=cactus_clamp_memory(2*gfa_id_size), + walltime=cactus_walltime(GAF_CHECK_SECS, io_bytes=gfa_id_size + in_gaf_id.size)) - paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event, in_gaf_map) + paf_job = Job.wrapJobFn(minigraph_map_all, options, config, gfa_id, seq_id_map, graph_event, in_gaf_map, + walltime=cactus_walltime()) root_job.addFollowOn(paf_job) collapse_paf_id = ref_collapse_paf_id @@ -432,21 +487,25 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa # if --refFromGFA is specified, we get the entire alignment from that, otherwise we just take contigs # that didn't get mapped by anything else gfa2paf_job = Job.wrapJobFn(extract_paf_from_gfa, gfa_id, options.minigraphGFA, options.reference, graph_event, paf_job.rv(0) if not options.refFromGFA else None, - disk=gfa_id_size, memory=cactus_clamp_memory(gfa_id_size)) + disk=gfa_id_size, memory=cactus_clamp_memory(gfa_id_size), + walltime=cactus_walltime(120 + RGFA2PAF_SECS_PER_GB * gfa_id_size / 1e9, + io_bytes=gfa_id_size + (0 if options.refFromGFA else paf_bytes))) if options.refFromGFA: root_job.addChild(gfa2paf_job) else: paf_job.addFollowOn(gfa2paf_job) collapse_mode = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "collapse", typeFn=str, default="none") if collapse_mode in ['reference', 'all', 'nonref']: - collapse_job = paf_job.addChildJobFn(self_align_all, config, seq_id_map, options.reference, collapse_mode) + collapse_job = paf_job.addChildJobFn(self_align_all, config, seq_id_map, options.reference, collapse_mode, walltime=cactus_walltime()) if ref_collapse_paf_id: collapse_paf_id = collapse_job.addFollowOnJobFn(merge_pafs, {"1":collapse_job.rv(), "2":ref_collapse_paf_id}, - disk=gfa_id_size).rv() + disk=gfa_id_size, + walltime=merge_pafs_walltime(paf_bytes)).rv() else: collapse_paf_id = collapse_job.rv() - merge_paf_job = Job.wrapJobFn(merge_pafs, {"1" : paf_job.rv(0), "2" : gfa2paf_job.rv()}, disk=gfa_id_size) + merge_paf_job = Job.wrapJobFn(merge_pafs, {"1" : paf_job.rv(0), "2" : gfa2paf_job.rv()}, disk=gfa_id_size, + walltime=merge_pafs_walltime(paf_bytes)) paf_job.addFollowOn(merge_paf_job) gfa2paf_job.addFollowOn(merge_paf_job) out_paf_id = merge_paf_job.rv() @@ -466,9 +525,15 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa del_filter_job = prev_job.addFollowOnJobFn(filter_paf_deletions, out_paf_id, gfa_id, del_filter, del_filter_threshold, del_size_threshold, disk=8*gfa_id_size, cores=mg_cores, - memory=cactus_clamp_memory(30*gfa_id_size)) + memory=cactus_clamp_memory(30*gfa_id_size), + walltime=cactus_walltime(600 + FILTER_PAF_DELETIONS_SECS_PER_GB * gfa_id_size / 1e9, + io_bytes=gfa_id_size + 2*paf_bytes)) + # the 600s floor is for the tail: per chromosome this gzip has a 79s median but a 1390s + # worst case, which is contention on the shared filesystem, not PAF size unfiltered_paf_id = prev_job.addFollowOnJobFn(zip_gz, 'mg.paf.unfiltered', out_paf_id, delete_original=False, - disk=gfa_id_size).rv() + disk=gfa_id_size, + walltime=cactus_walltime(600 + paf_bytes / GZIP_COMPRESS_BYTES_PER_SEC, + io_bytes=2*paf_bytes)).rv() out_paf_id = del_filter_job.rv(0) filtered_paf_log = del_filter_job.rv(1) paf_was_filtered = del_filter_job.rv(2) @@ -476,7 +541,8 @@ def minigraph_workflow(job, options, config, seq_id_map, gfa_id, graph_event, sa if collapse_paf_id: # note: the collapse paf doesn't get merged into unfiltered_paf - merge_collapse_job = prev_job.addFollowOnJobFn(merge_pafs, {"1" : out_paf_id, "2" : collapse_paf_id}, disk=gfa_id_size) + merge_collapse_job = prev_job.addFollowOnJobFn(merge_pafs, {"1" : out_paf_id, "2" : collapse_paf_id}, disk=gfa_id_size, + walltime=merge_pafs_walltime(paf_bytes)) out_paf_id = merge_collapse_job.rv() return out_paf_id, fa_id if options.outputFasta else None, paf_job.rv(1), unfiltered_paf_id, filtered_paf_log, paf_was_filtered @@ -511,13 +577,41 @@ def make_minigraph_fasta(job, gfa_file_id, gfa_file_path, name): return job.fileStore.writeGlobalFile(fa_path) +# minigraph mapping, per job and per GB of sanitized fasta. Both re-fitted on the 27,097 +# mappings of an HPRC v2.1 run, the first at scale with the faster minigraph, which separate +# cleanly into the two populations the line was drawn through: 456 whole-genome haplotypes +# (~3.1 GB of fasta, p50 712 s, p90 1053 s, max 1499 s) and 26,641 per-chromosome ones +# (0.05-0.26 GB, p50 100 s, p99 440 s, max 713 s). Through those the slope is 250 s/GB at the +# p90 and 330 s/GB at the p99, so 400 sits above the measurement everywhere; 1200 came from the +# slower fork, where the same whole-genome mapping ran to a p90 of 4062 s. +# +# The intercept is the tail allowance, and it used to be sized against a per-chromosome max of +# 3521 s over 11,390 invocations -- 3x its own p99, taken to be cluster contention rather than +# anything the fasta size can see. That tail is now 713 s against a p99 of 440 s, 1.6x rather +# than 3x, so 2000 was buying a cushion that costs more than it is worth: it put all 18,927 +# mapping jobs of that run above an hour, which on a cluster whose shortest partition is an hour +# is the entire scheduling decision. 600 holds every per-chromosome mapping under the hour while +# still covering the observed worst by 3x, and leaves the whole-genome pass -- where the per-GB +# term dominates anyway -- around 2 h against a 1499 s worst. +MINIGRAPH_MAP_SECS = 600 +MINIGRAPH_MAP_SECS_PER_GB = 400 + +# Re-deriving one genome's PAF from a GAF it already has (--inGAF): the same gaf2unstable/gaffilter/ +# gaf2paf chain minigraph_map_one runs, without the minigraph. Those three came to p99 18s and max +# 29s per genome across the HPRC runs (gaf2unstable|gaffilter n=11380), so this is the GAF read and +# the graph load rather than the chain itself, keyed off the shard the way the memory request is. +TRANSLATE_GAF_SECS_PER_GB = 400 + +# check_reusable_gaf loads the graph and resolves a sample of records against it. +GAF_CHECK_SECS = 600 + def minigraph_map_all(job, options, config, gfa_id, fa_id_map, graph_event, in_gaf_map=None): """ top-level job to run the minigraph mapping in parallel, returns paf. a genome that in_gaf_map already has mappings for has its PAF re-derived from them rather than being mapped again -- see translate_gaf_one() """ # hang everything on this job, to self-contain workflow - top_job = Job() + top_job = Job(walltime=cactus_walltime()) job.addChild(top_job) mg_cores = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "cpu", typeFn=int, default=1) @@ -552,24 +646,29 @@ def minigraph_map_all(job, options, config, gfa_id, fa_id_map, graph_event, in_g gaf_shard_id = in_gaf_map[event] map_job = top_job.addChildJobFn(translate_gaf_one, config, event_name, gaf_shard_id, gfa_id, genome_names, disk=12*gaf_shard_id.size + 2*gfa_id.size, - memory=cactus_clamp_memory(24*gaf_shard_id.size + 4*gfa_id.size)) + memory=cactus_clamp_memory(24*gaf_shard_id.size + 4*gfa_id.size), + walltime=cactus_walltime(TRANSLATE_GAF_SECS_PER_GB * gaf_shard_id.size / 1e9, + io_bytes=2*gaf_shard_id.size + gfa_id.size)) else: map_job = top_job.addChildJobFn(minigraph_map_one, config, event_name, fa_id, gfa_id, cores=mg_cores, disk=5*fa_id.size + gfa_id.size, - memory=cactus_clamp_memory(mem)) + memory=cactus_clamp_memory(mem), + walltime=cactus_walltime(MINIGRAPH_MAP_SECS + MINIGRAPH_MAP_SECS_PER_GB * fa_id.size / 1e9, + io_bytes=2*fa_id.size + gfa_id.size)) gaf_id_map[event] = map_job.rv(0) paf_id_map[event] = map_job.rv(1) # merge up. these two are the merges whose inputs scale with the number of genomes, so they get # sized off them rather than taking the default; the GAF one also bgzips, so give it the mapping - # cores instead of leaving bgzip single-threaded + # cores instead of leaving bgzip single-threaded. merge_pafs_sized resolves the promises and + # sets the real disk and walltime from them, so these two are just the coordination job merge_name = getattr(options, 'mg_chrom_name', None) if options.batch else None merge_name = merge_name if merge_name else 'merged' paf_merge_job = top_job.addFollowOnJobFn(merge_pafs_sized, paf_id_map, - merged_name='{}.paf'.format(merge_name)) + merged_name='{}.paf'.format(merge_name), walltime=cactus_walltime()) gaf_merge_job = top_job.addFollowOnJobFn(merge_pafs_sized, gaf_id_map, gzip=True, merged_name='{}.gaf'.format(merge_name), - gzip_cores=mg_cores) + gzip_cores=mg_cores, walltime=cactus_walltime()) return paf_merge_job.rv(), gaf_merge_job.rv() @@ -988,6 +1087,23 @@ def stable_gaf_to_paf(job, config, gaf_path, gfa_path, regranulated=False): # return the stable gaf (minigraph output) and the unstable paf return job.fileStore.writeGlobalFile(pansn_gaf_path), job.fileStore.writeGlobalFile(unstable_paf_path) +# What is left of a merge_pafs job once its staging is accounted for: catFiles runs no command, so +# this is worker startup and the python copy loop. +MERGE_PAF_SECS = 60 + +def merge_pafs_walltime(merged_bytes, gzip=False, cores=1): + """ walltime for a merge_pafs job whose output comes to roughly merged_bytes. The job is all + I/O -- every input is read out of the jobstore and the concatenation written back -- except + with gzip=True, which bgzips the result on the way out, threaded when it is given cores. + + io_bytes is 4x rather than 2x because the bytes move twice: once staging in and out of the + jobstore, and again locally, where catFiles reads every input back and writes the joined + file. At 2x the whole-panel merge of the HPRC v2.0 PAF came to under half an hour. """ + secs = MERGE_PAF_SECS + if gzip: + secs += merged_bytes / (GZIP_COMPRESS_BYTES_PER_SEC * max(1, cores)) + return cactus_walltime(secs, io_bytes=4*merged_bytes) + def merge_pafs(job, paf_file_id_map, gzip=False, merged_name=None): """ merge up some pafs. merged_name is what the merged file is called on disk: getLocalTempFile() would give it an anonymous .tmp, which is all anyone reading the log of the bgzip below would see. @@ -1013,7 +1129,8 @@ def merge_pafs_sized(job, paf_file_id_map, gzip=False, merged_name=None, gzip_co thread. it is passed on as the child's cores, where toil eating it is exactly what we want """ total_size = sum(paf_id.size for paf_id in paf_file_id_map.values() if paf_id) return job.addChildJobFn(merge_pafs, paf_file_id_map, gzip=gzip, merged_name=merged_name, - cores=gzip_cores, disk=max(total_size * 3, 2**31)).rv() + cores=gzip_cores, disk=max(total_size * 3, 2**31), + walltime=merge_pafs_walltime(total_size, gzip=gzip, cores=gzip_cores)).rv() def extract_paf_from_gfa(job, gfa_id, gfa_path, ref_event, graph_event, ignore_paf_id): """ make a paf directly from the rGFA tags. rgfa2paf supports other ranks, but we're only @@ -1040,11 +1157,17 @@ def extract_paf_from_gfa(job, gfa_id, gfa_path, ref_event, graph_event, ignore_p cactus_call(parameters=cmd, outfile=paf_path) return job.fileStore.writeGlobalFile(paf_path) +# Seconds per GB of fasta for a minimap2 -xasm5 self-alignment. There is no measurement behind +# this one: defaults to "none", so self_align ran in none of the runs we have +# logs for. It is minigraph's own mapping rate standing in, and wants replacing with a +# measurement the first time --collapse is used at panel scale. +SELF_ALIGN_SECS_PER_GB = 1200 + def self_align_all(job, config, seq_id_map, reference, collapse_mode): """ run self-alignment. if reference event given, just run on that, otherwise do all genomes """ assert collapse_mode in ['reference', 'all', 'nonref'] assert reference or collapse_mode == 'all' - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) events = [] for event in seq_id_map.keys(): @@ -1053,17 +1176,20 @@ def self_align_all(job, config, seq_id_map, reference, collapse_mode): events.append(event) mg_cores = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap"), "cpu", typeFn=int, default=1) paf_dict = {} - paf_size = 0 for event in events: - paf_size += seq_id_map[event].size collapse_job = root_job.addChildJobFn(self_align, config, event, seq_id_map[event], disk=4*seq_id_map[event].size, memory=4*seq_id_map[event].size, - cores=mg_cores) + cores=mg_cores, + walltime=cactus_walltime(600 + SELF_ALIGN_SECS_PER_GB * seq_id_map[event].size / 1e9, + io_bytes=2*seq_id_map[event].size)) paf_dict[event] = collapse_job.rv() - merge_paf_job = root_job.addFollowOnJobFn(merge_pafs, paf_dict, - disk=4*paf_size) + # every input here is a promise, so the merge is sized the same way the mapping merges are: by + # a coordination job that runs once they have resolved and can read their real sizes. what a + # self-alignment PAF comes to as a fraction of the fasta it came from has never been measured + # -- defaults to "none" -- and this way it does not have to be guessed + merge_paf_job = root_job.addFollowOnJobFn(merge_pafs_sized, paf_dict, walltime=cactus_walltime()) return merge_paf_job.rv() def self_align(job, config, seq_name, seq_id): @@ -1101,6 +1227,12 @@ def apply_mgsplit_filter_overrides(config_node): graphmap_node.attrib["minGAFBlockLength"] = "0" graphmap_node.attrib["delFilter"] = "-1" +# Seconds per GB of PAF for filter_paf: gaffilter is measured at 1475s on the 34.4 GiB whole-panel +# PAF and 270s worst case on the 2.2 GiB per-chromosome ones, and the python line-by-line pass that +# always runs adds about as much again at ~60 MB/s. Lives here, next to the job, because +# cactus-graphmap-split and cactus-align both schedule it. +FILTER_PAF_SECS_PER_GB = 60 + def filter_paf(job, paf_id, config, reference=None): """ run basic paf-filtering. these are quick filters that are best to do on-the-fly when reading the paf and as such, they are called by cactus-graphmap-split and cactus-align, not here diff --git a/src/cactus/refmap/cactus_graphmap_join.py b/src/cactus/refmap/cactus_graphmap_join.py index ecc694896..7bcc3c5a3 100644 --- a/src/cactus/refmap/cactus_graphmap_join.py +++ b/src/cactus/refmap/cactus_graphmap_join.py @@ -35,13 +35,13 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.multiCactusTree import MultiCactusTree -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import vg_chrom_name from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import unzip_gz, write_s3 @@ -66,8 +66,180 @@ import pysam +# --- per-job walltime estimates --------------------------------------------------------------- +# +# Nearly every job below scales with the size of the per-chromosome VG files that came in, so the +# estimates are expressed as seconds per GB of those files. They were fit on three whole-genome +# HPRC runs -- hprc-v2.0-mc-chm13-join, hprc-v2.1-mc-chm13 and its eval variant -- each 25 +# chromosomes of 0.005 to 33.4 GB summing to 408 GB, by grouping every logged command by the +# worker temp dir it ran in to recover per-job totals. Each number sits at or just above the +# worst per-GB ratio the three produced; --walltimeFactor is the margin on top of that. They are +# also all 64-core measurements, so a run with far fewer --indexCores is slower than this and +# gets there through --doubleTime. + +def scaled_walltime(secs_per_gb, size_bytes, io_multiple=2): + """ Walltime for a job whose cost tracks the size of what it reads: secs_per_gb of compute per + GB of size_bytes -- one chromosome's graph, or the sum over all of them -- plus io_multiple + times that many bytes staged through the jobstore. size_bytes must be a real FileID size (a + promise has none). """ + return cactus_walltime(secs_per_gb * size_bytes / 1e9, io_bytes=int(size_bytes * io_multiple)) + +# clip_vg, the whole load/GFAffix/clip/normalise/validate pipeline. n=75: the 'full' phase (the +# only one that runs GFAffix) came to 597-630 s/GB at p50 and 729-951 s/GB at p99, the 'clip' +# phase to 336-443 and 454-590. the largest chromosome, 33.4 GB, took 6.1 hours in 'full' +CLIP_VG_SECS_PER_GB = {'full' : 800, 'clip' : 600} + +# vg_clip_vg: vg clip three times plus vg validate. n=75, 292-349 s/GB at p50, 382-440 at the max +VG_CLIP_VG_SECS_PER_GB = 450 + +# vg_to_og: vg convert -f | grep, then odgi build. n=75, 307-310 s/GB at p50, 357-370 at the max +VG_TO_OG_SECS_PER_GB = 400 + +# vg_to_gfa: vg convert -f. n=225 (25 chromosomes x 3 phases), 161-196 s/GB at p99 and 212-228 at +# the max. the unchopped variant streams through an extra vg mod -u, which no logged run +# exercised, so its +40% is a guess. the GFA written back is several times the VG, hence the +# wider I/O multiple at those call sites +VG_TO_GFA_SECS_PER_GB = 250 +VG_TO_GFA_UNCHOPPED_SECS_PER_GB = 350 + +# drop_graph_event: one vg paths -d over the full-phase graph, written back out. it post-dates +# these logs, so it is taken from vg convert -f on the same graphs, which loads and re-serialises +# the same way at 36-230 s/GB, at the fast end of that since -d does much less in between +DROP_GRAPH_EVENT_SECS_PER_GB = 60 + +# snarl_stats: vg stats -R --snarl-sample. n=75, 24-26 s/GB at p50 and 38-64 at p99 (the high +# ratios are the small chromosomes, where fixed overhead dominates); the max was 1119 s on 33.4 GB +SNARL_STATS_SECS_PER_GB = 60 + +# make_odgi_viz: odgi sort, odgi paths -L and odgi viz. n=75, 76-83 s/GB at p50, 132-158 at p99. +# --draw instead runs odgi layout and odgi draw, which the option's own help calls very slow and +# which appear in no log at all, so 4x the viz number is a guess +ODGI_VIZ_SECS_PER_GB = 150 +ODGI_DRAW_SECS_PER_GB = 600 + +# compute_gref_paths: vg paths -L, -d and -u, then a rewritten graph. --gref was in no logged +# run; this is below the measured band for the per-chromosome jobs that load, modify and +# re-serialise a graph (292-590 s/GB), because it gets --indexCores and can short-circuit +COMPUTE_GREF_PATHS_SECS_PER_GB = 200 + +# path_coverage_job: vg paths -E -v (n=75, ~20 s/GB at the top) followed by interval merging in +# python, which issues no command and so is invisible in the logs. 3x the measured half +PATH_COVERAGE_SECS_PER_GB = 60 + +# ref_gaps_job: vg depth -m0, in no logged run. its real driver is the length of the reference +# contig -- it emits one line per reference base -- and that is not in scope at the call site, so +# this is per GB of VG instead, at ~5x the slowest per-chromosome vg command measured on this data +REF_GAPS_SECS_PER_GB = 150 + +# deconstruct, keyed on the phase graph (make_vcf is itself a job, so its ids are resolved). +# n=75, 59-80 s/GB at p50 but 127-256 at the max: the spread is wide because deconstruct tracks +# snarl and allele structure, not just bytes +DECONSTRUCT_SECS_PER_GB = 300 + +# vcfbub and chunked_vcfwave, per GB of the graph deconstruct ran on -- the VCFs they read are +# promises at the call site, but scale with the same panel. n=75 each: vcfbub 7-10 s/GB at p50 +# and 11-24 at the max; chunked_vcfwave (its own bcftools pipeline and chunk bgzips, not the +# vcfwave jobs it fans out to) 11-24 at p50 and 13-37 at p99 +VCFBUB_SECS_PER_GB = 20 +CHUNKED_VCFWAVE_SECS_PER_GB = 40 + +# join_vg: one vg ids -j over every chromosome, 3308-4909 s on 408 GB of graph (8-12 s/GB). it +# reads all of them mutable and writes all of them back, so the staging term is the bigger half +JOIN_VG_SECS_PER_GB = 15 + +# make_vg_indexes: merge the per-chromosome GFAs, then vg gbwt --gbz-format, bgzip and vg snarls. +# n=9, 7.0-9.1 ks over 408 GB, i.e. ~22 s/GB, at 64 cores. the unchopped call site asks for one +# core, and its bgzip of a whole-genome GFA is what does not survive that (371-553 s at 64 +# threads), hence the separate single-core number +MAKE_VG_INDEXES_SECS_PER_GB = 25 +MAKE_VG_INDEXES_1CORE_SECS_PER_GB = 60 + +# extract_gbz_fasta: vg paths -F over the whole-genome GBZ, once per --vcfReference. 508-522 s +# over 408 GB in all three runs (1.27 s/GB); 4x that covers staging the GBZ, which is a promise +# here and so cannot go in io_bytes +EXTRACT_GBZ_FASTA_SECS_PER_GB = 5 + +# extract_vg_fasta: the same extraction done as one vg paths -F per chromosome, staging every +# chromosome VG first. neither call site was exercised, so the compute term is 25 graph loads +# benchmarked against vg paths -E -v on the same graphs (86-666 s each); the staging is the +# honest dominant term and is exactly known here +EXTRACT_VG_FASTA_SECS_PER_GB = 20 + +# make_xg: vg convert -x, in no logged run. this is just above the top of the range of the +# whole-genome index builds measured on the identical GBZ (vg gbwt, vg index -j, vg snarls, vg +# haplotypes: 5-19 s/GB), XG being the largest of those formats +MAKE_XG_SECS_PER_GB = 30 + +# make_giraffe_indexes: vg index -j and vg minimizer. n=3, 2604-3279 s over 408 GB (6.4-8.0 +# s/GB) at 64 cores. 12 leaves room for the second, long-read minimizer pass --lrGiraffe adds, +# which none of the runs built +MAKE_GIRAFFE_INDEXES_SECS_PER_GB = 12 + +# make_haplo_index: vg index -j, vg gbwt -r and vg haplotypes -H. n=3, 8.5-12.1 ks over 408 GB +# (21-30 s/GB) at 64 cores +MAKE_HAPLO_INDEX_SECS_PER_GB = 35 + +# odgi_squeeze: no logged run asked for a whole-genome --odgi. squeeze concatenates graphs that +# are already built, so it belongs at the odgi sort end of the measured odgi work (286-707 s per +# chromosome) rather than the odgi build end (4.2-9.8 ks); staging every .og dominates either way +ODGI_SQUEEZE_SECS_PER_GB = 10 + +# merge_snarl_stats and merge_gref_segs: concatenate the per-chromosome TSVs, sort, compress. +# merge_snarl_stats was measured once at 177 s over 408 GB of graph (0.43 s/GB). the real driver +# is the TSV bytes, which are promises here, so this is per GB of graph at ~5x that observation +MERGE_CHROM_TSV_SECS_PER_GB = 2 + +# run_panacus, per GB of graph and per phase: no logged run passed --panacus. each phase is one +# bgzip -d of a merged GFA that make_vg_indexes' own bgzip and grep passes put at 400-550 GB, +# then one panacus histgrowth pass over it per countType (2 by default) +PANACUS_SECS_PER_GB = 20 + +# compute_exclusions_job: interval work in python over every chromosome's coverage table, whose +# only command is a tar. it post-dates these logs, so this is reasoned from shape, not measured +COMPUTE_EXCLUSIONS_SECS_PER_GB = 15 + +# vcf_cat over a whole genome, per GB of graph -- the per-chromosome VCFs are promises, which is +# why the disk request is reckoned the same way. measured: bcftools query/merge/concat came to +# 10-13 ks on the raw VCFs, 2.3-3.2 ks on the bub ones and ~0.8 ks on the wave ones, up to ~30 +# s/GB. the rest covers fix_vcf_ploidies, which is pure python and so absent from the logs +VCF_CAT_SECS_PER_GB = 45 + +# merge_hal, per GB of HAL: the longest job in the module, measured four times at 43.9-47.7 ks +# over ~445 GB of chromosome HALs (~103 s/GB), with ~2.5 ks of halRemoveGenome on top +MERGE_HAL_SECS_PER_GB = 110 + +# merge_sv_gfa, per GB of gzipped minigraph GFA: pure python, so it issues no command and is +# invisible in the logs. a gzipped GFA expands ~8x, and re-compressing that in-process runs at +# the ~16 MB/s this file already measures for python gzip (see fix_vcf_ploidies) +MERGE_SV_GFA_SECS_PER_GB = 500 + +# vcf_cat of the vcfwave chunks, per GB of the raw VCF they were cut from. n=75, 205-272 s/GB at +# p50 and 289-341 at the max, single-core (that call site asks for no cores) +VCF_CAT_CHUNK_SECS_PER_GB = 400 + +# vcfnorm, per GB of the VCF that drives it. the vcfwave site is measured n=75 at 4.9-6.7 ks per +# GB of raw VCF at p50 and 5.9-7.9 at p99, nearly all of it in the closing bcftools norm | +# vcffixup | bgzip. the vcfbub site (bcftoolsNorm, off in every logged run) keys off the bub VCF +# instead, which is only ~0.6x the raw one at HPRC scale, so its number is scaled up to match +VCFNORM_WAVE_SECS_PER_GB = 7000 +VCFNORM_BUB_SECS_PER_GB = 12000 + +# vcfwave, per MB of its bgzipped chunk and per core. pooled over the 5577 chunks of three runs: +# 25998 MB in 1835 hours, i.e. 254 s/MB. that is ~5x the median rate, because those runs used +# the vcflib build from before commit 8f76312f, whose pathological tail is baked into the +# aggregate; that build also ignored --vcfwaveCores, which is why this is a single-thread rate +VCFWAVE_SECS_PER_MB = 254 + +# cat_stats: concatenate the per-chromosome clip and path stats and gzip them. those TSVs are +# promises at the call site and nothing in scope stands in for them, so this is a flat number. +# it is well above what the logs imply (the equivalent tar there ran in 1.4-2.1 s) because +# path-stats can reach millions of rows on a heavily fragmented panel +CAT_STATS_SECS = 600 + + def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("--vg", required=False, nargs='+', default=None, help = "Input vg files (PackedGraph or HashGraph format)") parser.add_argument("--vgFull", nargs='+', default=None, @@ -675,7 +847,8 @@ def graphmap_join(options): wf_output = toil.start(Job.wrapJobFn(graphmap_join_workflow, options, config, vg_ids, hal_ids, sv_gfa_ids, bypass_full_ids, bypass_clip_ids, bypass_filter_ids, - contig_sizes_id=contig_sizes_id)) + contig_sizes_id=contig_sizes_id, + walltime=cactus_walltime())) else: # load up the vgs (from their real paths: options.vg has had the .raw tag stripped) vg_ids = [] @@ -685,7 +858,9 @@ def graphmap_join(options): # run the workflow wf_output = toil.start(Job.wrapJobFn(graphmap_join_workflow, options, config, vg_ids, hal_ids, sv_gfa_ids, - contig_sizes_id=contig_sizes_id)) + contig_sizes_id=contig_sizes_id, + walltime=cactus_walltime())) + #export the split data export_join_data(toil, options, wf_output[0], wf_output[1], wf_output[2], wf_output[3], wf_output[4], wf_output[5], @@ -695,7 +870,7 @@ def vcflib_checks(job, options, config_node): """ run the vcflib checks""" # vcfwave isn't included in the static binary release, so we start by checking it's available if options.vcfwave and options.vcf: - vcfwave_check_job = job.addFollowOnJobFn(check_vcfwave) + vcfwave_check_job = job.addFollowOnJobFn(check_vcfwave, walltime=cactus_walltime()) job = vcfwave_check_job # vcffixup isn't included in the static binary release, so we start by checking it's available @@ -703,7 +878,7 @@ def vcflib_checks(job, options, config_node): wave_norm = getOptionalAttrib(findRequiredNode(config_node, "graphmap_join"), "vcfwaveNorm", typeFn=bool, default=True) bub_norm = getOptionalAttrib(findRequiredNode(config_node, "graphmap_join"), "bcftoolsNorm", typeFn=bool, default=False) if options.vcf and merge_dup and (bub_norm or (options.vcfwave and wave_norm)): - vcffixup_check_job = job.addFollowOnJobFn(check_vcffixup) + vcffixup_check_job = job.addFollowOnJobFn(check_vcffixup, walltime=cactus_walltime()) job = vcffixup_check_job return job @@ -711,7 +886,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, bypass_full_ids=None, bypass_clip_ids=None, bypass_filter_ids=None, contig_sizes_id=None, split_log_id=None): - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) root_job = vcflib_checks(root_job, options, config.xmlRoot) @@ -763,12 +938,14 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, assert len(options.vg) == len(vg_ids) for vg_path, vg_id in zip(options.vg, vg_ids): full_job = Job.wrapJobFn(clip_vg, options, config, vg_path, vg_id, 'full', - disk=vg_id.size * 20, memory=max(2**31, min(vg_id.size * 20, max_mem))) + disk=vg_id.size * 20, memory=max(2**31, min(vg_id.size * 20, max_mem)), + walltime=scaled_walltime(CLIP_VG_SECS_PER_GB['full'], vg_id.size)) root_job.addChild(full_job) full_vg_ids.append(full_job.rv(0)) if 'full' in options.odgi + options.chrom_og + options.viz + options.draw: full_og_job = full_job.addFollowOnJobFn(vg_to_og, options, config, vg_path, full_job.rv(0), - disk=vg_id.size * 16, memory=min(max(og_min_size, vg_id.size * 32), max_mem)) + disk=vg_id.size * 16, memory=min(max(og_min_size, vg_id.size * 32), max_mem), + walltime=scaled_walltime(VG_TO_OG_SECS_PER_GB, vg_id.size, io_multiple=3)) og_chrom_ids['full']['og'].append(full_og_job.rv()) prev_job = root_job @@ -776,7 +953,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # join the ids join_job = prev_job.addFollowOnJobFn(join_vg, options, config, full_vg_ids, disk=sum([f.size for f in vg_ids]), - memory=min(max([f.size for f in vg_ids]) * 4, max_mem)) + memory=min(max([f.size for f in vg_ids]) * 4, max_mem), + walltime=scaled_walltime(JOIN_VG_SECS_PER_GB, + sum(f.size for f in vg_ids))) full_vg_ids = [join_job.rv(i) for i in range(len(vg_ids))] prev_job = join_job @@ -787,7 +966,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vg_path, vg_id, full_vg_id in zip(options.vg, vg_ids, full_vg_ids): drop_graph_event_job = join_job.addFollowOnJobFn(drop_graph_event, config, vg_path, full_vg_id, disk=vg_id.size * 3, - memory=cactus_clamp_memory(min(vg_id.size * 6, max_mem))) + memory=cactus_clamp_memory(min(vg_id.size * 6, max_mem)), + walltime=scaled_walltime(DROP_GRAPH_EVENT_SECS_PER_GB, + vg_id.size)) output_full_vg_ids.append(drop_graph_event_job.rv(0)) full_vg_empty.append(drop_graph_event_job.rv(1)) else: @@ -798,41 +979,49 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, clip_vg_ids = [] clipped_stats = None if options.clip or options.filter: - clip_root_job = Job() + clip_root_job = Job(walltime=cactus_walltime()) prev_job.addFollowOn(clip_root_job) clip_vg_stats = [] assert len(options.vg) == len(full_vg_ids) == len(vg_ids) for vg_path, vg_id, input_vg_id in zip(options.vg, full_vg_ids, vg_ids): clip_job = Job.wrapJobFn(clip_vg, options, config, vg_path, vg_id, 'clip', - disk=input_vg_id.size * 20, memory=max(2**31, min(input_vg_id.size * 20, max_mem))) + disk=input_vg_id.size * 20, memory=max(2**31, min(input_vg_id.size * 20, max_mem)), + walltime=scaled_walltime(CLIP_VG_SECS_PER_GB['clip'], input_vg_id.size)) clip_root_job.addChild(clip_job) clip_vg_ids.append(clip_job.rv(0)) clip_vg_stats.append(clip_job.rv(1)) if 'clip' in options.odgi + options.chrom_og + options.viz + options.draw: clip_og_job = clip_job.addFollowOnJobFn(vg_to_og, options, config, vg_path, clip_job.rv(0), disk=input_vg_id.size * 16, - memory=min(max(og_min_size, input_vg_id.size * 32), max_mem)) + memory=min(max(og_min_size, input_vg_id.size * 32), max_mem), + walltime=scaled_walltime(VG_TO_OG_SECS_PER_GB, + input_vg_id.size, io_multiple=3)) og_chrom_ids['clip']['og'].append(clip_og_job.rv()) # join the stats - clipped_stats = clip_root_job.addFollowOnJobFn(cat_stats, clip_vg_stats).rv() + clipped_stats = clip_root_job.addFollowOnJobFn(cat_stats, clip_vg_stats, + walltime=cactus_walltime(CAT_STATS_SECS)).rv() prev_job = clip_root_job # run the "filter" phase to do the vg clip clipping filter_vg_ids = [] if options.filter: - filter_root_job = Job() + filter_root_job = Job(walltime=cactus_walltime()) prev_job.addFollowOn(filter_root_job) assert len(options.vg) == len(clip_vg_ids) == len(vg_ids) for vg_path, vg_id, input_vg_id in zip(options.vg, clip_vg_ids, vg_ids): filter_job = filter_root_job.addChildJobFn(vg_clip_vg, options, config, vg_path, vg_id, disk=input_vg_id.size * 20, - memory=max(2**31, min(input_vg_id.size * 22, max_mem))) + memory=max(2**31, min(input_vg_id.size * 22, max_mem)), + walltime=scaled_walltime(VG_CLIP_VG_SECS_PER_GB, + input_vg_id.size)) filter_vg_ids.append(filter_job.rv()) if 'filter' in options.odgi + options.chrom_og + options.viz + options.draw: filter_og_job = filter_job.addFollowOnJobFn(vg_to_og, options, config, vg_path, filter_job.rv(), disk=input_vg_id.size * 16, - memory=min(max(og_min_size, input_vg_id.size * 64), max_mem)) + memory=min(max(og_min_size, input_vg_id.size * 64), max_mem), + walltime=scaled_walltime(VG_TO_OG_SECS_PER_GB, + input_vg_id.size, io_multiple=3)) og_chrom_ids['filter']['og'].append(filter_og_job.rv()) @@ -846,20 +1035,24 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, hal_merge_job = job.addChildJobFn(merge_hal, options, config, hal_ids, cores = 1, disk=sum(f.size for f in hal_ids) * 2, - memory=min(max(f.size for f in hal_ids) * 2, max_mem)) + memory=min(max(f.size for f in hal_ids) * 2, max_mem), + walltime=scaled_walltime(MERGE_HAL_SECS_PER_GB, + sum(f.size for f in hal_ids))) hal_id_dict = hal_merge_job.rv() out_dicts.append(hal_id_dict) # delete the chromosome hals - hal_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids) + hal_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_walltime()) # optional minigraph gfa merge if sv_gfa_ids: sv_gfa_merge_job = job.addChildJobFn(merge_sv_gfa, options, sv_gfa_ids, - disk=sum(f.size for f in sv_gfa_ids) * 3) + disk=sum(f.size for f in sv_gfa_ids) * 3, + walltime=scaled_walltime(MERGE_SV_GFA_SECS_PER_GB, + sum(f.size for f in sv_gfa_ids))) sv_gfa_id_dict = sv_gfa_merge_job.rv() out_dicts.append(sv_gfa_id_dict) # delete the chromosome gfas - sv_gfa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=sv_gfa_ids) + sv_gfa_merge_job.addFollowOnJobFn(clean_jobstore_files, file_ids=sv_gfa_ids, walltime=cactus_walltime()) if options.indexMemory: index_mem = options.indexMemory @@ -886,7 +1079,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, ref_vg_ids = bypass_full_ids or bypass_clip_ids or bypass_filter_ids ref_fasta_job = root_job.addFollowOnJobFn(extract_vg_fasta, options, ref_vg_ids, disk=sum(f.size for f in vg_ids) * 2, - memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4)) + memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4), + walltime=scaled_walltime(EXTRACT_VG_FASTA_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=1)) if not options.bypass: workflow_phases = [('full', full_vg_ids, join_job)] @@ -913,7 +1109,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for workflow_phase, phase_vg_ids, phase_root_job in workflow_phases: # make a gfa for each - gfa_root_job = Job() + gfa_root_job = Job(walltime=cactus_walltime()) phase_root_job.addFollowOn(gfa_root_job) gfa_ids = [] current_out_dict = None @@ -925,7 +1121,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vg_path, vg_id, input_vg_id in zip(options.vg, phase_vg_ids, vg_ids): gfa_job = gfa_root_job.addChildJobFn(vg_to_gfa, options, config, vg_path, vg_id, disk=input_vg_id.size * 10, - memory=min(max(2**31, input_vg_id.size * 16), max_mem)) + memory=min(max(2**31, input_vg_id.size * 16), max_mem), + walltime=scaled_walltime(VG_TO_GFA_SECS_PER_GB, + input_vg_id.size, io_multiple=5)) gfa_ids.append(gfa_job.rv()) gfa_merge_job = gfa_root_job.addFollowOnJobFn(make_vg_indexes, options, config, gfa_ids, @@ -933,7 +1131,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, do_gbz=do_gbz, cores=options.indexCores, disk=sum(f.size for f in vg_ids) * 6, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(MAKE_VG_INDEXES_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(gfa_merge_job.rv()) prev_job = gfa_merge_job current_out_dict = gfa_merge_job.rv() @@ -942,21 +1142,29 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, ref_fasta_job = gfa_merge_job.addFollowOnJobFn(extract_gbz_fasta, options, current_out_dict, tag=workflow_phase + '.', memory=index_mem, - disk=sum(f.size for f in vg_ids) * 2) + disk=sum(f.size for f in vg_ids) * 2, + # one vg paths -F pass per --vcfReference + walltime=scaled_walltime( + EXTRACT_GBZ_FASTA_SECS_PER_GB * max(1, len(options.vcfReference or [])), + sum(f.size for f in vg_ids), io_multiple=0.5)) # optional unchopped gfa if workflow_phase in options.unchopped_gfa: unchopped_gfa_ids = [] for vg_path, vg_id, input_vg_id in zip(options.vg, phase_vg_ids, vg_ids): unchopped_gfa_job = gfa_root_job.addChildJobFn(vg_to_gfa, options, config, vg_path, vg_id, unchopped=True, disk=input_vg_id.size * 10, - memory=min(max(2**31, input_vg_id.size * 16), max_mem)) + memory=min(max(2**31, input_vg_id.size * 16), max_mem), + walltime=scaled_walltime(VG_TO_GFA_UNCHOPPED_SECS_PER_GB, + input_vg_id.size, io_multiple=5)) unchopped_gfa_ids.append(unchopped_gfa_job.rv()) unchopped_gfa_merge_job = gfa_root_job.addFollowOnJobFn(make_vg_indexes, options, config, unchopped_gfa_ids, tag=workflow_phase + '.unchopped.', do_gbz=False, cores=1, - disk=sum(f.size for f in vg_ids) * 3) + disk=sum(f.size for f in vg_ids) * 3, + walltime=scaled_walltime(MAKE_VG_INDEXES_1CORE_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(unchopped_gfa_merge_job.rv()) @@ -966,7 +1174,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, tag=workflow_phase + '.', cores=max(options.indexCores, 4), disk = sum(f.size for f in vg_ids) * 10, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(MAKE_XG_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=1)) out_dicts.append(xg_job.rv()) # optional vcf @@ -974,7 +1185,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, for vcf_ref in options.vcfReference: vcf_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, workflow_phase, index_mem, vcf_ref, phase_vg_ids, - ref_fasta_job.rv() if ref_fasta_job else None) + ref_fasta_job.rv() if ref_fasta_job else None, walltime=cactus_walltime()) if ref_fasta_job: ref_fasta_job.addFollowOn(vcf_job) out_dicts.append(vcf_job.rv()) @@ -984,7 +1195,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, l_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, workflow_phase, index_mem, vcf_ref, phase_vg_ids, ref_fasta_job.rv() if ref_fasta_job else None, - decon_L=options.vcfL) + decon_L=options.vcfL, walltime=cactus_walltime()) if ref_fasta_job: ref_fasta_job.addFollowOn(l_job) out_dicts.append(l_job.rv()) @@ -1009,7 +1220,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # than a human one will exceed the cap and # get there through --doubleMem memory=cactus_clamp_memory( - min(index_mem * 3, 128 * 2**30))) + min(index_mem * 3, 128 * 2**30)), + walltime=scaled_walltime(MAKE_GIRAFFE_INDEXES_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.5)) out_dicts.append(giraffe_job.rv()) # optional haplo index @@ -1026,14 +1240,19 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # and 760Gi. scale up for the small end, where # it barely overflowed, and cap the big end memory=cactus_clamp_memory( - min(int(index_mem * 1.5), 128 * 2**30))) + min(int(index_mem * 1.5), 128 * 2**30)), + walltime=scaled_walltime(MAKE_HAPLO_INDEX_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.5)) out_dicts.append(haplo_job.rv()) # optional full-genome odgi if workflow_phase in options.odgi: odgi_job = gfa_root_job.addChildJobFn(odgi_squeeze, config, options.vg, og_chrom_ids[workflow_phase]['og'], tag=workflow_phase + '.', disk=sum(f.size for f in vg_ids) *4, - memory=index_mem, cores=options.indexCores) + memory=index_mem, cores=options.indexCores, + walltime=scaled_walltime(ODGI_SQUEEZE_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(odgi_job.rv()) # optional viz @@ -1047,7 +1266,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, viz_job = gfa_root_job.addChildJobFn(make_odgi_viz, config, options, vg_path, og_id, tag=workflow_phase, viz=do_viz, draw=do_draw, cores=options.indexCores, disk = input_vg_id.size * 10, - memory=min(max(og_min_size, input_vg_id.size * 32), max_mem)) + memory=min(max(og_min_size, input_vg_id.size * 32), max_mem), + walltime=scaled_walltime(ODGI_DRAW_SECS_PER_GB if do_draw + else ODGI_VIZ_SECS_PER_GB, + input_vg_id.size)) else: viz_job = None if do_viz: @@ -1063,12 +1285,17 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, snarl_stats_job = gfa_root_job.addChildJobFn(snarl_stats, options, config, vg_path, vg_id, disk=input_vg_id.size * 2, memory=cactus_clamp_memory(input_vg_id.size * 10), - cores=options.indexCores) + cores=options.indexCores, + walltime=scaled_walltime(SNARL_STATS_SECS_PER_GB, + input_vg_id.size)) snarl_stats_ids.append(snarl_stats_job.rv()) snarl_stats_merge_job = gfa_root_job.addFollowOnJobFn(merge_snarl_stats, options.vg, snarl_stats_ids, tag=workflow_phase + '.', - disk=sum(f.size for f in vg_ids) * 2) + disk=sum(f.size for f in vg_ids) * 2, + walltime=scaled_walltime(MERGE_CHROM_TSV_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.1)) out_dicts.append(snarl_stats_merge_job.rv()) # collect this phase's merged GFA; a single combined panacus report is built after the loop @@ -1085,7 +1312,11 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, panacus_job = Job.wrapJobFn(run_panacus, options, config, panacus_phase_dicts, cores=options.indexCores, disk=sum(f.size for f in vg_ids) * 6 * len(panacus_phase_dicts), - memory=index_mem) + memory=index_mem, + # every phase is a full decompress-and-scan of its own merged GFA + walltime=scaled_walltime(PANACUS_SECS_PER_GB * len(panacus_phase_dicts), + sum(f.size for f in vg_ids), + io_multiple=0.5 * len(panacus_phase_dicts))) for merge_job in panacus_merge_jobs: merge_job.addFollowOn(panacus_job) out_dicts.append(panacus_job.rv()) @@ -1117,7 +1348,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, gref_sample = gref_sample_name(vcf_ref) # augment each chromosome in parallel - gref_root_job = Job() + gref_root_job = Job(walltime=cactus_walltime()) gref_source_root_job.addFollowOn(gref_root_job) gref_vg_ids = [] @@ -1126,7 +1357,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, gref_job = gref_root_job.addChildJobFn(compute_gref_paths, config, options, vg_path, vg_id, vcf_ref, disk=input_vg_id.size * 20, memory=max(2**31, min(input_vg_id.size * 20, max_mem)), - cores=options.indexCores) + cores=options.indexCores, + walltime=scaled_walltime(COMPUTE_GREF_PATHS_SECS_PER_GB, + input_vg_id.size)) gref_vg_ids.append(gref_job.rv(0)) gref_segs_ids.append(gref_job.rv(1)) @@ -1138,7 +1371,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, gref_fasta_job = gref_root_job.addFollowOnJobFn(extract_vg_fasta, options, gref_vg_ids, vcf_ref=gref_sample, disk=sum(f.size for f in vg_ids) * 2, - memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4)) + memory=cactus_clamp_memory(sum(f.size for f in vg_ids) * 4), + walltime=scaled_walltime(EXTRACT_VG_FASTA_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=1)) gref_fasta_dict = gref_fasta_job.rv() gref_parent_job = gref_fasta_job @@ -1154,7 +1390,10 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # merge gref segments gref_segs_merge_job = gref_root_job.addFollowOnJobFn(merge_gref_segs, options.vg, gref_segs_ids, - disk=sum(f.size for f in vg_ids)) + disk=sum(f.size for f in vg_ids), + walltime=scaled_walltime(MERGE_CHROM_TSV_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.1)) out_dicts.append(gref_segs_merge_job.rv()) # All of the exclusion work hangs off one barrier, and that barrier is a descendant of @@ -1165,7 +1404,7 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, exclusion_ids = None if do_exclusions and workflow_phases: deepest_phase, deepest_vg_ids, deepest_root_job = workflow_phases[-1] - excl_root_job = Job() + excl_root_job = Job(walltime=cactus_walltime()) deepest_root_job.addFollowOn(excl_root_job) exclusion_coverage = {} @@ -1177,7 +1416,8 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, cov_job = excl_root_job.addChildJobFn( path_coverage_job, config, vg_path, phase_vg_id, chrom_name, workflow_phase, disk=input_vg_id.size * 3, - memory=cactus_clamp_memory(min(max(2**31, input_vg_id.size * 6), max_mem))) + memory=cactus_clamp_memory(min(max(2**31, input_vg_id.size * 6), max_mem)), + walltime=scaled_walltime(PATH_COVERAGE_SECS_PER_GB, input_vg_id.size)) phase_coverage.append((cov_job.rv(0), cov_job.rv(1), cov_job.rv(2), cov_job.rv(3))) exclusion_coverage[workflow_phase] = phase_coverage @@ -1202,7 +1442,8 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, # with ~30% headroom while 16Gi covers the walk to a 324Mbp reference contig. # a longer contig than that still gets there through --doubleMem memory=cactus_clamp_memory(min(16 * 2**30 + int(input_vg_id.size * 1.75), - max_mem))) + max_mem)), + walltime=scaled_walltime(REF_GAPS_SECS_PER_GB, input_vg_id.size)) gap_ids.append(gap_job.rv()) exclusion_refgap_ids[ref_event] = gap_ids @@ -1214,7 +1455,9 @@ def graphmap_join_workflow(job, options, config, vg_ids, hal_ids, sv_gfa_ids, compute_exclusions_job, config, options, exclusion_coverage, contig_sizes_id, split_log_id, exclusion_refgap_ids, chrom_names, disk=sum(f.size for f in vg_ids) * 2, - memory=cactus_clamp_memory(min(max(2**31, max(f.size for f in vg_ids) * 16), max_mem))) + memory=cactus_clamp_memory(min(max(2**31, max(f.size for f in vg_ids) * 16), max_mem)), + walltime=scaled_walltime(COMPUTE_EXCLUSIONS_SECS_PER_GB, sum(f.size for f in vg_ids), + io_multiple=0.25)) exclusion_ids = exclusion_job.rv() return (output_full_vg_ids, clip_vg_ids, clipped_stats, filter_vg_ids, out_dicts, og_chrom_ids, @@ -1694,7 +1937,7 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r ones. it never makes a wave VCF: -L has already merged away the alleles vcfwave would realign """ - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) if vcftag is None: vcftag = vcf_ref + '.' + workflow_phase if vcf_ref != options.reference[0] else workflow_phase @@ -1709,7 +1952,8 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r tag=os.path.splitext(os.path.basename(vg_path))[0] + '.' + vcftag + '.', cores=options.indexCores, disk = vg_id.size * 6, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(DECONSTRUCT_SECS_PER_GB, vg_id.size)) raw_vcf_id, raw_tbi_id = deconstruct_job.rv(0), deconstruct_job.rv(1) raw_vcf_tbi_ids.append((raw_vcf_id, raw_tbi_id)) @@ -1721,7 +1965,9 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r ref_fasta_dict, tag=os.path.splitext(os.path.basename(vg_path))[0] + '.' + vcftag + '.', disk = vg_id.size * 6, - memory=cactus_clamp_memory(vg_id.size * 2)) + memory=cactus_clamp_memory(vg_id.size * 2), + walltime=scaled_walltime(VCFBUB_SECS_PER_GB, vg_id.size, + io_multiple=0.5)) bub_vcf_id, bub_tbi_id = vcfbub_job.rv(0), vcfbub_job.rv(1) bub_vcf_tbi_ids.append((bub_vcf_id, bub_tbi_id)) @@ -1733,7 +1979,9 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r tag=os.path.splitext(os.path.basename(vg_path))[0] + '.' + vcftag + '.', cores=options.vcfwaveCores, disk=vg_id.size * 6, - memory=cactus_clamp_memory(options.vcfwaveMemory)) + memory=cactus_clamp_memory(options.vcfwaveMemory), + walltime=scaled_walltime(CHUNKED_VCFWAVE_SECS_PER_GB, + vg_id.size, io_multiple=0.5)) wave_vcf_id, wave_tbi_id = vcfwave_job.rv(0), vcfwave_job.rv(1) wave_vcf_tbi_ids.append((wave_vcf_id, wave_tbi_id)) @@ -1747,7 +1995,10 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r fix_ploidies=True, cores = cat_cores, disk = sum(f.size for f in vg_ids) * 26, - memory = cactus_clamp_memory(sum(f.size for f in vg_ids))) + memory = cactus_clamp_memory(sum(f.size for f in vg_ids)), + walltime=scaled_walltime(VCF_CAT_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.125)) out_dict = {'{}.raw.vcf.gz'.format(vcftag) : merge_vcf_job.rv(0), '{}.raw.vcf.gz.tbi'.format(vcftag) : merge_vcf_job.rv(1) } if bub_vcf_tbi_ids: @@ -1755,7 +2006,10 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r fix_ploidies=True, cores = cat_cores, disk = sum(f.size for f in vg_ids) * 26, - memory = cactus_clamp_memory(sum(f.size for f in vg_ids))) + memory = cactus_clamp_memory(sum(f.size for f in vg_ids)), + walltime=scaled_walltime(VCF_CAT_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.125)) out_dict['{}.vcf.gz'.format(vcftag)] = merge_bub_job.rv(0) out_dict['{}.vcf.gz.tbi'.format(vcftag)] = merge_bub_job.rv(1) if wave_vcf_tbi_ids: @@ -1763,7 +2017,10 @@ def make_vcf(job, config, options, workflow_phase, index_mem, vcf_ref, vg_ids, r fix_ploidies=True, cores = cat_cores, disk = sum(f.size for f in vg_ids) * 26, - memory = cactus_clamp_memory(sum(f.size for f in vg_ids))) + memory = cactus_clamp_memory(sum(f.size for f in vg_ids)), + walltime=scaled_walltime(VCF_CAT_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.125)) out_dict['{}.wave.vcf.gz'.format(vcftag)] = merge_wave_job.rv(0) out_dict['{}.wave.vcf.gz.tbi'.format(vcftag)] = merge_wave_job.rv(1) @@ -1945,7 +2202,9 @@ def vcfbub(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_allele, fasta if fasta_ref_dict is not None and \ getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap_join"), "bcftoolsNorm", typeFn=bool, default=False): norm_job = job.addChildJobFn(vcfnorm, config, vcf_ref, bub_vcf_id, vcfbub_path, bub_tbi_id, fasta_ref_dict, - disk=bub_vcf_id.size * 6) + disk=bub_vcf_id.size * 6, + walltime=scaled_walltime(VCFNORM_BUB_SECS_PER_GB, bub_vcf_id.size, + io_multiple=6)) return norm_job.rv() else: return bub_vcf_id, bub_tbi_id @@ -2220,7 +2479,7 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle chunk_paths = [vcfbub_path] # distribute on the chunks - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) chunk_vcf_tbi_ids = [] for chunk_path in chunk_paths: @@ -2233,7 +2492,10 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle wave_mem = min(job.memory, max(6 * 2**30, job.cores * 960 * 2**20, chunk_id.size * 800)) vcfwave_job = root_job.addChildJobFn(vcfwave, config, chunk_path, chunk_id, disk=chunk_id.size * 10, cores=job.cores, - memory=wave_mem) + memory=wave_mem, + walltime=cactus_walltime( + VCFWAVE_SECS_PER_MB * chunk_id.size / 1e6 / max(1, job.cores or 1), + io_bytes=chunk_id.size * 4)) chunk_vcf_tbi_ids.append(vcfwave_job.rv()) # combine the chunks @@ -2242,7 +2504,9 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle vcfwave_cat_job = root_job.addFollowOnJobFn(vcf_cat, chunk_vcf_tbi_ids, tag, sort=True, fix_ploidies=False, disk=vcf_id.size * 10, - memory=cactus_clamp_memory(vcf_id.size*5)) + memory=cactus_clamp_memory(vcf_id.size*5), + walltime=scaled_walltime(VCF_CAT_CHUNK_SECS_PER_GB, vcf_id.size, + io_multiple=4)) # normalize the output if fasta_ref_dict is not None and \ @@ -2252,7 +2516,9 @@ def chunked_vcfwave(job, config, out_name, vcf_ref, vcf_id, tbi_id, max_ref_alle norm_job = vcfwave_cat_job.addFollowOnJobFn(vcfnorm, config, vcf_ref, vcfwave_cat_job.rv(0), vcfwave_path, vcfwave_cat_job.rv(1), fasta_ref_dict, disk=vcf_id.size*12, - memory=cactus_clamp_memory(vcf_id.size*5)) + memory=cactus_clamp_memory(vcf_id.size*5), + walltime=scaled_walltime(VCFNORM_WAVE_SECS_PER_GB, vcf_id.size, + io_multiple=6)) return norm_job.rv() else: return vcfwave_cat_job.rv() @@ -2518,13 +2784,15 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, out_dicts = [] # per-chromosome GFA conversion - gfa_root_job = Job() + gfa_root_job = Job(walltime=cactus_walltime()) parent_job.addFollowOn(gfa_root_job) gfa_ids = [] for vg_path, vg_id, input_vg_id in zip(options.vg, phase_vg_ids, vg_ids): gfa_job = gfa_root_job.addChildJobFn(vg_to_gfa, options, config, vg_path, vg_id, disk=input_vg_id.size * 10, - memory=min(max(2**31, input_vg_id.size * 16), max_mem)) + memory=min(max(2**31, input_vg_id.size * 16), max_mem), + walltime=scaled_walltime(VG_TO_GFA_SECS_PER_GB, + input_vg_id.size, io_multiple=5)) gfa_ids.append(gfa_job.rv()) # GBZ from merged GFA. the gref graph reuses the base graph's snarls (identical topology), @@ -2534,7 +2802,9 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, ref_samples=(options.reference + [vcf_ref]) if is_gref else None, cores=options.indexCores, disk=sum(f.size for f in vg_ids) * 6, - memory=index_mem) + memory=index_mem, + walltime=scaled_walltime(MAKE_VG_INDEXES_SECS_PER_GB, + sum(f.size for f in vg_ids))) out_dicts.append(gbz_job.rv()) index_dict = gbz_job.rv() @@ -2543,7 +2813,7 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, if vcf_ref and options.vcf: vcf_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, tag.rstrip('.'), index_mem, vcf_ref, phase_vg_ids, - ref_fasta_dict, vcftag=vcftag) + ref_fasta_dict, vcftag=vcftag, walltime=cactus_walltime()) out_dicts.append(vcf_job.rv()) if decon_L is not None: @@ -2552,7 +2822,7 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, l_job = gfa_root_job.addFollowOnJobFn(make_vcf, config, options, tag.rstrip('.'), index_mem, vcf_ref, phase_vg_ids, ref_fasta_dict, vcftag=vcftag, - decon_L=decon_L) + decon_L=decon_L, walltime=cactus_walltime()) out_dicts.append(l_job.rv()) # optional haplo index @@ -2563,7 +2833,10 @@ def build_vg_indexes_and_vcf(parent_job, options, config, phase_vg_ids, vg_ids, disk=sum(f.size for f in vg_ids) * 16, # see the other make_haplo_index call site memory=cactus_clamp_memory( - min(int(index_mem * 1.5), 128 * 2**30))) + min(int(index_mem * 1.5), 128 * 2**30)), + walltime=scaled_walltime(MAKE_HAPLO_INDEX_SECS_PER_GB, + sum(f.size for f in vg_ids), + io_multiple=0.5)) out_dicts.append(haplo_job.rv()) return out_dicts diff --git a/src/cactus/refmap/cactus_graphmap_split.py b/src/cactus/refmap/cactus_graphmap_split.py index b65a1c392..5fca146fb 100644 --- a/src/cactus/refmap/cactus_graphmap_split.py +++ b/src/cactus/refmap/cactus_graphmap_split.py @@ -15,12 +15,12 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.multiCactusTree import MultiCactusTree -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import unzip_gz, write_s3 @@ -31,7 +31,9 @@ from cactus.preprocessor.fileMasking import get_mask_bed_from_fasta from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.refmap.cactus_graphmap import filter_paf, apply_mgsplit_filter_overrides +from cactus.refmap.cactus_graphmap import FILTER_PAF_SECS_PER_GB from cactus.refmap.cactus_minigraph import check_sample_names, minigraph_gfa_from_pansn, minigraph_gfa_to_pansn +from cactus.refmap.cactus_minigraph import GFA_RENAME_SECS_PER_GB, RAW_BYTES_PER_GZ_BYTE from toil.job import Job from toil.common import Toil from toil.statsAndLogging import logger @@ -44,6 +46,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (gzipped fastas supported)") parser.add_argument("minigraphGFA", help = "Minigraph-compatible reference graph in GFA format (can be gzipped)") @@ -177,15 +180,21 @@ def cactus_graphmap_split(options): # run the workflow wf_output = toil.start(Job.wrapJobFn(graphmap_split_workflow, options, config, input_seq_id_map, input_name_map, gfa_id, options.minigraphGFA, - paf_id, options.graphmapPAF)) + paf_id, options.graphmapPAF, walltime=cactus_walltime())) #export the split data export_split_data(toil, wf_output[0], wf_output[1], wf_output[2], wf_output[3], options.outDir, config) +# Seconds per GB of GFA-plus-PAF for rgfa-split: 1220s to split the 8.3 GiB GFA and 34.4 GiB PAF +# of the HPRC v2.0 whole panel, ie ~27 s/GB, with the per-chromosome sed renames that follow adding +# ~130s more. This is the heaviest job in cactus-graphmap-split, and separate_ref_contigs runs the +# same binary on the same kind of input. +RGFA_SPLIT_SECS_PER_GB = 32 + def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_id, gfa_path, paf_id, paf_path, sanitize=True, pansn_gfa_input=True): - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # can be a list coming in from cactus-pangenome, but we only need first item @@ -205,15 +214,20 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ # fix up the headers if sanitize: - sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = root_job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() else: - sanitize_job = Job() + sanitize_job = Job(walltime=cactus_walltime()) root_job.addChild(sanitize_job) # auto-set --refContigs if not options.refContigs: - refcontig_job = sanitize_job.addFollowOnJobFn(detect_ref_contigs, config, options, seq_id_map) + # samtools faidx of a whole reference is 21s measured; the rest is downloading the fasta, + # whose size is not in scope here (seq_id_map is a promise), so it goes in the constant -- + # 600s is a ~3 GB reference downloaded on a badly contended filesystem, and a bigger one + # still fits inside the safety factor + refcontig_job = sanitize_job.addFollowOnJobFn(detect_ref_contigs, config, options, seq_id_map, + walltime=cactus_walltime(600)) ref_contigs = refcontig_job.rv() options.otherContig = getOptionalAttrib(findRequiredNode(config.xmlRoot, "graphmap_split"), "otherContigName", typeFn=str, default="chrOther") sanitize_job = refcontig_job @@ -226,46 +240,58 @@ def graphmap_split_workflow(job, options, config, seq_id_map, seq_name_map, gfa_ # the renaming pass decompresses the GFA before bgzipping it back up, so it needs room for # the raw copy (reckoned at 10x, as below) on top of the compressed input and output rename_gfa_job = root_job.addChildJobFn(minigraph_gfa_from_pansn, genome_names, gfa_path, gfa_id, - disk=gfa_size*12) - new_root_job = Job() + disk=gfa_size*12, + walltime=cactus_walltime(GFA_RENAME_SECS_PER_GB * gfa_size / 1e9, + io_bytes=2*gfa_size)) + new_root_job = Job(walltime=cactus_walltime()) root_job.addFollowOn(new_root_job) root_job = new_root_job gfa_id = rename_gfa_job.rv(0) # use file extension to sniff out compressed input if gfa_path.endswith(".gz"): - gfa_id = root_job.addChildJobFn(unzip_gz, gfa_path, gfa_id, delete_original=False, disk=gfa_size * 10).rv() + # gunzip is fast (27s for the 0.83 GiB compressed whole-panel GFA); the cost is writing the + # ~10x bigger raw file back to the jobstore, which is what io_bytes is counting + gfa_id = root_job.addChildJobFn(unzip_gz, gfa_path, gfa_id, delete_original=False, disk=gfa_size * 10, + walltime=cactus_walltime(60, io_bytes=(1 + RAW_BYTES_PER_GZ_BYTE) * gfa_size)).rv() gfa_size *= 10 if paf_path.endswith(".gz"): - paf_id = root_job.addChildJobFn(unzip_gz, paf_path, paf_id, delete_original=False, disk=paf_id.size * 10).rv() + # the same, but on a PAF that reaches 34 GiB raw at panel scale + paf_id = root_job.addChildJobFn(unzip_gz, paf_path, paf_id, delete_original=False, disk=paf_id.size * 10, + walltime=cactus_walltime(120, io_bytes=(1 + RAW_BYTES_PER_GZ_BYTE) * paf_size)).rv() paf_size *= 10 # do some basic paf filtering paf_filter_mem = max(paf_id.size * 10, 2**32) paf_filter_job = root_job.addFollowOnJobFn(filter_paf, paf_id, config, reference=options.reference, - disk = paf_id.size * 10, memory=cactus_clamp_memory(paf_filter_mem)) + disk = paf_id.size * 10, memory=cactus_clamp_memory(paf_filter_mem), + walltime=cactus_walltime(60 + FILTER_PAF_SECS_PER_GB * paf_size / 1e9, + io_bytes=2*paf_size)) paf_id = paf_filter_job.rv() root_job = paf_filter_job mask_bed_id = None if options.maskFilter: - mask_bed_id = sanitize_job.addFollowOnJobFn(get_mask_bed, seq_id_map, options.maskFilter).rv() + mask_bed_id = sanitize_job.addFollowOnJobFn(get_mask_bed, seq_id_map, options.maskFilter, walltime=cactus_walltime()).rv() # use rgfa-split to split the gfa and paf up by contig split_gfa_job = root_job.addFollowOnJobFn(split_gfa, config, gfa_id, [paf_id], ref_contigs, options.otherContig, options.reference, mask_bed_id, disk=(gfa_size + paf_size) * 5, - memory=cactus_clamp_memory((gfa_size + paf_size) * 3)) + memory=cactus_clamp_memory((gfa_size + paf_size) * 3), + walltime=cactus_walltime(120 + RGFA_SPLIT_SECS_PER_GB * (gfa_size + paf_size) / 1e9, + io_bytes=2 * (gfa_size + paf_size))) # use the output of the above splitting to do the fasta splitting - split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0)) + split_fas_job = split_gfa_job.addFollowOnJobFn(split_fas, seq_id_map, seq_name_map, split_gfa_job.rv(0), walltime=cactus_walltime()) # gather everythign up into a table - gather_fas_job = split_fas_job.addFollowOnJobFn(gather_fas, split_gfa_job.rv(0), split_fas_job.rv(0), split_fas_job.rv(1)) + gather_fas_job = split_fas_job.addFollowOnJobFn(gather_fas, split_gfa_job.rv(0), split_fas_job.rv(0), split_fas_job.rv(1), walltime=cactus_walltime()) # lump "other" contigs together into one file (to make fewer align jobs downstream) bin_other_job = gather_fas_job.addFollowOnJobFn(bin_other_contigs, config, ref_contigs, options.otherContig, gather_fas_job.rv(0), - disk=(gfa_size + paf_size) * 2) + disk=(gfa_size + paf_size) * 2, + walltime=cactus_walltime(0, io_bytes=(gfa_size + paf_size) * 2)) # return all the files, as well as the 2 split logs return (seq_name_map, bin_other_job.rv(), split_gfa_job.rv(1), gather_fas_job.rv(1)) @@ -333,8 +359,14 @@ def get_mask_bed(job, seq_id_map, min_length): for event in seq_id_map.keys(): fa_id = seq_id_map[event] fa_path = '{}.fa'.format(event) - beds.append(job.addChildJobFn(get_mask_bed_from_fasta, event, fa_id, fa_path, min_length, disk=fa_id.size * 5).rv()) - return job.addFollowOnJobFn(cat_beds, beds).rv() + # cactus_softmask2hardmask scanning a vertebrate genome is p99 51s / max 89s over the 625 + # runs of the VGP 577-way; the rest is the fasta in and the BED out + beds.append(job.addChildJobFn(get_mask_bed_from_fasta, event, fa_id, fa_path, min_length, disk=fa_id.size * 5, + walltime=cactus_walltime(120, io_bytes=2*fa_id.size)).rv()) + # one BED per genome -- order tens of GiB across a whole panel -- concatenated and written back. + # bed_ids are promises and no fasta size is in scope either, so the whole cost has to be a + # constant: 1200s is ~30 GiB of round-trip staging at a pessimistic 30 MB/s + return job.addFollowOnJobFn(cat_beds, beds, walltime=cactus_walltime(1200)).rv() def cat_beds(job, bed_ids): in_beds = [job.fileStore.readGlobalFile(bed_id) for bed_id in bed_ids] @@ -573,7 +605,9 @@ def separate_ref_contigs_batch(job, config, graphmap_input_dict, graphmap_batch_ genome_names=sorted(seq_id_map.keys()), cores=mg_cores, disk=disk_size * 5, - memory=cactus_clamp_memory(tot_size * 3)).rv() + memory=cactus_clamp_memory(tot_size * 3), + walltime=cactus_walltime(120 + RGFA_SPLIT_SECS_PER_GB * tot_size / 1e9, + io_bytes=tot_size)).rv() return output_dict def separate_ref_contigs(job, config, chrom, gfa_id, gm_result, reference_event, @@ -704,7 +738,7 @@ def line_count(path): out_fa_id = job.fileStore.writeGlobalFile(out_fa_path) # the whole-genome one it replaces is nobody's input from here on. deleted from a # follow-on rather than inline so a retry of this job doesn't trip over its own delete - job.addFollowOnJobFn(clean_jobstore_files, file_ids=[gm_result[1]]) + job.addFollowOnJobFn(clean_jobstore_files, file_ids=[gm_result[1]], walltime=cactus_walltime()) # and the published per-chromosome graph, which graphmap-join merges pansn_gfa_path = os.path.join(work_dir, 'separated.sv.gfa.gz') @@ -714,13 +748,28 @@ def line_count(path): return (job.fileStore.writeGlobalFile(separated_paf_path), out_fa_id) + tuple(gm_result[2:]) + \ (job.fileStore.writeGlobalFile(log_path),) + ((pansn_gfa_id,) if whole_genome_ref else ()) +# Seconds per reference contig for split_fa_into_contigs. The contig count is the driver rather +# than the fasta size because each contig costs its own faidx and bgzip, and defaults to 128 -- five times the human case. +# +# Re-measured by grouping every timed command in the two HPRC v2.1 runs by the job work directory +# it ran in: 459 jobs per run, ~50 commands each, summing to a p99 of 646 s (sep16) and 604 s +# (sep17) against a worst of 654 s, over a 196-contig GRCh38 split. That is 3 s/contig at the +# p99, which is the convention WALLTIME_FACTOR's comment sets for these constants. The earlier +# 13 s/contig came from grouping by a 23-26 contig split, where the per-job fixed cost is spread +# over a fifth as many contigs and so lands in the slope. +# +# The difference is worth the re-fit because of how many of these there are: at 15 all 459 asked +# 2.1 h each, and at 3 they ask 37 min, which is the side of the one-hour partition they belong on. +SPLIT_FA_SECS_PER_REF_CONTIG = 3 + def split_fas(job, seq_id_map, seq_name_map, split_id_map): """ Use samtools to split a bunch of fasta files into reference contigs, using the output of rgfa-split as a guide""" if (seq_id_map, split_id_map) == (None, None): return None - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) # map event name to dict of contgs. ex fa_contigs["CHM13"]["chr13"] = file_id @@ -735,7 +784,9 @@ def split_fas(job, seq_id_map, seq_name_map, split_id_map): if fa_id.size: split_job = root_job.addChildJobFn(split_fa_into_contigs, event, fa_id, fa_path, split_id_map, strip_prefix=False, - disk=fa_id.size * 3) + disk=fa_id.size * 3, + walltime=cactus_walltime(60 + SPLIT_FA_SECS_PER_REF_CONTIG * len(split_id_map), + io_bytes=2*fa_id.size)) fa_contigs[event] = split_job.rv(0) fa_contig_sizes[event] = split_job.rv(1) diff --git a/src/cactus/refmap/cactus_minigraph.py b/src/cactus/refmap/cactus_minigraph.py index e98ee650e..031ccfdc0 100644 --- a/src/cactus/refmap/cactus_minigraph.py +++ b/src/cactus/refmap/cactus_minigraph.py @@ -17,13 +17,13 @@ import gzip from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.refmap.pangenome_exclusions import event_to_pansn_prefix from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles, write_s3 from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import clean_jobstore_files @@ -32,6 +32,7 @@ from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers from cactus.paf.last_scoring import last_train from toil.job import Job +from toil.job import PromisedRequirement from toil.common import Toil from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options @@ -39,11 +40,13 @@ from toil.lib.conversions import bytes2human from cactus.shared.common import cactus_cpu_count from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import GZIP_COMPRESS_BYTES_PER_SEC from cactus.progressive.multiCactusTree import MultiCactusTree from sonLib.bioio import getTempDirectory, getTempFile def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (or chromfile with --batch)") parser.add_argument("outputGFA", help = "Output Minigraph GFA (or directory in --batch mode)") @@ -141,7 +144,7 @@ def main(): # output_dict: chrom-> (gfa_id, pansn_gfa_id, uncollapsed_pansn_gfa_id, collapse_report_id, train_id) output_dict = toil.start(Job.wrapJobFn(minigraph_construct_batch_workflow, options, config_node, input_dict, options.outputGFA, - in_gfa_id=in_gfa_id)) + in_gfa_id=in_gfa_id, walltime=cactus_walltime())) export_minigraph_construct_output(options, input_seqfiles, output_dict, toil) @@ -316,7 +319,8 @@ def minigraph_construct_batch_workflow(job, options, config_node, input_dict, gf else: gfa_path = options.outputGFA mgwf_job = job.addChildJobFn(minigraph_construct_workflow, options, config_node, seq_id_map, seq_order, gfa_path, sanitize, - construct_seq_id_map=construct_seq_id_map, in_gfa_id=in_gfa_id) + construct_seq_id_map=construct_seq_id_map, in_gfa_id=in_gfa_id, + walltime=cactus_walltime()) output_dict[chrom] = mgwf_job.rv() return output_dict @@ -341,10 +345,13 @@ def minigraph_construct_workflow(job, options, config_node, seq_id_map, seq_orde # the renaming pass decompresses the GFA before bgzipping it back up, so it needs room for # the raw copy (reckoned at 10x, as elsewhere) on top of the compressed input and output rename_job = job.addChildJobFn(minigraph_gfa_from_pansn, set(seq_id_map.keys()), options.inGFA, in_gfa_id, - disk=in_gfa_id.size*12) + disk=in_gfa_id.size*12, + walltime=cactus_walltime(GFA_RENAME_SECS_PER_GB * in_gfa_id.size / 1e9, + io_bytes=RAW_BYTES_PER_GZ_BYTE * in_gfa_id.size)) run_job = rename_job.addFollowOnJobFn(minigraph_construct_run, options, config_node, seq_id_map, seq_order, gfa_path, sanitize, construct_seq_id_map, - rename_job.rv(0), rename_job.rv(1), in_gfa_id) + rename_job.rv(0), rename_job.rv(1), in_gfa_id, + walltime=cactus_walltime()) # all five slots: minigraph_construct_run returns # (gfa, pansn_gfa, uncollapsed_pansn_gfa, collapse_report, train). Truncating here to three # left cactus_pangenome's rv(3)/rv(4) reading off the end of the tuple, which Toil reports as @@ -386,9 +393,15 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf # reads the output name to decide whether to unzip, so it is re-emitted to match RealtimeLogger.info('Resuming from the {} genomes in {}: nothing left to construct'.format( len(seed_events), options.inGFA)) + # one bgzip or gunzip of each of the two seed graphs, so it is the compression rate + # over a raw GFA ~10x the compressed input + seed_gfa_size = seed_gfa_id.size if hasattr(seed_gfa_id, 'size') else 0 match_job = job.addChildJobFn(match_gfa_compression, seed_gfa_id, seed_pansn_gfa_id, options.inGFA, gfa_path, - disk=12 * (seed_gfa_id.size if hasattr(seed_gfa_id, 'size') else 0)) + disk=12 * seed_gfa_size, + walltime=cactus_walltime( + 2 * RAW_BYTES_PER_GZ_BYTE * seed_gfa_size / GZIP_COMPRESS_BYTES_PER_SEC, + io_bytes=4 * seed_gfa_size)) # last_train reads fastas, not the graph, so resuming is no reason to skip it: without # this --lastTrain would quietly fall back to the default scoring matrix train_id = None @@ -396,7 +409,9 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf train_job = job.addChildJobFn(last_train, config_node, train_seq_order, train_seq_id_map, ref_name=options.reference[0], cores=options.mgCores, disk=8*ref_size, - memory=cactus_clamp_memory(max(8*ref_size, 12*10**9))) + memory=cactus_clamp_memory(max(8*ref_size, 12*10**9)), + walltime=cactus_walltime(LAST_TRAIN_SECS + ref_size / 1e6, + io_bytes=3 * ref_size)) train_id = train_job.rv() # same five slots as the constructing path below. nothing was constructed, so there # is no pre-collapse graph and no report: the graph handed back is the seed as it was. @@ -414,17 +429,18 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf if construct_seq_id_map: construct_seq_id_map = {seq: construct_seq_id_map[seq] for seq in refonly_seq_order} if sanitize: - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) sanitized_seq_id_map = sanitize_job.rv() else: sanitized_seq_id_map = seq_id_map - sanitize_job = Job() + sanitize_job = Job(walltime=cactus_walltime()) job.addChild(sanitize_job) xml_node = findRequiredNode(config_node, "graphmap") sort_type = getOptionalAttrib(xml_node, "minigraphSortInput", str, default=None) if sort_type == "mash" and len(seq_id_map) > 2: sort_job = sanitize_job.addFollowOnJobFn(sort_minigraph_input_with_mash, options, config_node, sanitized_seq_id_map, seq_order, - ref_name=options.reference[0] if seed_events is not None else None) + ref_name=options.reference[0] if seed_events is not None else None, + walltime=cactus_walltime()) seq_order = sort_job.rv() prev_job = sort_job else: @@ -433,7 +449,8 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf construct_seq_id_map if construct_seq_id_map else sanitized_seq_id_map, seq_order, gfa_path, whole_genome_ref=bool(construct_seq_id_map), - seed_gfa_id=seed_gfa_id, graph_names=graph_names) + seed_gfa_id=seed_gfa_id, graph_names=graph_names, + walltime=cactus_walltime()) # optionally rewrite inverted alleles stored as novel sequence into inversion edges. when this # runs, the collapsed graph REPLACES the constructed one in both namings: it is what graphmap, @@ -455,8 +472,28 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf collapse_job = minigraph_job.addFollowOnJobFn(collapse_inversions, options, config_node, minigraph_job.rv(1), gfa_path, cores=options.mgCores, - disk=8*ref_size, - memory=cactus_clamp_memory(8*ref_size)) + # sized from the graph it collapses, not the + # reference: rgfa-collapse's cost is almost all + # fixed. Over 24 HPRC chromosomes it used + # 5.9-6.0 GiB of memory and up to 6.57 GiB of + # disk while the gfa spanned 6000x (0.03 MB to + # 190 MB). Both are flat in graph size, so the + # floors are what is measured -- roughly 2x the + # observed peak each -- and the per-byte terms + # only bite above them, as insurance for graphs + # larger than any seen. The walltime follows + # the same shape, and off the same promise. + disk=PromisedRequirement( + lambda gfa: max(24 * gfa.size, 16 * 2**30), + minigraph_job.rv(1)), + memory=PromisedRequirement( + lambda gfa: cactus_clamp_memory(max(64 * gfa.size, 12 * 2**30)), + minigraph_job.rv(1)), + walltime=PromisedRequirement( + lambda gfa: cactus_walltime( + collapse_inversions_walltime(gfa.size, options.mgCores), + io_bytes=4 * gfa.size), + minigraph_job.rv(1))) # graph_names, the same set the forward rename uses, and for the same reason its comment # gives: it has to resolve every SN tag in the finished graph, which on the --inGFA extend # path is more genomes than minigraph is given. It is captured above before seq_id_map is @@ -465,7 +502,9 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf rename_job = collapse_job.addFollowOnJobFn(minigraph_gfa_from_pansn, graph_names, gfa_path, collapse_job.rv(0), disk=12*ref_size, - memory=cactus_clamp_memory(4*ref_size)) + memory=cactus_clamp_memory(4*ref_size), + walltime=cactus_walltime(GFA_RENAME_SECS_PER_GB * ref_size / 1e9, + io_bytes=RAW_BYTES_PER_GZ_BYTE * ref_size)) uncollapsed_pansn_gfa_id = minigraph_job.rv(1) collapse_report_id = collapse_job.rv(1) # rv(0), not rv(): master's minigraph_gfa_from_pansn returns @@ -479,17 +518,68 @@ def minigraph_construct_run(job, options, config_node, seq_id_map, seq_order, gf if options.lastTrain and len(seq_id_map) > 1: # note: somehow last training memory overruns don't seem to be detected by slurm so we # give 12G at least whenever possible, as --doubleMem won't help... + # lastdb dominates the runtime and is erratic: over the 24 per-chromosome runs of the + # HPRC v2.1 pangenome (2.0e8-byte reference, 8 cores) it took 55 s at the p50 but + # 3934 s at the worst, while the 16x bigger whole-genome reference of HPRC v2.0 + # (32 cores) took 2821 s. last-train itself adds 409-680 s on top. So the estimate is + # mostly a flat allowance for that tail, with a small linear term so that small inputs + # aren't over-provisioned. The I/O is the reference plus the training partner, which + # last_train() only picks inside the job but constrains to at least half the reference. last_train_job = prev_job.addFollowOnJobFn(last_train, config_node, seq_order, sanitized_seq_id_map, ref_name=options.reference[0] if seed_events is not None else None, cores=options.mgCores, disk=8*ref_size, - memory=cactus_clamp_memory(max(8*ref_size, 12*10**9))) + memory=cactus_clamp_memory(max(8*ref_size, 12*10**9)), + walltime=cactus_walltime(LAST_TRAIN_SECS + ref_size / 1e6, + io_bytes=3 * ref_size)) train_id = last_train_job.rv() # (cactus-named graph, PanSN graph, the PanSN graph before collapsing or None, per-call report # or None, LAST scoring model or None). slots 0 and 1 are the graph the pipeline uses. return gfa_ids[0], gfa_ids[1], uncollapsed_pansn_gfa_id, collapse_report_id, train_id +# rgfa-collapse's runtime is set by its slowest single minimap2 alignment, not by the total work: +# the note in collapse_inversions records individual calls running over three hours at -t 2 on +# CHM13 chr9, and a wide -j cannot spread one alignment across chunks. So the estimate is keyed +# on the threads that one alignment gets rather than on the core count. 3.5 h x 2 threads is the +# measured tail; minimap2 scales well within an alignment, so more threads buy it back. +COLLAPSE_TAIL_THREAD_SECS = 25200 + +# The bulk on top of that tail: the vg snarl decomposition and the many small alignments, per GB +# of the graph being collapsed. Small, because rgfa-collapse's cost turns out to be almost all +# fixed -- over 24 HPRC chromosomes its memory and disk barely moved while the graph spanned +# 6000x -- so this is insurance for graphs larger than any of those rather than the main term. + +def collapse_minimap2_threads(cores): + """ threads rgfa-collapse gives each minimap2 -- the same split collapse_inversions makes, kept + here so the walltime estimate and the job itself cannot drift apart """ + cores = max(1, int(cores or 1)) + jobs = min(8, max(1, cores // 8)) + return max(1, cores // jobs) + +COLLAPSE_INVERSIONS_SECS_PER_GB = 2000 + +# Fixed seconds for a last_train job. last-train's cost is set by how divergent the pair it picks +# is, not by how big the reference is, and the two are close to anti-correlated here: over the 27 +# per-chromosome buckets of two HPRC v2.1 runs the median job took 130 s and the largest reference +# (chr1) finished in 188 s, while the *second smallest* -- the unplaced/unlocalized bucket -- ran +# 9807 s in one run and 6931 s in the other. last_train() picks the furthest genome in the mash +# order that clears its size floor, and that bucket's mash distances span the whole range, so its +# pick sits at 0.106 against 0.0007-0.0037 for a real chromosome. +# +# ref_size is therefore the wrong regressor and this flat term is what carries the pathology. The +# old 3000 asked 2:05:30 for that job; it ran 2:43:26 and survived only because the partition it +# was in did not enforce the limit (its two retries doubled memory, correctly leaving the walltime +# alone -- both failures were MEMLIMIT). 6000 asks 4:10:32, which clears the worse of the two runs +# by 1.5x, enough for the 41% they differed by. It costs the other 26 jobs nothing that matters: +# they already ask over two hours, so none of them changes partition. +LAST_TRAIN_SECS = 6000 + +def collapse_inversions_walltime(gfa_size, cores): + """ estimated seconds for one collapse_inversions job, from the graph it is handed """ + return (COLLAPSE_TAIL_THREAD_SECS / collapse_minimap2_threads(cores) + + COLLAPSE_INVERSIONS_SECS_PER_GB * gfa_size / 1e9) + def collapse_inversions(job, options, config_node, pansn_gfa_id, gfa_path): """ rewrite inverted alleles that minigraph stored as novel sequence into proper inversion edges through the reference. @@ -532,8 +622,8 @@ def collapse_inversions(job, options, config_node, pansn_gfa_id, gfa_path): # -j 4 -t 2). A chunk holds ~200 query sequences, so minimap2 keeps a high -t busy on its # own. Note minimap2 runs about two threads more than -t asks for, so this deliberately # leaves headroom rather than saturating. - jobs = min(8, max(1, int(job.cores) // 8)) - threads = max(1, int(job.cores) // jobs) + threads = collapse_minimap2_threads(job.cores) + jobs = max(1, int(job.cores) // threads) cmd = ['rgfa-collapse'] + opts.split() + \ ['-j', str(jobs), '-t', str(threads), '-r', os.path.basename(report), os.path.basename(in_gfa), os.path.basename(snarls)] @@ -547,6 +637,18 @@ def collapse_inversions(job, options, config_node, pansn_gfa_id, gfa_path): out_gfa += '.gz' return job.fileStore.writeGlobalFile(out_gfa), job.fileStore.writeGlobalFile(report) +# Bytes of reference fasta `mash sketch` gets through per second. It took 225 s on the whole +# 3.15e9-byte CHM13 of HPRC v2.0 (1.4e7 B/s) and a p50 of 3.4 s on the ~1.3e8-byte chromosome +# references of HPRC v2.1 (3.9e7 B/s); this is the slow end of that. +MASH_SKETCH_BYTES_PER_SEC = 1e7 + +# Bytes of query fasta one mash_dist job gets through per second. `mash dist` alone runs at +# 7e6-1.9e7 B/s (694 calls over the 232 mash_dist jobs of HPRC v2.0: p50 168 s, p99 429 s for a +# 3.1e9-byte haplotype), but the job also concatenates the sample's haplotypes and counts every +# base of each with Bio.SeqIO.parse, neither of which shows up in the logs as its own command. +# Budgeting those two at ~2e7 and ~2e8 B/s respectively lands the whole job here. +MASH_DIST_BYTES_PER_SEC = 3e6 + def match_gfa_compression(job, gfa_id, pansn_gfa_id, in_path, out_path): """ re-emit an unchanged seed graph at the compression its output path asks for. @@ -585,11 +687,14 @@ def sort_minigraph_input_with_mash(job, options, config_node, seq_id_map, seq_or # assumption : reference is first mash_dists = [(0, sys.maxsize)] # start by sketching the reference to avoid a bunch of recomputation + ref_bytes = seq_id_map[seq_order[0]].size sketch_job = job.addChildJobFn(mash_sketch, seq_order[0], seq_id_map, - disk = seq_id_map[seq_order[0]].size * 2) + disk = ref_bytes * 2, + walltime=cactus_walltime(ref_bytes / MASH_SKETCH_BYTES_PER_SEC, + io_bytes=ref_bytes)) ref_sketch_id = sketch_job.rv() - dist_root_job = Job() + dist_root_job = Job(walltime=cactus_walltime()) sketch_job.addFollowOn(dist_root_job) xml_node = findRequiredNode(config_node, "graphmap") @@ -611,11 +716,15 @@ def sort_minigraph_input_with_mash(job, options, config_node, seq_id_map, seq_or # list of dictionary (promises) that map genome name to mash distance output dist_maps = [] for sample, names in seq_by_sample.items(): + sample_bytes = sum(seq_id_map[x].size for x in names) dist_map = dist_root_job.addChildJobFn(mash_dist, names, seq_order[0], seq_id_map, ref_sketch_id, - disk = 2 * sum(seq_id_map[x].size for x in names) + seq_id_map[seq_order[0]].size).rv() + disk = 2 * sample_bytes + ref_bytes, + walltime=cactus_walltime(sample_bytes / MASH_DIST_BYTES_PER_SEC, + io_bytes=sample_bytes)).rv() dist_maps.append(dist_map) - return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps, trim_ref).rv() + return dist_root_job.addFollowOnJobFn(mash_distance_order, options, config_node, seq_order, dist_maps, trim_ref, + walltime=cactus_walltime()).rv() def mash_sketch(job, ref_seq, seq_id_map): """ get the sketch """ @@ -721,6 +830,72 @@ def mash_distance_order(job, options, config_node, seq_order, mash_output_maps, return mash_order[1:] if trim_ref else mash_order +# Bytes of input fasta `minigraph -xggs` gets through per second per core. Re-fitted on the +# 207 batch constructs of an HPRC v2.1 run (64 cores, 24 chromosomes, 2.7-13.0 GB per batch), +# which is the first at-scale run of the faster minigraph: dividing the batch bytes by the +# observed seconds gives p50 7.5e5, p10 3.3e5 and a single worst point of 7.9e4 B/s/core. +# +# The old value of 4e4 came from the slower fork (8- and 32-core runs of HPRC v2.0/v2.1), and +# carrying it over asked 29 h for a chr1 batch that now takes 2.3 h and 72 h for the last one -- +# every construct into the longest partition, which is the opposite of the point. +# +# The value is set by chrY and nothing else. Solving both runs for the rate at which each batch's +# ask would exactly equal its observed time, the four tightest points of 307 are all chrY (1.8e5 +# to 2.4e5) and the next is chr16 at 6.3e5, against a p50 of 4.1e6 -- so any value that covers +# chrY leaves every other chromosome several times over-provisioned, and there is no way to tell +# them apart from bytes. chrY is slow because it parallelises badly: it holds a CPU factor of 3.9 +# against a p50 of 7.6, and chr2, the next worst, holds 3.2. +# +# 1.5e5 clears chrY's worst observed run by 1.3x, which is the margin that matters because chrY +# swung 14% between the two runs. The cost of covering it is what pushes the median construct to +# 8 h and 46 of 207 batches past 12 h; 2e5 would leave only 14 past 12 h but clears chrY by 1.0x, +# which is to say it times out. Nothing reaches even the shorter of the two ceilings these runs +# saw (84 h), and --doubleTime covers a third run worse than either of these. +MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE = 1.5e5 + +# ...but not linearly. minigraph parallelises over query contigs, and the faster fork adds more +# parallel sections on top of that, but parts of construction remain single-threaded -- so it is +# Amdahl's law rather than a hard ceiling, and how far it scales depends on the data. Measured +# from the CPU factor minigraph reports itself (cputime/elapsed, the `*N` in its +# [M::ggen_map::T*N] lines), time-weighted over every construct process of an HPRC run: at +# --mgCores 64 on the current fork it averages 7.96 cores busy, which is a serial fraction of +# 0.112 and an asymptote near nine cores. +# +# Fitted at one core count, so the asymptote is measured and the shape is Amdahl's assumption. +# It matters most in the middle: at --mgCores 8 this gives 4.5 effective cores where crediting +# the request in full would give 8, and simply capping at the asymptote would too. +MINIGRAPH_SERIAL_FRACTION = 0.112 + +def minigraph_effective_cores(cores): + """ cores minigraph construction can actually keep busy at this core count, by Amdahl's law """ + cores = max(1, int(cores or 1)) + s = MINIGRAPH_SERIAL_FRACTION + return 1.0 / (s + (1.0 - s) / cores) + +# Fixed cost of a construct batch on top of the alignment itself: staging in the previous +# batch's GFA (or the seed graph when extending), which is a promise here and so cannot be +# sized, and -- on the final batch only -- the in-python PanSN rename plus its bgzip, which is +# under 310 s even for the 830 MB whole-genome GFA of HPRC v2.0. +MINIGRAPH_CONSTRUCT_OVERHEAD_SECS = 600 + +# Each batch aligns its genomes against everything the batches before it already put in the +# graph, so the same amount of new sequence costs more the later it arrives. Measured over 23 +# chromosomes of an HPRC run, as batch i's wall time against batch 0's: the p90 runs 1.23, 1.58, +# 1.81, 1.92, 2.08 for i = 1..5 and then flattens, so the growth saturates rather than +# compounding. Keyed off the bytes already in the graph rather than the batch index, so uneven +# batches and the --inGFA seed graph are handled the same way. +# +# The graph itself is the better predictor, but it reaches this point as prev_job.rv(), a promise +# with no size, so the sequence that went into it is the closest thing in scope. +MINIGRAPH_GRAPH_GROWTH = 0.25 +MINIGRAPH_GRAPH_GROWTH_MAX = 2.5 + +def minigraph_graph_growth(prior_bytes, batch_bytes): + """ how much slower this batch is than the first, for the graph already built ahead of it """ + if batch_bytes <= 0: + return 1.0 + return min(1.0 + MINIGRAPH_GRAPH_GROWTH * (prior_bytes / batch_bytes), MINIGRAPH_GRAPH_GROWTH_MAX) + def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_order, gfa_path, whole_genome_ref=False, seed_gfa_id=None, graph_names=None): """ Make minigraph in sequential batches. @@ -770,6 +945,17 @@ def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_or # minigraph_construct() only uses this to name its local copy, but keep the compression # suffix honest since that is what says whether the file it reads is bgzipped seed_gfa_path = 'extend.gfa.gz' if options.inGFA.endswith('.gz') else 'extend.gfa' + def batch_construct_work(i): + """ (bytes batch i adds to the graph, seconds minigraph spends adding them) """ + start = i * max_batch_size + batch_size = len(seq_order) - start if i == num_batches - 1 else max_batch_size + batch_bytes = sum(seq_id_map[e].size for e in seq_order[start:start + batch_size]) + # everything already in the graph this batch has to align against + prior_bytes = sum(seq_id_map[e].size for e in seq_order[:start]) + return batch_bytes, (batch_bytes * minigraph_graph_growth(prior_bytes, batch_bytes) / + (MINIGRAPH_CONSTRUCT_BYTES_PER_SEC_PER_CORE * + minigraph_effective_cores(options.mgCores))) + for i in range(num_batches): batch_size = len(seq_order) - i * max_batch_size if i == num_batches - 1 else max_batch_size input_seq_order = seq_order[i * max_batch_size : (i * max_batch_size) + batch_size] @@ -782,15 +968,29 @@ def minigraph_construct_in_batches(job, options, config_node, seq_id_map, seq_or else: out_gfa_path = '{}.{}'.format(gfa_path, i) pan_sn_output = False + # batch 0 pays for batch 1 as well. Toil chains a job into its predecessor's allocation + # whenever the successor's memory, cores and disk all fit -- walltime is not among the + # things nextChainable() looks at -- so a chained pair runs under the *first* job's Slurm + # time limit. Every batch of a chromosome is issued with identical requirements, and + # batch 0 is the only one whose sole successor is the next batch: from batch 1 on, each + # also carries the clean_jobstore_files follow-on below, and two successors end the chain. + # So batches 0 and 1 always share one allocation, and asking only for batch 0 is what + # killed chrY's first batch in both of the runs these estimates are drawn from. + chained = [i] + ([1] if i == 0 and num_batches > 1 else []) + work = [batch_construct_work(j) for j in chained] minigraph_job = Job.wrapJobFn(minigraph_construct, options, config_node, seq_id_map, input_seq_order, out_gfa_path, prev_job.rv() if prev_job else seed_gfa_id, prev_gfa_path if prev_job else seed_gfa_path, pan_sn_output, graph_names, - disk=disk, memory=mem, cores=options.mgCores) + disk=disk, memory=mem, cores=options.mgCores, + walltime=cactus_walltime( + MINIGRAPH_CONSTRUCT_OVERHEAD_SECS * len(chained) + + sum(secs for _, secs in work), + io_bytes=sum(nbytes for nbytes, _ in work))) if prev_job: prev_job.addFollowOn(minigraph_job) # delete the output of the previous batch from the job store - minigraph_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[prev_job.rv()]) + minigraph_job.addFollowOnJobFn(clean_jobstore_files, file_ids=[prev_job.rv()], walltime=cactus_walltime()) else: job.addChild(minigraph_job) prev_job = minigraph_job @@ -903,6 +1103,18 @@ def minigraph_gfa_to_pansn(names, gfa_path, out_gfa_path, threads=1): out_file.close() bgzip_gfa_rename(raw_out_path, out_gfa_path, threads) +# A bgzipped GFA or PAF decompresses to about 10x its size, the figure the disk requests here are +# already reckoned at. +RAW_BYTES_PER_GZ_BYTE = 10 + +# Seconds per GB of *compressed* GFA to rename it between PanSN and Cactus. Nothing measures this +# one directly -- cactus-pangenome passes pansn_gfa_input=False, so it only runs from the +# standalone cactus-graphmap and cactus-graphmap-split entry points and it fired in none of the +# runs we have logs for. It rewrites every S-line of the decompressed GFA in python (~40 MB/s) and +# bgzips the result back up (~25 MB/s, the rate of the whole-panel bgzips that were measured), both +# single-threaded and both over a raw GFA ~10x the compressed input it is handed. +GFA_RENAME_SECS_PER_GB = 700 + def minigraph_gfa_from_pansn(job, names, gfa_path, gfa_id): """ hack to convert PanSN names like simChimp#0#simpChimp.chr6 to Cactus names like id=simChimp.0|simChimp.chr6 so that a minigrpah GFA (as converted panSN by minigraph_gfa_to_pansn() above) can be read back into Cactus diff --git a/src/cactus/refmap/cactus_pangenome.py b/src/cactus/refmap/cactus_pangenome.py index 7fbc31194..47051593a 100644 --- a/src/cactus/refmap/cactus_pangenome.py +++ b/src/cactus/refmap/cactus_pangenome.py @@ -21,13 +21,13 @@ from operator import itemgetter from cactus.progressive.seqFile import SeqFile -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import cactusRootPath from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import makeURL, catFiles from cactus.shared.common import RAW_VG_SUFFIX from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import cactus_call from cactus.shared.common import getOptionalAttrib, findRequiredNode from cactus.shared.common import clean_jobstore_files @@ -108,7 +108,7 @@ def pangenome_options(parser): "Standard suffixes like K, Ki, M, Mi, G or Gi are supported (default=bytes))", default=None) parser.add_argument("--consRetainPages", choices=['auto', '0', '1'], default=None, help="Whether cactus_consolidated keeps the memory pages jemalloc frees, which is much faster but takes 2-3x the peak memory. " - "auto (the default, from in the config) keeps them unless the memory estimate exceeds what the job can be given") + "auto (the default, from in the config) keeps them only when the estimate fits in memory_retain_auto_fraction of what the job can be given") # cactus-graphmap options parser.add_argument("--collapseRefPAF", help ="Incorporate given reference self-alignments in PAF format") @@ -279,6 +279,7 @@ def pangenome_config_overrides(options, config_node): def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (will be modified if necessary to include graph Fasta sequence)") parser.add_argument("--outDir", help = "Output directory", required=True) @@ -367,7 +368,7 @@ def main(): input_seq_order.remove(genome) toil.start(Job.wrapJobFn(pangenome_end_to_end_workflow, options, config_wrapper, input_seq_id_map, input_path_map, input_seq_order, ref_collapse_paf_id, last_scores_id, - in_gfa_id=in_gfa_id, in_gaf_id=in_gaf_id)) + in_gfa_id=in_gfa_id, in_gaf_id=in_gaf_id, walltime=cactus_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -473,7 +474,7 @@ def sanitize_fasta_headers_batch(job, chromfile_id_map): seq_id_map = value[0] else: seq_id_map = value - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) if type(value) is list: out_id_map[chrom] = [sanitize_job.rv()] + value[1:] else: @@ -568,7 +569,8 @@ def export_pruned_minigraph_gfa_wrapper(job, options, graphmap_batch_results, un # the whole-genome graphs the pruned ones replaced. freed from a follow-on rather than inline so # a retry of this job doesn't trip over its own delete if unused_ids: - job.addFollowOnJobFn(clean_jobstore_files, file_ids=unused_ids, allow_none=True) + job.addFollowOnJobFn(clean_jobstore_files, file_ids=unused_ids, allow_none=True, + walltime=cactus_walltime()) return pansn_gfa_ids def make_batch_align_jobs_wrapper(job, options, chromfile_path, config_wrapper, last_scores_id): @@ -632,15 +634,26 @@ def export_join_wrapper(job, options, wf_output, contig_sizes_id=None): def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, in_gfa_id=None, in_gaf_id=None): """ chain the entire workflow together, doing exports after each step to mitigate annoyance of failures """ - root_job = Job() + root_job = Job(walltime=cactus_walltime()) job.addChild(root_job) config_node = config_wrapper.xmlRoot + # Every file this pipeline exports -- the GFA, the PAF, the seqfile, the whole join output -- + # is produced by a job further down and reaches its export site as a promise, which has no + # size. The assemblies that went in are the one real size we have here, and since the + # pangenome collapses the sequence they share, every one of those outputs is smaller than + # their total. So it is used as the bound for the export jobs' I/O below: generous for the + # small ones, but these run once per workflow, and the alternative is guessing. The two jobs + # that pull the split tree back into the jobstore (import_minigraph_batch_wrapper and + # make_batch_align_jobs_wrapper) move the same sequence in the other direction, so they use it + # too. + input_seq_bytes = sum(seq_id.size for seq_id in seq_id_map.values()) + # make sure this is done up front root_job = vcflib_checks(root_job, options, config_node) # sanitize headers (once here, skip in all workflows below) - sanitize_job = root_job.addFollowOnJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True) + sanitize_job = root_job.addFollowOnJobFn(sanitize_fasta_headers, seq_id_map, pangenome=True, walltime=cactus_walltime()) seq_id_map = sanitize_job.rv() # snapshot the input contig sizes while the sanitized fastas still exist: they are the baseline @@ -655,7 +668,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ prev_job = sanitize_job.addFollowOnJobFn( contig_sizes_job, seq_id_map, getOptionalAttrib(findRequiredNode(config_node, "graphmap"), "assemblyName", - default="_MINIGRAPH_")) + default="_MINIGRAPH_"), walltime=cactus_walltime()) contig_sizes_id = prev_job.rv() assert type(options.reference) == list @@ -688,7 +701,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ split_config_node = config_node split_config_wrapper = config_wrapper minigraph_job = prev_job.addFollowOnJobFn(minigraph_construct_workflow, mg_options, split_config_node, seq_id_map, seq_order, sv_gfa_path, sanitize=False, - in_gfa_id=in_gfa_id) + in_gfa_id=in_gfa_id, walltime=cactus_walltime()) sv_gfa_id = minigraph_job.rv(0) pansn_sv_gfa_id = minigraph_job.rv(1) if not last_scores_id: @@ -698,7 +711,8 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ # only build reference graph on first pass when doing minigraph-by-chrom pipeline minigraph_wrapper_job = minigraph_job.addFollowOnJobFn(export_minigraph_wrapper, options, pansn_sv_gfa_id, sv_gfa_path, last_scores_id, uncollapsed_pansn_gfa_id=minigraph_job.rv(2), - collapse_report_id=minigraph_job.rv(3)) + collapse_report_id=minigraph_job.rv(3), + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) # cactus_graphmap paf_path = os.path.join(options.outDir, first_pass_name + '.paf') @@ -710,17 +724,19 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ if options.mgSplit: gm_options.collapse = False graphmap_job = minigraph_wrapper_job.addFollowOnJobFn(minigraph_workflow, gm_options, split_config_wrapper, seq_id_map, sv_gfa_id, graph_event, False, ref_collapse_paf_id, pansn_gfa_input=False, - in_gaf_id=in_gaf_id) + in_gaf_id=in_gaf_id, walltime=cactus_walltime()) paf_id, gfa_fa_id, gaf_id, unfiltered_paf_id, paf_filter_log = graphmap_job.rv(0), graphmap_job.rv(1), graphmap_job.rv(2), graphmap_job.rv(3), graphmap_job.rv(4) - graphmap_export_job = graphmap_job.addFollowOnJobFn(export_graphmap_wrapper, options, paf_id, paf_path, gaf_id, unfiltered_paf_id, paf_filter_log) + graphmap_export_job = graphmap_job.addFollowOnJobFn(export_graphmap_wrapper, options, paf_id, paf_path, gaf_id, unfiltered_paf_id, paf_filter_log, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) # we need to update the seqfile with the phonied in minigraph event - update_seqfile_job = graphmap_export_job.addFollowOnJobFn(update_seqfile, options, seq_id_map, seq_path_map, seq_order, gfa_fa_id, gfa_fa_path, graph_event) + update_seqfile_job = graphmap_export_job.addFollowOnJobFn(update_seqfile, options, seq_id_map, seq_path_map, seq_order, gfa_fa_id, gfa_fa_path, graph_event, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) seq_id_map, seq_path_map, seq_name_map = update_seqfile_job.rv(0), update_seqfile_job.rv(1), update_seqfile_job.rv(2) if options.noSplit: # we phony in the entire alignment as one chromsome called 'all' - phony_chromfile_job = update_seqfile_job.addFollowOnJobFn(phony_chromfile, options, paf_path) + phony_chromfile_job = update_seqfile_job.addFollowOnJobFn(phony_chromfile, options, paf_path, walltime=cactus_walltime()) chromfile_path = phony_chromfile_job.rv() split_export_job = phony_chromfile_job # nothing is binned or dropped without a split, so the exclusion report has no split log @@ -729,11 +745,12 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ else: # cactus_graphmap_split split_job = update_seqfile_job.addFollowOnJobFn(graphmap_split_workflow, options, split_config_wrapper, seq_id_map, seq_name_map, sv_gfa_id, - sv_gfa_path, paf_id, paf_path, sanitize=False, pansn_gfa_input=False) + sv_gfa_path, paf_id, paf_path, sanitize=False, pansn_gfa_input=False, walltime=cactus_walltime()) wf_output = split_job.rv() split_log_id = split_job.rv(2) split_out_path = os.path.join(options.outDir, 'chrom-subproblems') - split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper) + split_export_job = split_job.addFollowOnJobFn(export_split_wrapper, wf_output, split_out_path, split_config_wrapper, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) chromfile_path = os.path.join(split_out_path, 'chromfile.txt') # with --mgSplitWholeGenomeRef the second pass builds each chromosome's graph against the whole @@ -741,31 +758,35 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ wg_ref_id_map = None clean_seq_id_map = seq_id_map if options.mgSplitWholeGenomeRef: - split_ref_job = split_export_job.addFollowOnJobFn(split_reference_ids, seq_id_map, options.reference[:1]) + split_ref_job = split_export_job.addFollowOnJobFn(split_reference_ids, seq_id_map, options.reference[:1], + walltime=cactus_walltime()) wg_ref_id_map, clean_seq_id_map = split_ref_job.rv(0), split_ref_job.rv(1) split_export_job = split_ref_job # clean out some jobstore files we no longer need clean_jobstore_job = split_export_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[clean_seq_id_map] if not options.noSplit else None, - file_ids=[sv_gfa_id, paf_id]) + file_ids=[sv_gfa_id, paf_id], walltime=cactus_walltime()) options.batch = True minigraph_pansn_sv_gfa_ids = [] if options.mgSplit: # rerun cactus_minigraph but on a per-chromosome basis minigraph_batch_import_job = clean_jobstore_job.addFollowOnJobFn(import_minigraph_batch_wrapper, options, config_wrapper, - chromfile_path) + chromfile_path, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) input_seqfiles = minigraph_batch_import_job.rv(0) raw_input_map = minigraph_batch_import_job.rv(1) - sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map) + sanitize_job = minigraph_batch_import_job.addFollowOnJobFn(sanitize_fasta_headers_batch, raw_input_map, walltime=cactus_walltime()) input_map = sanitize_job.rv() options.outputGFA='' minigraph_batch_job = sanitize_job.addFollowOnJobFn(minigraph_construct_batch_workflow, options, config_node, input_map, None, sanitize=False, - construct_ref_id_map=wg_ref_id_map) + construct_ref_id_map=wg_ref_id_map, + walltime=cactus_walltime()) minigraph_batch_results = minigraph_batch_job.rv() minigraph_batch_export_job = minigraph_batch_job.addFollowOnJobFn(export_minigraph_batch_wrapper, options, config_node, - input_seqfiles, input_map, minigraph_batch_results) + input_seqfiles, input_map, minigraph_batch_results, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) # now rerun cactus_graphmap but on a per-chromosome bassis graphmap_input_dict = minigraph_batch_export_job.rv(0) @@ -775,7 +796,7 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ unpruned_pansn_sv_gfa_map = minigraph_batch_export_job.rv(4) graphmap_batch_job = minigraph_batch_export_job.addFollowOnJobFn(minigraph_batch_workflow, options, config_wrapper, graphmap_input_dict, graph_event, sanitize=False, - pansn_gfa_input=False) + pansn_gfa_input=False, walltime=cactus_walltime()) # hold the contigs of any multi-reference-contig bin apart. the export has to be chained # onto this job, not onto graphmap_batch_job alongside it, or it runs while the separation # pass's children are still going and reads their promises unresolved @@ -786,32 +807,37 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ # that pass is what cut the rest of the reference out of each chromosome's graph, so the # published .sv.gfa.gz files only exist to write once it's done pruned_gfa_export_job = separate_job.addFollowOnJobFn(export_pruned_minigraph_gfa_wrapper, options, - graphmap_batch_results, unpruned_pansn_sv_gfa_map) + graphmap_batch_results, unpruned_pansn_sv_gfa_map, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) minigraph_pansn_sv_gfa_ids = pruned_gfa_export_job.rv() prev_batch_job = pruned_gfa_export_job graphmap_batch_export_job = prev_batch_job.addFollowOnJobFn(export_graphmap_batch_wrapper, options, config_node, - graphmap_batch_results, input_seqfiles) + graphmap_batch_results, input_seqfiles, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) graphmap_file_ids = graphmap_batch_export_job.rv(0) chromfile_path = graphmap_batch_export_job.rv(1) # clean out the jobstore, as cactus_align reads everything from disk - clean_jobstore_job = graphmap_batch_export_job.addFollowOnJobFn(clean_jobstore_files, file_ids=graphmap_file_ids) + clean_jobstore_job = graphmap_batch_export_job.addFollowOnJobFn(clean_jobstore_files, file_ids=graphmap_file_ids, walltime=cactus_walltime()) clean_jobstore_job = clean_jobstore_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=minigraph_output_maps, - file_ids=minigraph_output_ids, allow_none=True) + file_ids=minigraph_output_ids, allow_none=True, walltime=cactus_walltime()) if options.mgSplitWholeGenomeRef: # the whole-genome reference fastas held back above: every construct that wanted them # has long since run by here clean_jobstore_job = clean_jobstore_job.addFollowOnJobFn(clean_jobstore_files, - file_id_maps=[wg_ref_id_map], allow_none=True) + file_id_maps=[wg_ref_id_map], allow_none=True, + walltime=cactus_walltime()) # cactus_align options.scoresFromChromfile = options.lastTrain and options.mgSplit align_jobs_make_job = clean_jobstore_job.addFollowOnJobFn(make_batch_align_jobs_wrapper, options, chromfile_path, config_wrapper, - last_scores_id) + last_scores_id, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) align_jobs = align_jobs_make_job.rv() - align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs) + align_job = align_jobs_make_job.addFollowOnJobFn(batch_align_jobs, align_jobs, walltime=cactus_walltime()) results_dict = align_job.rv() - align_export_job = align_job.addFollowOnJobFn(export_align_wrapper, options, results_dict) + align_export_job = align_job.addFollowOnJobFn(export_align_wrapper, options, results_dict, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) join_options, vg_ids, hal_ids = align_export_job.rv(0), align_export_job.rv(1), align_export_job.rv(2) # cactus_graphmap_join @@ -821,17 +847,19 @@ def pangenome_end_to_end_workflow(job, options, config_wrapper, seq_id_map, seq_ join_job = align_export_job.addFollowOnJobFn(graphmap_join_workflow, join_options, config_wrapper, vg_ids, [] if options.noHal else hal_ids, minigraph_pansn_sv_gfa_ids, contig_sizes_id=contig_sizes_id, - split_log_id=split_log_id) + split_log_id=split_log_id, walltime=cactus_walltime()) join_wf_output = join_job.rv() if options.noHal: - join_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids) + join_job.addFollowOnJobFn(clean_jobstore_files, file_ids=hal_ids, walltime=cactus_walltime()) # cactus-panpatch exports the chromosome vgs itself (they're the only thing export_join_data # would write, given it turns every other output off), so let it skip this to avoid writing # the biggest output twice if not options.noJoinExport: + # the biggest export of the run: every chromosome graph, index and VCF the join made join_job.addFollowOnJobFn(export_join_wrapper, join_options, join_wf_output, - contig_sizes_id=contig_sizes_id) + contig_sizes_id=contig_sizes_id, + walltime=cactus_walltime(0, io_bytes=2 * input_seq_bytes)) return join_options, join_wf_output, seq_id_map diff --git a/src/cactus/refmap/cactus_panpatch.py b/src/cactus/refmap/cactus_panpatch.py index e28aac7de..dc77a3690 100644 --- a/src/cactus/refmap/cactus_panpatch.py +++ b/src/cactus/refmap/cactus_panpatch.py @@ -28,7 +28,7 @@ from cactus.progressive.seqFile import SeqFile from cactus.progressive.cactus_prepare import human2bytesN -from cactus.shared.common import importSingularityImage +from cactus.shared.common import importSingularityImage, cactus_walltime, add_cactus_toil_options from cactus.shared.common import makeURL from cactus.shared.common import cactus_call from cactus.shared.common import cactus_clamp_memory @@ -50,6 +50,7 @@ def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", nargs='?', default=None, help = "Seq file (as with cactus-pangenome), or, with --batch, a chromfile with one " @@ -244,7 +245,7 @@ def main(): mask_bed_ids, target_error_bed_ids)) toil.start(Job.wrapJobFn(panpatch_batch_workflow, options, config_wrapper, run_inputs, - exclude_bed_id)) + exclude_bed_id, walltime=cactus_walltime())) end_time = timeit.default_timer() run_time = end_time - start_time @@ -596,6 +597,31 @@ def pangenome_dir(options, run): """ where a run's intermediate cactus-pangenome output goes (deleted on success) """ return os.path.join(options.outDir, run['name'] + '.cactus-scratch') +# Walltime estimates for the jobs below. cactus-panpatch is newer than any run we have logs for, +# so none of these three rates was measured on it: each is the slow end of the nearest thing that +# was measured, and each covers one job per sample or one per chromosome graph. + +# Bytes of input assembly per second for the error-masking pass, which rewrites the whole assembly +# in a pure-python per-line loop -- an order of magnitude slower per byte than the compiled +# cactus_sanitizeFastaHeaders that reads its output. Reckoned against the FileID's size, which is +# the *compressed* size when the input is gzipped even though the loop runs over the ~3x larger +# sequence, so keep it pessimistic. +MASK_FASTA_BYTES_PER_SEC = 4e6 + +# Bytes of chromosome graph per second for one single-threaded panpatch job. panpatch itself takes +# a couple of minutes on a whole human pangenome (see the module docstring); what dominates is +# loading the graph, and it gets loaded twice -- once by the `vg paths -L -x` that lists the samples +# in it, once by panpatch. 20 MB/s is the slow end of the vg graph loads measured on the HPRC v2.1 +# chromosome graphs (`vg paths` p90 165s, `vg paths|awk` p90 471s, on graphs of ~24 GiB). +PANPATCH_BYTES_PER_SEC = 20e6 + +# Bytes of output assembly per second for the gather pass: single-threaded bgzip of each output +# haplotype (21-27 MB/s, measured on the HPRC PAFs -- see GZIP_COMPRESS_BYTES_PER_SEC in +# cactus.shared.common), +# plus two whole-genome python passes over it (the masked-target revert and the dropped-contig +# rescue) plus panpatch's telomere report over each finished fasta. +GATHER_BYTES_PER_SEC = 8e6 + def panpatch_batch_workflow(job, options, config_wrapper, run_inputs, exclude_bed_id): """ patch each sample. the runs are completely independent of each other: they share no input files in the jobstore (see the import loop in main()) """ @@ -603,10 +629,11 @@ def panpatch_batch_workflow(job, options, config_wrapper, run_inputs, exclude_be for run, pg_options, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, target_fasta_ids, mask_bed_ids, target_error_bed_ids in run_inputs: run_job = job.addChildJobFn(panpatch_run_workflow, options, pg_options, config_wrapper, run, seq_id_map, seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, exclude_bed_id, - target_fasta_ids, mask_bed_ids, target_error_bed_ids) + target_fasta_ids, mask_bed_ids, target_error_bed_ids, + walltime=cactus_walltime()) summaries.append((run['name'], run_job.rv())) # once every sample is done, roll the per-sample reports up into one cross-sample summary - job.addFollowOnJobFn(write_batch_summary, options, summaries) + job.addFollowOnJobFn(write_batch_summary, options, summaries, walltime=cactus_walltime()) def mask_assembly_errors(job, fasta_id, bed_id): """ return a copy of an input assembly fasta with each error-BED interval replaced by an equal-length @@ -674,18 +701,21 @@ def panpatch_run_workflow(job, options, pg_options, config_wrapper, run, seq_id_ for sample, bed_id in mask_bed_ids.items(): mask_job = job.addChildJobFn(mask_assembly_errors, seq_id_map[sample], bed_id, disk=seq_id_map[sample].size * 4 + 2**30, - memory=cactus_clamp_memory(max(2**32, seq_id_map[sample].size * 2))) + memory=cactus_clamp_memory(max(2**32, seq_id_map[sample].size * 2)), + walltime=cactus_walltime(seq_id_map[sample].size / MASK_FASTA_BYTES_PER_SEC, + io_bytes=seq_id_map[sample].size * 4)) masked[sample] = mask_job.rv() # the pangenome runs as a follow-on so any masking children have completed first (their promises # resolved); a follow-on of the job hosting the pangenome then runs once its entire child subtree is # done, which is what makes panpatch_workflow safe pangenome_job = job.addFollowOnJobFn(pangenome_end_to_end_workflow, pg_options, config_wrapper, masked, - seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id) + seq_path_map, seq_order, ref_collapse_paf_id, last_scores_id, + walltime=cactus_walltime()) # return the run's output-file map (its report id feeds the cross-sample batch summary) return pangenome_job.addFollowOnJobFn(panpatch_workflow, options, run, pangenome_job.rv(0), pangenome_job.rv(1), pangenome_job.rv(2), exclude_bed_id, target_fasta_ids, - target_error_bed_ids).rv() + target_error_bed_ids, walltime=cactus_walltime()).rv() def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_map, exclude_bed_id, target_fasta_ids, target_error_bed_ids): @@ -725,7 +755,9 @@ def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_ma mem = min(mem, options.indexMemory) cj = job.addChildJobFn(run_panpatch_chrom, options, run, vg_id, vg_name, exclude_bed_id, target_error_bed_ids, - cores=1, memory=mem, disk=vg_id.size * 4 + 2**30) + cores=1, memory=mem, disk=vg_id.size * 4 + 2**30, + walltime=cactus_walltime(vg_id.size / PANPATCH_BYTES_PER_SEC, + io_bytes=vg_id.size)) chrom_jobs.append(cj) # gather the per-chromosome outputs, rescue dropped contigs, and bgzip. the concatenated fastas @@ -739,9 +771,15 @@ def panpatch_workflow(job, options, run, join_options, join_wf_output, seq_id_ma gather_job = job.addFollowOnJobFn(gather_panpatch, options, run, [cj.rv() for cj in chrom_jobs], target_fasta_ids, target_error_bed_ids, memory=cactus_clamp_memory(max(2**32, gather_mem)), - disk=gather_disk) + disk=gather_disk, + walltime=cactus_walltime(run['ploidy'] * ref_size / GATHER_BYTES_PER_SEC, + io_bytes=gather_disk)) + # this copies the patched assemblies out, and with --keepGraphs every chromosome graph too, + # so it is bounded by the same bytes its disk request is + export_bytes = sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30 gather_job.addFollowOnJobFn(export_panpatch_wrapper, options, run, gather_job.rv(), full_vg_ids, vg_names, - disk=sum(vg_id.size for vg_id in full_vg_ids) * 2 + 2**30) + disk=export_bytes, + walltime=cactus_walltime(0, io_bytes=export_bytes)) # surface this run's output-file map (incl. its report id) up to the cross-sample batch summary return gather_job.rv() @@ -1146,7 +1184,10 @@ def export_panpatch_wrapper(job, options, run, output_id_map, full_vg_ids, vg_na job.fileStore.exportFile(vg_id, makeURL(os.path.join(chrom_dir, vg_name))) if not options.keepPangenome: - job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run) + # deleting the scratch tree costs one operation per file (~n_chroms * n_genomes of them), + # not one per byte: seconds of rmtree locally, but a serial paginated delete loop on an + # s3:// outDir, which is why it is off the coordination tier + job.addFollowOnJobFn(cleanup_pangenome_wrapper, options, run, walltime=cactus_walltime(600)) def concat_reports(chrom_report_paths, cap_lines, out_path): """ concatenate the per-chromosome panpatch reports (in the order given) into one sample report: diff --git a/src/cactus/refmap/cactus_refmap.py b/src/cactus/refmap/cactus_refmap.py index ed032daf0..74c620368 100644 --- a/src/cactus/refmap/cactus_refmap.py +++ b/src/cactus/refmap/cactus_refmap.py @@ -36,12 +36,12 @@ from cactus.refmap import fasta_preprocessing from cactus.refmap import apply_dipcall_bed_filter -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import makeURL from cactus.shared.common import cactus_call from cactus.shared.configWrapper import ConfigWrapper from cactus.shared.common import cactusRootPath -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set from cactus.preprocessor.checkUniqueHeaders import sanitize_fasta_headers @@ -139,8 +139,12 @@ def run_cactus_reference_align(job, assembly_files, reference, debug_export=Fals """ Preprocesses assemblies, then runs mappings. """ - sanitize_job = job.addChildJobFn(sanitize_fasta_headers, assembly_files) - mappings = sanitize_job.addFollowOnJobFn(map_all_to_ref, sanitize_job.rv(), reference, debug_export, dipcall_bed_filter, dipcall_vcf_filter).rv() + # map_all_to_ref reads every assembly out of the jobstore before it schedules anything, so + # it is bounded by the inputs; sanitizing does not change their size + input_seq_bytes = sum(seq_id.size for seq_id in assembly_files.values()) + sanitize_job = job.addChildJobFn(sanitize_fasta_headers, assembly_files, walltime=cactus_walltime()) + mappings = sanitize_job.addFollowOnJobFn(map_all_to_ref, sanitize_job.rv(), reference, debug_export, dipcall_bed_filter, dipcall_vcf_filter, + walltime=cactus_walltime(0, io_bytes=input_seq_bytes)).rv() return mappings def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_bed_filter=False, dipcall_vcf_filter=False): @@ -162,7 +166,16 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b * Filters out all mappings below min_var_len=50k and min_mapq=5 from a lastz file Defaults to False. """ - lead_job = job.addChildJobFn(empty) + lead_job = job.addChildJobFn(empty, walltime=cactus_walltime()) + + # None of the runs we have logs for exercise cactus-refmap, so the walltimes below are sized + # from the input assemblies rather than measured. minimap2 asm5 is budgeted at 5e5 bytes of + # combined input a second: the whole-genome minigraph mapping rate (1.3e6 B/s at 8 cores) + # halved, since this job asks for no cores and minimap2 defaults to 3 threads. The python + # passes downstream of it are sized off the assembly too -- the pafs they actually read are + # promises here and have no size at scheduling time -- taking a paf to be a fifth of the + # bytes of the assembly that produced it and a line-by-line python pass to run at ~5e7 B/s. + total_bytes = sum(seq_id.size for seq_id in assembly_files.values()) # map all assemblies to the reference. Don't map reference to reference, though. ref_mappings = dict() @@ -172,14 +185,26 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b if assembly != reference: # map to a to b print("about to run map a to b. a:", assembly, job.fileStore.readGlobalFile(assembly_file), "b (ref):", reference, job.fileStore.readGlobalFile(assembly_files[reference])) - map_job = lead_job.addChildJobFn(map_a_to_b, assembly_file, assembly_files[reference], (dipcall_bed_filter or dipcall_vcf_filter)) + pair_bytes = assembly_file.size + assembly_files[reference].size + map_job = lead_job.addChildJobFn(map_a_to_b, assembly_file, assembly_files[reference], (dipcall_bed_filter or dipcall_vcf_filter), + walltime=cactus_walltime(pair_bytes / 5e5, io_bytes=pair_bytes)) ref_mappings[assembly] = map_job.rv() if dipcall_bed_filter: - secondaries_filter_job = map_job.addFollowOnJobFn(filter_out_secondaries_from_paf, ref_mappings[assembly]) + secondaries_filter_job = map_job.addFollowOnJobFn(filter_out_secondaries_from_paf, ref_mappings[assembly], + walltime=cactus_walltime(assembly_file.size / 2.5e8, + io_bytes=2 * assembly_file.size / 5)) primary_paf = secondaries_filter_job.rv() - dipcall_bed_filter_job = secondaries_filter_job.addFollowOnJobFn(apply_dipcall_bed_filter.apply_dipcall_bed_filter, primary_paf) + # the bed filter is not a pass over the paf at all: extract_single_mappings() + # scans every single-mapping region for every large mapping (its own todo at + # the top of apply_dipcall_bed_filter.py), so it is quadratic in the mappings + # of a chromosome, and it print()s the whole region dict into the worker log + # on the way. A rate per byte of assembly is only a stand-in for that, so it + # is a slow one -- ~3100 s for a human assembly. + dipcall_bed_filter_job = secondaries_filter_job.addFollowOnJobFn(apply_dipcall_bed_filter.apply_dipcall_bed_filter, primary_paf, + walltime=cactus_walltime(assembly_file.size / 1e6, + io_bytes=2 * assembly_file.size / 5)) bed_filtered_primary_mappings = dipcall_bed_filter_job.rv() paf_mappings = bed_filtered_primary_mappings else: @@ -191,7 +216,9 @@ def map_all_to_ref(job, assembly_files, reference, debug_export=False, dipcall_b secondary_mappings[assembly] = None # consolidate the primary mappings into a single file; same for secondary mappings. - all_primary = lead_job.addFollowOnJobFn(consolidate_mappings, primary_mappings).rv() + all_primary = lead_job.addFollowOnJobFn(consolidate_mappings, primary_mappings, + walltime=cactus_walltime(total_bytes / 2.5e8, + io_bytes=2 * total_bytes / 5)).rv() return all_primary def map_a_to_b(job, a, b, dipcall_filter): @@ -228,6 +255,7 @@ def map_a_to_b(job, a, b, dipcall_filter): def get_options(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) # addCactusWorkflowOptions(parser) # ### For quick debugging of apply_dipcall_bed_filter: @@ -327,14 +355,19 @@ def main(): ## Perform alignments: if not toil.options.restart: - alignments = toil.start(Job.wrapJobFn(run_cactus_reference_align, input_seq_id_map, options.reference, options.debug_export, options.dipcall_bed_filter, options.dipcall_vcf_filter)) + alignments = toil.start(Job.wrapJobFn(run_cactus_reference_align, input_seq_id_map, options.reference, options.debug_export, options.dipcall_bed_filter, options.dipcall_vcf_filter, walltime=cactus_walltime())) else: alignments = toil.restart() ## Save alignments: if options.dipcall_vcf_filter: # this is substantially less restrictive than the dipcall_bed_filter. - dipcall_filtered = toil.start(Job.wrapJobFn(apply_dipcall_vcf_filter, alignments)) + # variation_length() print()s once per cigar token of every line and the toil worker + # captures all of it, so this job is paced by the printing, not by the parsing: + # reckon on ~1e6 bytes of input a second until that print goes away. + dipcall_filtered = toil.start(Job.wrapJobFn(apply_dipcall_vcf_filter, alignments, + walltime=cactus_walltime(alignments.size / 1e6, + io_bytes=2 * alignments.size))) toil.exportFile(dipcall_filtered, makeURL(options.outputFile)) else: toil.exportFile(alignments, makeURL(options.outputFile)) diff --git a/src/cactus/refmap/pangenome_exclusions.py b/src/cactus/refmap/pangenome_exclusions.py index 8dbfd17f2..ac0138c2c 100644 --- a/src/cactus/refmap/pangenome_exclusions.py +++ b/src/cactus/refmap/pangenome_exclusions.py @@ -25,7 +25,7 @@ import re import shutil -from cactus.shared.common import cactus_call, getOptionalAttrib, findRequiredNode +from cactus.shared.common import cactus_call, getOptionalAttrib, findRequiredNode, cactus_walltime from cactus.shared.common import cactus_clamp_memory from toil.realtimeLogger import RealtimeLogger @@ -1139,11 +1139,15 @@ def contig_sizes_job(job, seq_id_map, graph_event): # sanitized fasta first, and cgroup accounting charges that page cache to the job -- which is # what killed one of these with MEMLIMIT on a 3GB genome under the 2GiB default. size it off # the fasta the way sanitize_fasta_header, which stages the same file, already does, but keep - # the old default as a floor: scaling alone would ask a small genome for less than the worker + # the old default as a floor: scaling alone would ask a small genome for less than the worker. + # the faidx is 21s measured, so the walltime is the staging of that same fasta per_event[event] = job.addChildJobFn(contig_sizes_for_event, fa_id, event, memory=cactus_clamp_memory(max(fa_id.size * 2, 2**31)), - disk=fa_id.size * 3).rv() - return job.addFollowOnJobFn(merge_contig_sizes, per_event).rv() + disk=fa_id.size * 3, + walltime=cactus_walltime(60, io_bytes=fa_id.size)).rv() + # the rows arrive through promises, so this stages no file at all: it deserialises a few + # million tuples and writes one small gzipped TSV. no size is in scope to key off + return job.addFollowOnJobFn(merge_contig_sizes, per_event, walltime=cactus_walltime(300)).rv() def contig_sizes_for_event(job, fa_id, event): diff --git a/src/cactus/setup/cactus_align.py b/src/cactus/setup/cactus_align.py index 3eb4c5560..7c712e704 100644 --- a/src/cactus/setup/cactus_align.py +++ b/src/cactus/setup/cactus_align.py @@ -15,20 +15,22 @@ import multiprocessing from operator import itemgetter -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.pipeline.cactus_workflow import cactus_cons_with_resources from cactus.progressive.progressive_decomposition import compute_outgroups, parse_seqfile, get_subtree, get_spanning_subtree, get_event_set, get_ancestor_scaled_tree from cactus.progressive.cactus_progressive import export_hal from cactus.shared.common import makeURL, catFiles from cactus.shared.common import RAW_VG_SUFFIX from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import findRequiredNode from cactus.shared.common import getOptionalAttrib from cactus.shared.common import cactus_call from cactus.shared.common import write_s3, has_s3, get_aws_region, unzip_gzs, unzip_gz from cactus.shared.common import cactusRootPath from cactus.shared.common import cactus_clamp_memory +from cactus.shared.common import unzip_gz_walltime +from cactus.pipeline.cactus_workflow import cons_core_scale from cactus.shared.common import clean_jobstore_files from cactus.shared.version import cactus_commit from cactus.shared.configWrapper import ConfigWrapper @@ -47,8 +49,37 @@ from sonLib.nxnewick import NXNewick from sonLib.bioio import getTempDirectory, getTempFile +# How much bigger a PAF gets when it is unzipped. Measured over the 576 VGP 577-way +# alignments, whose gzipped PAFs decompress to about three times their compressed size. +PAF_GZIP_RATIO = 3 + +# align_toil hosts a whole nested cactus-align on one node. The size term borrows the +# runtime fit's exponent and raises its coefficient for the HAL export and +# optional hal2vg the nested run also pays for; the startup term is nested-Toil startup alone. +ALIGN_TOIL_STARTUP_SECS = 600 +ALIGN_TOIL_SECS_PER_GB = 8000 +ALIGN_TOIL_EXPONENT = 0.78 + +# The pangenome PAF filter, in seconds per GB of PAF. Two sequential passes over every line. +# The python filter loop in filter_paf() runs no subprocess and so is timed in no log; at ~150 +# bytes a line and a split()/int()/float() per line it does ~20 MiB/s, i.e. ~48 s/GB. The +# optional `gaffilter` overlap pass on top of it measures 45 s/GB across the 52 HPRC v2.0 and +# v2.1 chromosomes (1.46 GB in 66.5 s at the median, 34.4 GB in 1475 s at the worst). +FILTER_PAF_SECS_PER_GB = 95 + +# hal2vg, in seconds per GB of HAL. Across the 50 hal2vg jobs of two HPRC pangenomes (177 h +# over 1.1 TB of HAL) the cost per GB is flat: 566 s/GB in aggregate, and the slowest run -- +# 8.5 h -- was at most 830 s/GB against the largest HAL in either run, 36.7 GB. 600 puts the +# largest of them at 1.8x what it took. HAL sizes here are recovered from each job's disk +# request, which is exactly 3x the HAL; the memory request is not usable for this because +# cactus_clamp_memory bounds it, which flattens the small end and manufactures a superlinear +# trend that is not there. The `vg view -g | gzip` that --outGFA adds ran in none of the +# logged runs, so cactus_walltime()'s factor is what covers it. +HAL2VG_SECS_PER_GB = 600 + def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("seqFile", help = "Seq file (or chromfile with --batch)") parser.add_argument("pafFile", nargs='?', default='', type=str, help = "Pairiwse aliginments (from cactus-blast, cactus-refmap or cactus-graphmap)") @@ -109,7 +140,7 @@ def main(): "Standard suffixes like K, Ki, M, Mi, G or Gi are supported (default=bytes))", default=None) parser.add_argument("--consRetainPages", choices=['auto', '0', '1'], default=None, help="Whether cactus_consolidated keeps the memory pages jemalloc frees, which is much faster but takes 2-3x the peak memory. " - "auto (the default, from in the config) keeps them unless the memory estimate exceeds what the job can be given") + "auto (the default, from in the config) keeps them only when the estimate fits in memory_retain_auto_fraction of what the job can be given") parser.add_argument("--chromInfo", help="Two-column file mapping genome (col 1) to comma-separated list of sex chromosomes. This information " "will be used to guide outgroup selection so that, where possible, all chromosomes are present in" @@ -195,7 +226,7 @@ def main(): results_dict = toil.restart() else: align_jobs = make_batch_align_jobs(options, toil) - results_dict = toil.start(Job.wrapJobFn(batch_align_jobs, align_jobs)) + results_dict = toil.start(Job.wrapJobFn(batch_align_jobs, align_jobs, walltime=cactus_walltime())) # when using s3 output urls, things get checkpointed as they're made so no reason to export # todo: make a more unified interface throughout cactus for this @@ -406,21 +437,28 @@ def make_align_job(options, toil, config_wrapper=None, chrom_name=None): do_filter_paf=options.pangenome, chrom_name=chrom_name, scores_id=scores_id, - branch_scale=options.branchScale) + branch_scale=options.branchScale, walltime=cactus_walltime()) return align_job def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, paf_id, paf_path, root_name, og_map, checkpointInfo, doVG, doGFA, delay=0, referenceEvents=None, pafMaskFilter=None, paf2Stable=False, cons_cores = None, cons_memory = None, cons_retain_pages = None, do_filter_paf=False, chrom_name=None, scores_id=None, branch_scale=1.0): - head_job = Job() + head_job = Job(walltime=cactus_walltime()) job.addChild(head_job) event_list = input_seq_id_map.keys() - # unzip the PAF if it's gzipped + # unzip the PAF if it's gzipped. Hold on to the size the alignment will actually have once + # unzipped: paf_id becomes a promise below, and a promise has no .size, so everything after + # this point has to size itself from paf_size instead. (filter_paf's disk and memory were + # already reading paf_id.size past this rebinding, so a gzipped PAF with --pangenome raised + # an AttributeError before it ever got as far as scheduling the job.) + paf_size = paf_id.size if paf_path and paf_path.endswith('.gz'): - unzip_job = head_job.addChildJobFn(unzip_gz, paf_path, paf_id, disk=10*paf_id.size) + unzip_job = head_job.addChildJobFn(unzip_gz, paf_path, paf_id, disk=10*paf_id.size, + walltime=unzip_gz_walltime(paf_id.size)) paf_id = unzip_job.rv() + paf_size = int(paf_size * PAF_GZIP_RATIO) # parse the scores file into the config if scores_id: @@ -430,7 +468,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, apply_scores_to_config(score_dict, config_wrapper.xmlRoot) # unzip the input sequences and enforce unique header prefixes - sanitize_job = head_job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, pangenome=doVG or doGFA or do_filter_paf) + sanitize_job = head_job.addChildJobFn(sanitize_fasta_headers, input_seq_id_map, pangenome=doVG or doGFA or do_filter_paf, walltime=cactus_walltime()) new_seq_id_map = sanitize_job.rv() # run pangenome-specific paf filter. this also runs in cactus-graphmap-split, but that stage is @@ -446,10 +484,12 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # anyway. that is the common path, not just --noSplit. referenceEvents[0] matches the "first # reference" convention cactus-graphmap-split already normalises to. if do_filter_paf: - paf_filter_mem = max(paf_id.size * 10, 2**32) + paf_filter_mem = max(paf_size * 10, 2**32) paf_filter_job = head_job.addChildJobFn(filter_paf, paf_id, config_wrapper, reference=referenceEvents[0] if referenceEvents else None, - disk = paf_id.size * 10, memory=paf_filter_mem) + disk = paf_size * 10, memory=paf_filter_mem, + walltime=cactus_walltime(FILTER_PAF_SECS_PER_GB * paf_size / 1e9, + io_bytes=2 * paf_size)) paf_id = paf_filter_job.rv() # apply tree scaling to reflect branch scaling and/or uncertainty in ancestor placement/sequences @@ -468,7 +508,7 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # run consolidated cons_job = head_job.addFollowOnJobFn(cactus_cons_with_resources, spanning_tree, root_name, config_wrapper.xmlRoot, new_seq_id_map, og_map, paf_id, cons_cores = cons_cores, cons_memory=cons_memory, chrom_name=chrom_name, - cons_retain_pages=cons_retain_pages) + cons_retain_pages=cons_retain_pages, walltime=cactus_walltime()) results = {root_name : (cons_job.rv(1), cons_job.rv(2))} # get the immediate subtree (which is all export_hal can use) @@ -476,18 +516,23 @@ def cactus_align(job, config_wrapper, mc_tree, input_seq_map, input_seq_id_map, # run the hal export allow_collapse = getOptionalAttrib(findRequiredNode(config_wrapper.xmlRoot, "graphmap"), "collapse", typeFn=str, default="none") in ['reference', 'all'] + # export_hal called without has_resources stages no file: it resolves the c2h/fasta promises + # into sizes and re-dispatches itself with them, so this pass is coordination. halAppendCactusSubtree + # runs in the resourced pass, which sizes its own walltime. hal_job = cons_job.addFollowOnJobFn(export_hal, sub_tree, config_wrapper.xmlRoot, new_seq_id_map, og_map, results, event=root_name, inMemory=True, checkpointInfo=checkpointInfo, acyclicEvent=referenceEvents[0] if referenceEvents and not allow_collapse else None, - memory_override=cons_memory) + walltime=cactus_walltime()) # clean out some of the intermediate jobstore files - hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id]) + hal_job.addFollowOnJobFn(clean_jobstore_files, file_id_maps=[new_seq_id_map], file_ids=[paf_id], walltime=cactus_walltime()) # optionally create the VG if doVG or doGFA: + # as with export_hal above, the pass without a resource_spec only sizes the HAL promise and + # re-dispatches itself; hal2vg runs in the second pass vg_export_job = hal_job.addFollowOnJobFn(export_vg, hal_job.rv(), config_wrapper, doVG, doGFA, referenceEvents, checkpointInfo=checkpointInfo, memory_override=cons_memory, - vg_tag=RAW_VG_SUFFIX if chrom_name else '') + vg_tag=RAW_VG_SUFFIX if chrom_name else '', walltime=cactus_walltime()) vg_file_id, gfa_file_id = vg_export_job.rv(0), vg_export_job.rv(1) else: vg_file_id, gfa_file_id = None, None @@ -514,7 +559,9 @@ def export_vg(job, hal_id, config_wrapper, doVG, doGFA, referenceEvents, checkpo return job.addChildJobFn(export_vg, hal_id, config_wrapper, doVG, doGFA, referenceEvents, checkpointInfo, resource_spec = True, vg_tag=vg_tag, disk=hal_id.size * 3, - memory=cactus_clamp_memory(max(vg_memory, min_memory))).rv() + memory=cactus_clamp_memory(max(vg_memory, min_memory)), + walltime=cactus_walltime(HAL2VG_SECS_PER_GB * hal_id.size / 1e9, + io_bytes=3 * hal_id.size)).rv() work_dir = job.fileStore.getLocalTempDir() hal_path = os.path.join(work_dir, "out.hal") @@ -570,6 +617,7 @@ def main_batch(): cons: less efficient use of resources """ parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("chromFile", help = "chroms file") parser.add_argument("outHal", type=str, help = "Output directory (can be s3://)") @@ -654,7 +702,7 @@ def main_batch(): chrom_dict[chrom] = toil.importFile(makeURL(seqfile)), toil.importFile(makeURL(alnFile)) if chrom in options.configOverrides: options.configOverrides[chrom][1] = toil.importFile(makeURL(options.configOverrides[chrom][0])) - results_dict = toil.start(Job.wrapJobFn(align_toil_batch, chrom_dict, config_id, options)) + results_dict = toil.start(Job.wrapJobFn(align_toil_batch, chrom_dict, config_id, options, walltime=cactus_walltime())) # when using s3 output urls, things get checkpointed as they're made so no reason to export # todo: make a more unified interface throughout cactus for this @@ -684,12 +732,26 @@ def align_toil_batch(job, chrom_dict, config_id, options): for chrom in chrom_dict.keys(): seq_file_id, paf_file_id = chrom_dict[chrom] # apply the overrides from --alignCoresOverrides and --configOverrides - options.aligCores = options.alignCoresOverrides[chrom] if chrom in options.alignCoresOverrides else orig_cores + options.alignCores = options.alignCoresOverrides[chrom] if chrom in options.alignCoresOverrides else orig_cores options.configFile = options.configOverrides[chrom][0] if chrom in options.configOverrides else orig_config config_id = options.configOverrides[chrom][1] if chrom in options.configOverrides else orig_config_id # spawn the chromosome job + # this job hosts an entire nested cactus-align on one node, so its walltime covers that whole + # pipeline -- consolidated, the HAL export, optionally hal2vg -- run one after another with no + # cluster to fan out onto. Only the paf size is in scope (seq_file_id is the seqfile *text*, + # a few hundred bytes), so the estimate keys off it, with the exponent of the + # runtime fit and a coefficient raised over it for the export stages the nested run also pays + # for. The nested run is handed --consCores, so it takes the same core scaling. + # ALIGN_TOIL_STARTUP_SECS is nested-Toil startup only -- deliberately not a floor for the + # work, which would hand an evolver-sized test PAF hours it cannot use. It is the roughest + # estimate here: the seq:paf ratio it assumes is 8x in VGP but 43x in the HPRC pangenome. align_job = job.addChildJobFn(align_toil, chrom, seq_file_id, paf_file_id, config_id, options, - cores=options.alignCores) + cores=options.alignCores, + walltime=cactus_walltime( + ALIGN_TOIL_STARTUP_SECS + + ALIGN_TOIL_SECS_PER_GB * (paf_file_id.size / 1e9) ** ALIGN_TOIL_EXPONENT + * cons_core_scale(options.alignCores), + io_bytes=4 * paf_file_id.size)) results_dict[chrom] = align_job.rv() return results_dict diff --git a/src/cactus/shared/common.py b/src/cactus/shared/common.py index 6129097d7..d1b595a00 100644 --- a/src/cactus/shared/common.py +++ b/src/cactus/shared/common.py @@ -125,6 +125,42 @@ def cactus_slurm_max_memory(options): # node's full RealMemory can otherwise sit pending forever. return int(max_mb * 1024 * 1024 * 0.95) +def cactus_slurm_max_walltime(options): + """ Query the Slurm cluster for the longest walltime any reachable partition will accept, + in seconds, or None if it can't be determined (or if some partition takes unlimited jobs). + + This is the walltime analogue of cactus_slurm_max_memory, and it matters more than it looks: + when Toil cannot find a partition that fits a job's walltime it does not fall back to a + default, it raises, which kills the workflow at submission time. So we clamp every walltime + estimate to what the cluster can actually run, exactly as we clamp memory to what the biggest + node can actually provide. + + As with memory, we narrow to the partition jobs will actually land on when --slurmPartition + pins one. """ + try: + from toil.batchSystems.slurm import SlurmBatchSystem + partition_set = SlurmBatchSystem.PartitionSet() + partitions = partition_set.all_partitions + except Exception as e: + logger.warning('Unable to query Slurm for partition time limits ({}); not clamping walltime'.format(e)) + return None + if not partitions: + return None + + target_partition = getattr(options, 'slurm_partition', None) + if target_partition: + limits = [p.time_limit for p in partitions if p.partition_name == target_partition] + else: + limits = [p.time_limit for p in partitions] + limits = [t for t in limits if t and t > 0] + if not limits: + return None + max_time = max(limits) + if math.isinf(max_time): + # some partition takes unlimited jobs, so there is nothing to clamp to + return None + return int(max_time) + def cactus_override_toil_options(options): """ Mess with some toil options to create useful defaults. """ if options.retryCount is None and options.batchSystem.lower() not in ['single_machine', 'singleMachine']: @@ -177,6 +213,31 @@ def cactus_override_toil_options(options): os.environ['CACTUS_MAX_MEMORY'] = str(max_mem) os.environ['CACTUS_DEFAULT_MEMORY'] = str(human2bytes(str(options.defaultMemory)) if options.defaultMemory else 2**31) + # store the walltime knobs here so cactus_walltime() can get at them without carrying + # options around, and so they propagate to workers just like the memory limits above + os.environ['CACTUS_WALLTIME_FACTOR'] = str(getattr(options, 'walltimeFactor', WALLTIME_FACTOR)) + os.environ['CACTUS_MIN_WALLTIME'] = str(getattr(options, 'minWalltime', WALLTIME_MIN)) + max_walltime = getattr(options, 'maxWalltime', 0) or 0 + if not max_walltime and options.batchSystem.lower() == 'slurm': + # no partition can run a job for longer than its time limit, and Toil raises (rather + # than falling back to a default) when it can't find one that fits, so cap our + # estimates at what the cluster will actually accept + slurm_max_walltime = cactus_slurm_max_walltime(options) + if slurm_max_walltime: + max_walltime = slurm_max_walltime + if getattr(options, 'doubleTime', False): + # ...and that raise happens on the batch system's own thread, which takes the + # whole workflow down with it, not just the job. --doubleTime retries a job + # Slurm killed for running over with twice the walltime, so an estimate sitting + # at the longest partition's limit turns its own first retry into exactly that + # crash. Keep half the ceiling in reserve so the retry still has somewhere to + # go; a job that needs more than half the longest partition was going to end up + # there regardless, and only pays for one attempt on the way. + max_walltime = int(max_walltime / WALLTIME_DOUBLETIME_RESERVE) + logger.info('Clamping maximum job walltime to {} seconds (from the longest Slurm partition time limit of {})'.format( + max_walltime, slurm_max_walltime)) + os.environ['CACTUS_MAX_WALLTIME'] = str(int(max_walltime)) + # auto-set cactus_log_memory try: subprocess.check_call(['/usr/bin/time', '-v', 'ls'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -188,6 +249,112 @@ def cactus_clamp_memory(memory_bytes): """ use the environment variables from --maxMemory and --defaultMemory to clamp a given memory value """ return max(min(int(os.environ['CACTUS_MAX_MEMORY']), int(memory_bytes)), int(os.environ['CACTUS_DEFAULT_MEMORY'])) +# --------------------------------------------------------------------------- +# Per-job walltime +# --------------------------------------------------------------------------- +# Slurm picks each job's partition from its time limit, so one global walltime +# (--slurmTime 100:00:00) drops every job -- including the many thousands that finish in +# seconds -- into the slowest queue. So Cactus estimates a walltime per job, the same way +# it estimates memory, and turns on Toil's --doubleTime so a job that overruns is retried +# with twice the time instead of failing the run. +# +# Call sites pass their best estimate of how long the job actually takes. The safety +# margin (--walltimeFactor) and the [--minWalltime, --maxWalltime] bounds are applied here, +# centrally, so there is one place to tune and one place for a cluster-specific override. + +# Estimated runtime, in seconds, of a job that only schedules other jobs and stages no +# large file. These finish in well under a second; the number is worker startup overhead +# (and, with --binariesMode singularity, cooking the image). This is the default, so a +# bare cactus_walltime() is the right call for a coordination job. +WALLTIME_COORDINATION = 120 + +# Assumed jobstore throughput, in bytes/second, for the io_bytes term. Staging is what makes an +# otherwise trivial job slow, and a shared cluster filesystem with a few hundred concurrent +# Cactus jobs on it goes nowhere near its headline number: in the first at-scale Slurm run with +# walltimes on (HPRC, 519 concurrent sanitize_fasta_header jobs) the ones that overran were +# spending upwards of 900 s moving ~4 GB each, while their own commands took 240 s. 100 MiB/s +# was the first guess and it was an order of magnitude out under that load. +WALLTIME_IO_RATE = 25 * 1024**2 + +# Multiplier applied to every estimate, overridable with --walltimeFactor. The estimates +# at the call sites aim at roughly the p99 of what we have measured on the largest runs we +# have logs for, so this is headroom on top of that. +WALLTIME_FACTOR = 2.5 + +# Floor for any walltime request, overridable with --minWalltime. A short partition is an hour +# on the clusters this runs on, so every request below that routes the same way and precision +# under it buys nothing -- while a request that is too short by a minute costs a whole +# --doubleTime retry. 600 was too tight for exactly that reason: 60 of the 519 +# sanitize_fasta_header jobs of the run above were killed at ~620 s, and every one then +# succeeded when --doubleTime handed it ~1250 s. Half the short partition leaves room for the +# staging tail that no size model sees, and still lands in it. +WALLTIME_MIN = 1800 + +# How much of the longest Slurm partition to keep in reserve for --doubleTime to retry into. +# 2 leaves room for exactly one doubling; see cactus_override_toil_options. +WALLTIME_DOUBLETIME_RESERVE = 2 + +def cactus_walltime(seconds=WALLTIME_COORDINATION, io_bytes=0): + """ Turn an estimate of how long a job takes into the walltime to request for it. + + seconds: estimated compute time, in seconds. Defaults to the coordination tier, so a + bare cactus_walltime() is what a job that only schedules other jobs wants. + Pass None to say the estimate is *unknown* -- that returns None, leaving the + job on Toil's --defaultWalltime rather than guessing. A job whose cost is all + I/O should pass 0 and use io_bytes. + io_bytes: bytes the job stages in and out of the jobstore, which on a busy shared + filesystem routinely dwarfs the compute. Pass real sizes here (a Toil + FileID has .size); a promise has no size at scheduling time, so where the + inputs are promises fold the I/O into a constant `seconds` instead. + + Returns None -- meaning no per-job walltime, so the job falls back to Toil's + --defaultWalltime -- when estimation is turned off with --walltimeFactor 0. + + Mirrors cactus_clamp_memory: the value travels to workers through the environment (see + cactus_override_toil_options), so this works on the leader and in a worker alike. """ + if seconds is None: + return None + factor = float(os.environ.get('CACTUS_WALLTIME_FACTOR', WALLTIME_FACTOR)) + if factor <= 0: + return None + estimate = float(seconds) + if io_bytes: + estimate += float(io_bytes) / WALLTIME_IO_RATE + estimate *= factor + estimate = max(estimate, float(os.environ.get('CACTUS_MIN_WALLTIME', WALLTIME_MIN))) + max_walltime = float(os.environ.get('CACTUS_MAX_WALLTIME', 0)) + if max_walltime > 0: + estimate = min(estimate, max_walltime) + return int(math.ceil(estimate)) + +def add_cactus_toil_options(parser): + """ Add cactus-specific options on top of Toil's default argument parser + (Job.Runner.getDefaultArgumentParser). Call this right after creating the parser. """ + parser.add_argument("--walltimeFactor", type=float, default=WALLTIME_FACTOR, + help="Safety multiplier applied to every per-job walltime Cactus estimates " + "[default: {}]. Raise it if jobs are being killed for running over " + "(a slower cluster than the estimates were tuned on); pass 0 to switch " + "per-job walltimes off entirely, leaving every job on Toil's " + "--defaultWalltime.".format(WALLTIME_FACTOR)) + parser.add_argument("--minWalltime", type=int, default=WALLTIME_MIN, + help="Floor, in seconds, for any walltime Cactus requests [default: {}]. " + "Cactus schedules many jobs that finish in under a second; this keeps " + "worker startup and jobstore I/O from pushing them over their " + "limit.".format(WALLTIME_MIN)) + parser.add_argument("--maxWalltime", type=int, default=0, + help="Ceiling, in seconds, for any walltime Cactus requests [default: on " + "Slurm, the longest time limit of any partition the jobs could land " + "on; otherwise none]. This matters because Toil raises, rather than " + "falling back to a default, when no partition can fit a job's " + "walltime.") + + # --doubleTime is what makes tight per-job walltimes safe: a job Slurm kills for running + # over is retried with twice the time rather than failing the run. Default it on the way + # retryCount is defaulted up, while still letting "--doubleTime false" win. Guarded + # because it only exists in Toil newer than 9.5.0 (see toil-requirement.txt). + if any('--doubleTime' in (action.option_strings or []) for action in parser._actions): + parser.set_defaults(doubleTime=True) + def makeURL(path_or_url): if urlparse(path_or_url).scheme == '': return "file://" + os.path.abspath(path_or_url) @@ -1089,7 +1256,7 @@ class RoundedJob(Job): # Default rounding amount: 100 MiB roundingAmount = 100*1024*1024 def __init__(self, memory=None, cores=None, disk=None, preemptable=None, - unitName=None, checkpoint=False, accelerators=None): + unitName=None, checkpoint=False, accelerators=None, walltime=None): if memory is not None: memory = self.roundUp(memory) if disk is not None: @@ -1099,7 +1266,8 @@ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, disk = 1500*1024*1024 + self.roundUp(disk) super(RoundedJob, self).__init__(memory=memory, cores=cores, disk=disk, preemptable=preemptable, unitName=unitName, - checkpoint=checkpoint, accelerators=accelerators) + checkpoint=checkpoint, accelerators=accelerators, + walltime=walltime) def roundUp(self, bytesRequirement): """ @@ -1149,12 +1317,12 @@ class ChildTreeJob(RoundedJob): slightly, but reducing the wall-clock time taken dramatically. """ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, - unitName=None, checkpoint=False, maxChildrenPerJob=20): + unitName=None, checkpoint=False, maxChildrenPerJob=20, walltime=None): self.queuedChildJobs = [] self.maxChildrenPerJob = maxChildrenPerJob super(ChildTreeJob, self).__init__(memory=memory, cores=cores, disk=disk, preemptable=preemptable, unitName=unitName, - checkpoint=checkpoint) + checkpoint=checkpoint, walltime=walltime) def addChild(self, job): self.queuedChildJobs.append(job) @@ -1234,6 +1402,25 @@ def enableDumpStack(sig=signal.SIGUSR1): """enable dumping stacks when the specified signal is received""" signal.signal(sig, dumpStacksHandler) +# Single-threaded gzip/bgzip, in bytes of *input* per second. Compression measured at 21-27 +# MB/s on the HPRC PAFs: 1727 s to gzip the 34.4 GiB whole-panel PAF and 868 s to bgzip its +# GAF, against medians of 79 s and 50 s per chromosome. Decompression at ~38 MB/s of +# compressed input, over the 576 VGP alignments' gzipped PAFs. Both are rates over whatever +# gzip is handed, so the compressed side of each pair is the smaller number of bytes. +GZIP_COMPRESS_BYTES_PER_SEC = 21e6 +GZIP_DECOMPRESS_BYTES_PER_SEC = 38e6 + +def unzip_gz_walltime(compressed_bytes): + """ walltime for an unzip_gz job on a file of compressed_bytes. It writes the decompressed + file back to the jobstore, which for a PAF or fasta is roughly 3x what it read. """ + return cactus_walltime(compressed_bytes / GZIP_DECOMPRESS_BYTES_PER_SEC, + io_bytes=4 * compressed_bytes) + +def zip_gz_walltime(uncompressed_bytes): + """ walltime for a zip_gz job on a file of uncompressed_bytes """ + return cactus_walltime(uncompressed_bytes / GZIP_COMPRESS_BYTES_PER_SEC, + io_bytes=2 * uncompressed_bytes) + def unzip_gzs(job, input_paths, input_ids, delete_original=True): """ go through a list of files and unzip any that end with .gz and return a list of updated ids. files that don't end in .gz are just passed through. relying on the extension @@ -1242,7 +1429,8 @@ def unzip_gzs(job, input_paths, input_ids, delete_original=True): for input_path, input_id in zip(input_paths, input_ids): if input_path.endswith('.gz'): unzip_job = job.addChildJobFn(unzip_gz, input_path, input_id, delete_original=delete_original, - disk=10*input_id.size) + disk=10*input_id.size, + walltime=unzip_gz_walltime(input_id.size)) unzipped_ids.append(unzip_job.rv()) else: unzipped_ids.append(input_id) @@ -1271,13 +1459,15 @@ def zip_gzs(job, input_paths, input_ids, list_elems = None, delete_original=True for i, elem in enumerate(input_list): if not list_elems or i in list_elems: output_list.append(job.addChildJobFn(zip_gz, input_path, elem, delete_original=delete_original, - disk=2*elem.size).rv()) + disk=2*elem.size, + walltime=zip_gz_walltime(elem.size)).rv()) else: output_list.append(elem) zipped_ids.append(output_list) else: zipped_ids.append(job.addChildJobFn(zip_gz, input_path, input_list, delete_original=delete_original, - disk=2*input_list.size).rv()) + disk=2*input_list.size, + walltime=zip_gz_walltime(input_list.size)).rv()) else: zipped_ids.append(input_list) return zipped_ids diff --git a/src/cactus/update/cactus_hal2seqfile.py b/src/cactus/update/cactus_hal2seqfile.py index f443a4175..f5bc8815b 100644 --- a/src/cactus/update/cactus_hal2seqfile.py +++ b/src/cactus/update/cactus_hal2seqfile.py @@ -30,9 +30,9 @@ from toil.statsAndLogging import logger from toil.statsAndLogging import set_logging_from_options -from cactus.shared.common import setupBinaries, importSingularityImage +from cactus.shared.common import setupBinaries, importSingularityImage, cactus_walltime from cactus.shared.common import enableDumpStack -from cactus.shared.common import cactus_override_toil_options +from cactus.shared.common import cactus_override_toil_options, add_cactus_toil_options from cactus.shared.common import makeURL, cactus_call, cactus_clamp_memory from cactus.shared.version import cactus_commit @@ -90,16 +90,30 @@ def to_newick(node_id): return to_newick(root_id) + ';', genome_names(root_id) +# Seconds of hal2fasta|bgzip per Gb of extracted genome. Across the 576 per-ancestor exports +# of the VGP 577-way (genomes of 0.007 to 4.3 Gb) the p99 was 57 s/Gb and the worst 70 s/Gb. +HAL2FASTA_SECS_PER_GB = 60 + + def export_subtree_fastas(job, hal_id, hal_name, genomes, lengths): """Fan out one hal2fasta|bgzip job per genome (run in parallel by Toil). Returns a {genome: fasta file id} map.""" fa_ids = {} for genome in genomes: - # the HAL is read by symlink (below), so each job only needs disk for its own FASTA - length = lengths.get(genome, hal_id.size) + # the HAL is read by symlink (below), so each job only needs disk for its own FASTA. + # A genome missing from `lengths` falls back to the whole HAL for disk, which is merely + # wasteful, but as a walltime it would ask for the entire alignment's worth of time for + # one genome -- so an unknown length gets no estimate at all instead. + length = lengths.get(genome) + disk_length = length if length is not None else hal_id.size fa_ids[genome] = job.addChildJobFn(hal2fasta_gz, hal_id, hal_name, genome, memory=cactus_clamp_memory(3000000000), - disk=max(int(length * 2), 2**20)).rv() + disk=max(int(disk_length * 2), 2**20), + # only the bgzipped FASTA is written back; the HAL + # arrives by symlink and is not staged + walltime=cactus_walltime(HAL2FASTA_SECS_PER_GB * length / 1e9, + io_bytes=length // 4) + if length is not None else cactus_walltime(None)).rv() return fa_ids @@ -117,6 +131,7 @@ def hal2fasta_gz(job, hal_id, hal_name, genome): def main(): parser = Job.Runner.getDefaultArgumentParser() + add_cactus_toil_options(parser) parser.add_argument("halFile", help="input HAL alignment") parser.add_argument("outDir", help="output directory for the gzipped per-genome FASTAs (created if needed)") @@ -170,7 +185,7 @@ def main(): # defaults to symlink=True in Toil, but we set it explicitly so this holds within the tool hal_id = toil.importFile(makeURL(options.halFile), symlink=True) fa_ids = toil.start(Job.wrapJobFn(export_subtree_fastas, hal_id, - os.path.basename(options.halFile), genomes, lengths)) + os.path.basename(options.halFile), genomes, lengths, walltime=cactus_walltime())) # export each gzipped fasta to /.fa.gz for genome, fa_id in fa_ids.items(): diff --git a/toil-requirement.txt b/toil-requirement.txt index 30248feae..e4deff228 100644 --- a/toil-requirement.txt +++ b/toil-requirement.txt @@ -1,2 +1,6 @@ backports.zoneinfo[tzdata];python_version<"3.9" -toil[aws]==9.5.0 +# Pinned to Toil master for the per-job walltime requirement and --doubleTime, neither of which +# is in a release yet (9.5.0 lacks both). +# a9681fe0 = "Add `--doubleTime` argument (#5576)". +# TODO: revert to a released version (toil[aws]==) once these land in a Toil release. +toil[aws] @ git+https://github.com/DataBiosphere/toil.git@a9681fe077c15fff10f583478a8cb7ecf444c278