diff --git a/.github/workflows/mcstas-conda-testsuite.yml b/.github/workflows/mcstas-conda-testsuite.yml index d1663a08b..d6368105b 100644 --- a/.github/workflows/mcstas-conda-testsuite.yml +++ b/.github/workflows/mcstas-conda-testsuite.yml @@ -133,3 +133,119 @@ jobs: if: always() && (inputs.manual-debugging == true) with: limit-access-to-actor: true + + compare: + name: mcstas-conda-testsuite-comparison + needs: build + if: always() + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + path: src + + - name: Get conda dependency list + id: conda-deps + run: | + ./src/devel/bin/mccode-create-conda-yml -m mcstas -n mcstas -o dependencies.yml + + - uses: mamba-org/setup-micromamba@v3 + with: + environment-name: mcstas + environment-file: dependencies.yml + + - name: Configure build and install mcstas + id: mcstas-install + run: | + # Only the Python tooling (mcviewtest, mcplotdiff-html, mcplot-html) is + # actually needed here - the diff/report generation is platform-agnostic + # regardless of which OS originally produced the monitor data - but we + # reuse the exact same install path as the build job for consistency, + # rather than maintaining a second, lighter-weight install mechanism. + ./src/devel/bin/mccode-build-conda -m mcstas -s $PWD/src -b $PWD/build_mcstas + + - name: 'Download all platform test artifacts' + id: download + uses: actions/download-artifact@v6 + with: + # matches mcstas-artefacts-ubuntu-latest.openmpi.conda-testsuite, + # mcstas-artefacts-macos-latest.openmpi.conda-testsuite, + # mcstas-artefacts-windows-latest.msmpi.conda-testsuite - by pattern + # rather than hardcoded names, so this keeps working if the build + # matrix gains/loses a platform + pattern: mcstas-artefacts-*.conda-testsuite + path: downloaded_artifacts + + - name: 'Merge platform columns into one shared testdir' + id: merge + run: | + set -e + set -u + set -x + + mkdir combined + REFLABEL="" + + # Each platform's tarball contains exactly one label directory + # directly under run_test-suite/ (e.g. + # mcstas-3.99.99_mpi_x_2_ubuntu-latest_openmpi_1e6_Linux_). + # mcviewtest treats each such directory as one comparison "column", + # so merging them as siblings under one shared testdir is all that's + # needed to make them comparable. + for tgz in downloaded_artifacts/*/*.tgz; do + work=$(mktemp -d) + tar xzf "$tgz" -C "$work" + labeldir=$(ls -d "$work"/run_test-suite/*/) + mv "$labeldir" combined/ + + # Linux (ubuntu-latest) is the reference platform; the label + # name itself embeds a version string and timestamp we can't + # predict, so identify it via the source artifact/tarball name + # instead of hardcoding it. + if [[ "$tgz" == *"ubuntu-latest"* ]]; then + REFLABEL=$(basename "$labeldir") + fi + rm -rf "$work" + done + + if [ -z "$REFLABEL" ]; then + echo "ERROR: could not identify the Linux/reference column (ubuntu-latest artifact missing?)" + exit 1 + fi + + echo "REFLABEL=$REFLABEL" >> "$GITHUB_ENV" + echo "Reference column: $REFLABEL" + ls combined + + - name: 'Run mcviewtest across all platform columns' + id: viewtest + run: | + set -e + set -u + set -x + mcviewtest --testdir "$PWD/combined" --reflabel "$REFLABEL" --nobrowse + + - name: 'Tar combined output (all platform folders + comparison table)' + id: tar-package + if: always() + run: | + tar cvfz mcstas-conda-testsuite-comparison.tgz combined + + - name: 'Upload comparison artifact' + id: comparison-upload + uses: actions/upload-artifact@v6 + if: always() + with: + name: mcstas-testsuite-comparison + path: mcstas-conda-testsuite-comparison.tgz + + - name: Setup tmate session for manual debugging + uses: mxschmitt/action-tmate@v3 + if: always() && (inputs.manual-debugging == true) + with: + limit-access-to-actor: true diff --git a/.github/workflows/mcxtrace-conda-testsuite.yml b/.github/workflows/mcxtrace-conda-testsuite.yml index b1d90060c..bbc0da7ca 100644 --- a/.github/workflows/mcxtrace-conda-testsuite.yml +++ b/.github/workflows/mcxtrace-conda-testsuite.yml @@ -133,3 +133,119 @@ jobs: if: always() && (inputs.manual-debugging == true) with: limit-access-to-actor: true + + compare: + name: mcxtrace-conda-testsuite-comparison + needs: build + if: always() + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + path: src + + - name: Get conda dependency list + id: conda-deps + run: | + ./src/devel/bin/mccode-create-conda-yml -m mcxtrace -n mcxtrace -o dependencies.yml + + - uses: mamba-org/setup-micromamba@v3 + with: + environment-file: dependencies.yml + environment-name: mcxtrace + + - name: Configure build and install mcxtrace + id: mcxtrace-install + run: | + # Only the Python tooling (mxviewtest, mxplotdiff-html, mxplot-html) is + # actually needed here - the diff/report generation is platform-agnostic + # regardless of which OS originally produced the monitor data - but we + # reuse the exact same install path as the build job for consistency, + # rather than maintaining a second, lighter-weight install mechanism. + ./src/devel/bin/mccode-build-conda -m mcxtrace -s $PWD/src -b $PWD/build_mcxtrace + + - name: 'Download all platform test artifacts' + id: download + uses: actions/download-artifact@v6 + with: + # matches mcxtrace-artefacts-ubuntu-latest.openmpi.conda-testsuite, + # mcxtrace-artefacts-macos-latest.openmpi.conda-testsuite, + # mcxtrace-artefacts-windows-latest.msmpi.conda-testsuite - by pattern + # rather than hardcoded names, so this keeps working if the build + # matrix gains/loses a platform + pattern: mcxtrace-artefacts-*.conda-testsuite + path: downloaded_artifacts + + - name: 'Merge platform columns into one shared testdir' + id: merge + run: | + set -e + set -u + set -x + + mkdir combined + REFLABEL="" + + # Each platform's tarball contains exactly one label directory + # directly under run_test-suite/ (e.g. + # mcxtrace-1.99.99_mpi_x_2_ubuntu-latest_openmpi_1e6_Linux_). + # mxviewtest treats each such directory as one comparison "column", + # so merging them as siblings under one shared testdir is all that's + # needed to make them comparable. + for tgz in downloaded_artifacts/*/*.tgz; do + work=$(mktemp -d) + tar xzf "$tgz" -C "$work" + labeldir=$(ls -d "$work"/run_test-suite/*/) + mv "$labeldir" combined/ + + # Linux (ubuntu-latest) is the reference platform; the label + # name itself embeds a version string and timestamp we can't + # predict, so identify it via the source artifact/tarball name + # instead of hardcoding it. + if [[ "$tgz" == *"ubuntu-latest"* ]]; then + REFLABEL=$(basename "$labeldir") + fi + rm -rf "$work" + done + + if [ -z "$REFLABEL" ]; then + echo "ERROR: could not identify the Linux/reference column (ubuntu-latest artifact missing?)" + exit 1 + fi + + echo "REFLABEL=$REFLABEL" >> "$GITHUB_ENV" + echo "Reference column: $REFLABEL" + ls combined + + - name: 'Run mxviewtest across all platform columns' + id: viewtest + run: | + set -e + set -u + set -x + mxviewtest --testdir "$PWD/combined" --reflabel "$REFLABEL" --nobrowse + + - name: 'Tar combined output (all platform folders + comparison table)' + id: tar-package + if: always() + run: | + tar cvfz mcxtrace-conda-testsuite-comparison.tgz combined + + - name: 'Upload comparison artifact' + id: comparison-upload + uses: actions/upload-artifact@v6 + if: always() + with: + name: mcxtrace-testsuite-comparison + path: mcxtrace-conda-testsuite-comparison.tgz + + - name: Setup tmate session for manual debugging + uses: mxschmitt/action-tmate@v3 + if: always() && (inputs.manual-debugging == true) + with: + limit-access-to-actor: true diff --git a/tools/Python/mccodelib/mcplotloader.py b/tools/Python/mccodelib/mcplotloader.py index fe90a305f..e8b35115e 100644 --- a/tools/Python/mccodelib/mcplotloader.py +++ b/tools/Python/mccodelib/mcplotloader.py @@ -5,6 +5,7 @@ import glob import re import os +import numpy as np from os.path import isfile, isdir, join, dirname, basename, splitext, exists from os import walk from decimal import Decimal @@ -13,6 +14,47 @@ from .plotgraph import PlotGraphPrint, DataHandle, PNMultiple, PNSingle +def _parse_data_block(lines_block): + ''' Fast, vectorized parser for a block of whitespace-separated numeric + data lines (e.g. the "# Data" or "# Events" section of a 2D monitor, + or the data rows of a 1D monitor), returning a list of rows (each a + list of floats). + + This replaces what used to be a per-line, per-value Python float() + loop in _parse_1D_monitor()/_parse_2D_monitor() - for a 2D monitor + that's one Python-level float() call per grid cell (e.g. 90000 for a + 300x300 PSD monitor, doubled if an Events section is present), which + dominates load time for instruments with sizeable 2D monitors. + np.fromstring(..., sep=' ') parses the whole block in one call + instead of looping in Python. + + Falls back to the slower, more permissive line-by-line parser (which + silently skips any line that fails to parse, matching the historical + behaviour) if the fast path doesn't cleanly apply - e.g. a genuinely + malformed/truncated file, blank lines within the block, or a jagged + block with inconsistent column counts between rows. This keeps the + common case fast without sacrificing correctness on the edge cases + the original loop happened to tolerate. ''' + if not lines_block: + return [] + try: + ncols = len(lines_block[0].split()) + if ncols == 0: + raise ValueError('empty first line in data block') + flat = np.fromstring(' '.join(lines_block), dtype=float, sep=' ') + if flat.size != ncols * len(lines_block): + raise ValueError('unexpected element count (jagged or malformed data block)') + return flat.reshape(len(lines_block), ncols).tolist() + except Exception: + rows = [] + for l in lines_block: + try: + rows.append([float(item) for item in l.strip().split()]) + except Exception: + pass + return rows + + ''' McCode simulation output data types. ''' @@ -193,24 +235,13 @@ def _parse_1D_monitor(text): # load the actual data lines = text.splitlines() - xvals = [] - yvals = [] - y_err_vals = [] - Nvals = [] - for l in lines: - if '#' in l: - continue + data_lines = [l for l in lines if '#' not in l] + rows = _parse_data_block(data_lines) - vals = l.split() - xvals.append(float(vals[0])) - yvals.append(float(vals[1])) - y_err_vals.append(float(vals[2])) - Nvals.append(float(vals[3])) - - data.xvals = xvals - data.yvals = yvals - data.y_err_vals = y_err_vals - data.Nvals = Nvals + data.xvals = [r[0] for r in rows] + data.yvals = [r[1] for r in rows] + data.y_err_vals = [r[2] for r in rows] + data.Nvals = [r[3] for r in rows] except Exception as e: print('Data1D load error.') @@ -279,13 +310,15 @@ def _parse_2D_monitor(text): dat = True datacount=0; events = False + data_lines = [] + events_lines = [] for l in lines: if '# Data ' in l: datacount=datacount+1; # In case we meet mutiple # Data entries, file is probably saved via -USR2 or from # Progress_bar(). In that case, better flush earlier data entries: if (datacount>1): - data.zvals=[] + data_lines=[] dat = True continue @@ -301,18 +334,13 @@ def _parse_2D_monitor(text): continue if dat: - try: - vals = [float(item) for item in l.strip().split()] - data.zvals.append(vals) - except: - pass + data_lines.append(l) if events: - try: - vals = [float(item) for item in l.strip().split()] - data.counts.append(vals) - except: - pass + events_lines.append(l) + + data.zvals = _parse_data_block(data_lines) + data.counts = _parse_data_block(events_lines) except Exception as e: print('Data2D load error.') diff --git a/tools/Python/mcplot/html/mcplot.py b/tools/Python/mcplot/html/mcplot.py index a66694762..0f453a705 100644 --- a/tools/Python/mcplot/html/mcplot.py +++ b/tools/Python/mcplot/html/mcplot.py @@ -10,6 +10,7 @@ import base64 import json import subprocess +import concurrent.futures sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..')) @@ -23,6 +24,18 @@ WIDTH = 1024 HEIGHT = 768 +def get_default_workers(): + ''' Number of per-monitor plot jobs to run in parallel by default: the + number of processors available to this process. Prefers + os.sched_getaffinity(0) (Linux only) over os.cpu_count(), since the + former respects cgroup/taskset CPU restrictions (e.g. inside a + container or batch job allocation) that the latter ignores. ''' + try: + return len(os.sched_getaffinity(0)) + except AttributeError: + # os.sched_getaffinity doesn't exist on macOS/Windows + return os.cpu_count() or 4 + def file_base_name(file_name): if '.' in file_name: separator_index = file_name.index('.') @@ -35,13 +48,14 @@ def path_base_name(path): file_name = os.path.basename(path) return file_base_name(file_name) -def get_html(template_name, params, simfile): +def get_html(template_name, params, simfile, use_logscale): ''' ''' text = open(os.path.join(os.path.dirname(__file__),template_name)).read() text = text.replace("@PARAMS@", params) text = text.replace("@DATAFILE@", simfile) - - logscalestr = "true" if logscale==True else "false" + text = text.replace("@DATALINK@", "Click for ascii data") + + logscalestr = "true" if use_logscale else "false" text = text.replace("@LOGSCALE@", logscalestr) text = text.replace("@LIBPATH@", libpath) return text @@ -220,27 +234,42 @@ def browse(html_filepath): def plotfunc(node, filename=None): ''' plot a plotnode to a html file as an svg-plot using plotfuncs.js and d3.v4.min.js ''' - global logscale if isinstance(filename, list): filename = filename[0] # get data and set file path if type(node) is PNSingle: data = node.getdata_idx(0) - return plotfunc_single(data, filename) + return plotfunc_single(data, filename, use_logscale=logscale) elif type(node) is PNMultiple: - data = node.getdata_lst() - data_lst = data - - f = [] - f_log = [] - count = 0 + data_lst = node.getdata_lst() + + # Each monitor's linear and log-scale pages are fully independent + # (own numpy arrays, own PIL image, own output file - no shared + # matplotlib-style global figure/axes state the way the + # mcplot-matplotlib tool has), so unlike that tool this is safe to + # parallelize directly: build the full list of (monitor, use_log) + # jobs up front and run them through a bounded thread pool, instead + # of the previous sequential loop that flipped a shared global + # `logscale` flag between calls (unsafe to parallelize as-is, since + # concurrent threads would race on that flag). + jobs = [] for dat in data_lst: - logscale = False - f.append(plotfunc_single(dat)) - logscale = True - f_log.append(plotfunc_single(dat)) - count += 1 + jobs.append((dat, False)) + jobs.append((dat, True)) + + def _run_job(job): + dat, use_log = job + return plotfunc_single(dat, None, use_logscale=use_log) + + max_workers = get_default_workers() + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + # executor.map preserves input order regardless of completion + # order, so results[0::2]/[1::2] line up with data_lst as before + results = list(executor.map(_run_job, jobs)) + + f = results[0::2] + f_log = results[1::2] # now create an overview HTML (index.html with \n") outfile.write("\n") outfile.write("
\n") + outfile.write("\n") outfile.write(f"[ {basename} ]\n") if fname_log: basename_log = os.path.basename(fname_log) outfile.write(f"[ {basename_log} ]\n") - + outfile.write("
\n") # links to the pre-existing mcplot-html plots of the two # original monitors, if they were found on disk a_lin = _relhref(entry.get('a_lin'), outdir) @@ -347,16 +365,17 @@ def write_index(outdir, entries, label_a, label_b): b_lin = _relhref(entry.get('b_lin'), outdir) b_log = _relhref(entry.get('b_log'), outdir) if a_lin or a_log or b_lin or b_log: - outfile.write("|\n") + outfile.write("
\n") if a_lin: - outfile.write(f"[ {label_a} ]\n") + outfile.write(f"[ A ]\n") if a_log: - outfile.write(f"[ {label_a} (log) ]\n") + outfile.write(f"[ A (log) ]\n") if b_lin: - outfile.write(f"[ {label_b} ]\n") + outfile.write(f"[ B ]\n") if b_log: - outfile.write(f"[ {label_b} (log) ]\n") - + outfile.write(f"[ B (log) ]\n") + if a_lin or a_log or b_lin or b_log: + outfile.write("\n") outfile.write("
\n") outfile.write("\n") outfile.write("\n") diff --git a/tools/Python/mcplot/html/template_1d.html b/tools/Python/mcplot/html/template_1d.html index e4356acbf..ece1e1cb6 100644 --- a/tools/Python/mcplot/html/template_1d.html +++ b/tools/Python/mcplot/html/template_1d.html @@ -13,6 +13,6 @@ plt.rgstrMouseRClickPlot( () => { console.log("right click"); } );

-Click for ascii data +@DATALINK@ diff --git a/tools/Python/mcplot/html/template_2d.html b/tools/Python/mcplot/html/template_2d.html index 217fcaf6f..fdd0a7fbf 100644 --- a/tools/Python/mcplot/html/template_2d.html +++ b/tools/Python/mcplot/html/template_2d.html @@ -13,6 +13,6 @@ plt.rgstrMouseRClickPlot( () => { console.log("right click"); } );

-Click for ascii data +@DATALINK@ diff --git a/tools/Python/mcplot/matplotlib/plotfuncs.py b/tools/Python/mcplot/matplotlib/plotfuncs.py index 5b70bb220..53ab8bb45 100644 --- a/tools/Python/mcplot/matplotlib/plotfuncs.py +++ b/tools/Python/mcplot/matplotlib/plotfuncs.py @@ -211,7 +211,16 @@ def plot_single_data(node, i, n, log): #ax = Axes3D(pylab.gcf()) #pylab.contour(x,y,zvals) #ax.plot_surface(x,y,zvals) - pylab.pcolor(x,y,zvals) + # pcolormesh (QuadMesh) instead of pcolor (PolyCollection): pcolor + # builds one vector polygon per grid cell, which is dramatically + # slower to construct and render for vector output formats + # (mctest.py generates a PDF overview per test via --format=pdf), + # and produces much larger files. pcolormesh is a same-signature + # drop-in given our regularly-gridded x/y (from linspace), and + # rasterized=True has vector backends embed it as a single bitmap + # rather than thousands of individual polygons; PNG/interactive + # output is raster either way and unaffected. + pylab.pcolormesh(x, y, zvals, shading='auto', rasterized=True) pylab.colorbar() pylab.xlabel(data.xlabel, fontsize=fontsize, fontweight='bold') diff --git a/tools/Python/mctest/main.template b/tools/Python/mctest/main.template index 85e554c46..e22bfaa22 100644 --- a/tools/Python/mctest/main.template +++ b/tools/Python/mctest/main.template @@ -50,11 +50,17 @@ a:visited { color: lightgrey; background-color: transparent; text-decoration: no C:{{ c.1 }}/R:{{ c.2 }}
I={{ c.3 }} ({{ c.4 }})
{{ c.6 }}
PLOTS: [Interactive HTML] [PDF overview] + {%- if c.8 %} +


DIFF: [vs ref] + {%- endif %} {%- elif c.0 == 2 %} C:{{ c.1 }}/R:{{ c.2 }}
I={{ c.3 }} ({{ c.4 }})
{{ c.6 }}
PLOTS: [Interactive HTML] [PDF overview] + {%- if c.8 %} +

DIFF: [vs ref] + {%- endif %} {%- elif c.0 == 3 %} C:{{ c.1 }} diff --git a/tools/Python/mctest/mcviewtest.md.in b/tools/Python/mctest/mcviewtest.md.in index c9fe61781..6b911bf6b 100644 --- a/tools/Python/mctest/mcviewtest.md.in +++ b/tools/Python/mctest/mcviewtest.md.in @@ -9,13 +9,26 @@ the @MCCODE_PREFIX@test toool, testing the @MCCODE_NAME@ installation # SYNOPSIS -**@MCCODE_PREFIX@viewtest** [-h] [--reflabel [REFLABEL]] [--testroot [TESTROOT]] [--verbose] [--nobrowse] [testdir] +**@MCCODE_PREFIX@viewtest** [-h] [--reflabel [REFLABEL]] [--testroot [TESTROOT]] [--verbose] [--nobrowse] [--nodiff] [--diff-errors-only] [--diffmax DIFFMAX] [--diffworkers DIFFWORKERS] [testdir] # DESCRIPTION The tool **@MCCODE_PREFIX@viewtest** is a program visualise testesults from the current @MCCODE_NAME@ installation. +When more than one test run is present (a reference run and one or more +other runs, e.g. different platforms, GPU vs. CPU, or different @MCCODE_NAME@ +versions/branches), taking the first/reference test run's results as the +'reference' column, **@MCCODE_PREFIX@viewtest** also generates a +**@MCCODE_PREFIX@plotdiff-html** comparison between each other column's +monitor output and the reference's, and adds a `DIFF: [vs ref]` link to the +test report next to each such row. These comparisons are cached on disk (in +a `diff_vs_` subfolder next to the test's own output), so re-running +**@MCCODE_PREFIX@viewtest** over the same test data does not regenerate +existing comparisons, and they are generated in parallel (see +`--diffworkers` below) to keep this reasonably fast even for large test +suites (a few hundred instruments). + # OPTIONS **-h, --help** @@ -35,6 +48,26 @@ from the current @MCCODE_NAME@ installation. **--nobrowse** Do not spawn browser on exit +**--nodiff** + Do not generate **@MCCODE_PREFIX@plotdiff-html** comparison cells + against the reference column at all + +**--diff-errors-only** + Only generate diff cells for rows that show a discrepancy against + the reference (i.e. skip rows that already match the reference within + tolerance). Default is to diff every row that has valid data on both + sides, since these comparisons are cached and run in parallel; use this + switch to further cut down the number of comparisons generated, e.g. on + a very large test suite + +**--diffmax DIFFMAX** + Maximum time (s) allowed per **@MCCODE_PREFIX@plotdiff-html** + comparison (default 300s) + +**--diffworkers DIFFWORKERS** + Number of **@MCCODE_PREFIX@plotdiff-html** comparisons to run in + parallel (default: number of processors available to this process) + **testdir** Indicate location of result directories (PWD is used if unset) @@ -50,10 +83,16 @@ http://www.@FLAVOR@.org `@MCCODE_PREFIX@viewtest` +View test results, but skip generating diff cells against the reference column entirely (fastest, no comparisons at all) +: `@MCCODE_PREFIX@viewtest --nodiff` + +View test results, only diffing rows that show a discrepancy, using 8 parallel workers +: `@MCCODE_PREFIX@viewtest --diff-errors-only --diffworkers 8` + # AUTHORS @MCCODE_NAME@ Team (@FLAVOR@.org) # SEE ALSO -@FLAVOR@(1), @MCCODE_PREFIX@test(1), @MCCODE_PREFIX@doc(1), @MCCODE_PREFIX@plot(1), @MCCODE_PREFIX@run(1), @MCCODE_PREFIX@gui(1), @MCCODE_PREFIX@display(1) +@FLAVOR@(1), @MCCODE_PREFIX@test(1), @MCCODE_PREFIX@doc(1), @MCCODE_PREFIX@plot(1), @MCCODE_PREFIX@plotdiff-html(1), @MCCODE_PREFIX@run(1), @MCCODE_PREFIX@gui(1), @MCCODE_PREFIX@display(1) diff --git a/tools/Python/mctest/mcviewtest.py b/tools/Python/mctest/mcviewtest.py old mode 100755 new mode 100644 index c219325ec..5a6d8e3ba --- a/tools/Python/mctest/mcviewtest.py +++ b/tools/Python/mctest/mcviewtest.py @@ -9,6 +9,7 @@ from os import walk import subprocess import shutil +import concurrent.futures import jinja2 ERROR_PERCENT_THRESSHOLD_ACCEPT = 20 @@ -31,9 +32,28 @@ def get_oldest_dir(directory_name): files.sort(key=lambda x:x[0]) return files[0][1] -def run_normal_mode(testdir, reflabel): +def get_default_diffworkers(): + ''' Number of mcplotdiff-html comparisons to run in parallel by default: + the number of processors available to this process. Prefers + os.sched_getaffinity(0) (Linux only) over os.cpu_count(), since the + former respects cgroup/taskset CPU restrictions (e.g. inside a + container or batch job allocation) that the latter ignores. ''' + try: + return len(os.sched_getaffinity(0)) + except AttributeError: + # os.sched_getaffinity doesn't exist on macOS/Windows + return os.cpu_count() or 4 + +def run_normal_mode(testdir, reflabel, nodiff=False, diffmax=300, diffall=True, diffworkers=None): ''' load test data and print to html label ''' + if diffworkers is None: + diffworkers = get_default_diffworkers() + + # jobs collected during row-building, run in parallel afterwards (see + # plan_diff_link() below for why this is a two-phase process) + pending_diff_jobs = [] + def get_col_header(label, meta): try: return "
".join((label + " - " + meta.get("ncount", ""), meta.get("hostname", ""), "CPU: " + meta.get("cpu_type", ""), "GPU: " + meta.get("gpu_type", ""), meta.get("date", ""))) @@ -53,7 +73,70 @@ def get_header_lst(meta): lst.append(meta["date"]) return lst - def get_cell_tuple(cellobj, refval=None): + def get_data_url(cellobj): + ''' Reconstructs the relative "label/instrname/testnb/" data directory + for a cellobj, the same way get_cell_tuple() does for its own + cell - used here to also locate the *reference* column's data + directory when generating a diff link. ''' + label = cellobj["localfile"].split("/") + if len(label) == 1: + label = cellobj["localfile"].split("\\") + label = label[len(label) - 3] + return label + "/" + cellobj["instrname"] + "/" + str(cellobj["testnb"]) + "/" + + def plan_diff_link(refcellobj, url, label, row, col_idx): + ''' Decides whether a diff against the reference column is + applicable, and if so either: + - returns the link immediately, if a cached diff already + exists on disk, or + - registers a pending job to be run later (see the parallel + execution pass after all rows have been built, below), and + optimistically returns the link the job is expected to + produce. If the job later fails, the corresponding cell in + `row` is patched to drop the link (see the execution pass). + + This is deliberately NOT where the mcplotdiff-html subprocess is + actually run: with ~300+ rows x multiple columns, running these + one at a time while building the table serializes what is an + embarrassingly parallel, purely I/O+CPU bound batch of + independent jobs (disjoint output directories, no shared + state). Collecting them here and running them via a bounded + thread pool afterwards is a straightforward, safe win. ''' + if nodiff: + return None + if refcellobj is None or refcellobj.get("testval") is None: + # no valid reference data to diff against + return None + + ref_url = get_data_url(refcellobj) + test_abs = join(testdir, url) + ref_abs = join(testdir, ref_url) + if not (os.path.isfile(join(test_abs, "mccode.sim")) and os.path.isfile(join(ref_abs, "mccode.sim"))): + # NeXus-only output (mccode.h5) or otherwise nothing mcplotdiff-html can compare + return None + + outdir_rel = join(url, "diff_vs_%s" % reflabel) + outdir_abs = join(testdir, outdir_rel) + index_abs = join(outdir_abs, "index.html") + link = (outdir_rel + "/index.html").replace(os.sep, '/').replace('//', '/') + + if os.path.isfile(index_abs): + # already cached from a previous run, nothing to do + return link + + diffplotter = mccode_config.configuration["MCPLOT"].split('-')[0] + "diff-html" + cmd = '%s "%s" "%s" --nobrowse -A "%s" -B "%s" --output "%s"' % ( + diffplotter, test_abs, ref_abs, label, reflabel, outdir_abs) + + pending_diff_jobs.append({ + 'cmd': cmd, + 'index_abs': index_abs, + 'row': row, + 'col_idx': col_idx, + }) + return link + + def get_cell_tuple(cellobj, refval=None, refcellobj=None, row=None, col_idx=None): ''' set up and format cell data ''' state = None compiletime = None @@ -141,7 +224,11 @@ def get_cell_tuple(cellobj, refval=None): else: refp = "%2.f" % refp + "%" - return (state, compiletime, runtime, testval, refp, url, display, displayurl) + diffurl = None + if diffall or state == 2: + diffurl = plan_diff_link(refcellobj, url, label, row, col_idx) + + return (state, compiletime, runtime, testval, refp, url, display, displayurl, diffurl) def get_empty_cell_tuple(tag=None): ''' return a "state_four" black cell, optionally with a tag, this could be "no ref" or "no test" etc. ''' @@ -188,7 +275,14 @@ def iterate_obj_to_populate_rows(iterobj, otherobjs, rows, ncols, use_iterobj_re targetval = o.get("targetval", None) if use_iterobj_refvalue: targetval = iterobj[key]["targetval"] - row.append(get_cell_tuple(o, targetval)) + # diff cells always compare against the true reference + # column (refobj), regardless of which object is + # currently driving iteration (iterobj may be a + # fallback "lead" column rather than refobj itself, in + # the untested multi-column case below) + refcellobj = refobj.get(key, None) + col_idx = len(row) + row.append(get_cell_tuple(o, targetval, refcellobj=refcellobj, row=row, col_idx=col_idx)) # delete "used" cell keys if del_used_from_overobjs: @@ -244,6 +338,37 @@ def iterate_obj_to_populate_rows(iterobj, otherobjs, rows, ncols, use_iterobj_re leadcol = testobjs.pop(0) iterate_obj_to_populate_rows(leadcol, testobjs, rows, ncols=numcols, use_iterobj_refvalue=False) + # Run all collected mcplotdiff-html jobs in parallel now that every row + # has been built (see plan_diff_link() above). Each job's cell already + # holds the *expected* link optimistically; if the job fails or times + # out, patch that specific cell back to drop the link rather than + # leaving a dead one in the rendered report. + if pending_diff_jobs: + logging.info("Running %d mcplotdiff-html comparison(s) (up to %d in parallel)..." + % (len(pending_diff_jobs), diffworkers)) + + def _run_diff_job(job): + try: + utils.run_subtool_noread(job['cmd'], cwd=testdir, timeout=diffmax) + except Exception as e: + logging.info("diff job failed: %s" % str(e)) + return job + + done = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=diffworkers) as executor: + futures = [executor.submit(_run_diff_job, job) for job in pending_diff_jobs] + for future in concurrent.futures.as_completed(futures): + job = future.result() + if not os.path.isfile(job['index_abs']): + # job failed/timed out: drop the optimistically-set link + row = job['row'] + col_idx = job['col_idx'] + cell = row[col_idx] + row[col_idx] = cell[:8] + (None,) + done += 1 + if done % 20 == 0 or done == len(pending_diff_jobs): + logging.info(" ...%d/%d diffs done" % (done, len(pending_diff_jobs))) + text = open(join(dirname(__file__), "main.template")).read() html = jinja2.Template(text).render(hrow=hrow, rows=rows, header=get_header_lst(refmeta)) @@ -304,7 +429,15 @@ def main(args): print("No testdir defined, will use current dir") print("--> Using testdir=%s\n" % os.getcwd()) testdir=os.getcwd() - run_normal_mode(testdir, reflabel) + diffmax = 300 + if args.diffmax: + diffmax = int(args.diffmax[0]) + diffworkers = get_default_diffworkers() + if args.diffworkers: + diffworkers = int(args.diffworkers[0]) + diffall = not args.diff_errors_only + run_normal_mode(testdir, reflabel, nodiff=args.nodiff, diffmax=diffmax, + diffall=diffall, diffworkers=diffworkers) if not args.nobrowse: subprocess.Popen('%s %s' % (mccode_config.configuration['BROWSER'], os.path.join(testdir,os.path.basename(testdir) +'_output.html')), shell=True) @@ -318,6 +451,10 @@ def main(args): parser.add_argument('--testroot', nargs="?", help='test root folder for test result management') parser.add_argument('--verbose', action='store_true', help='output excessive information for debug purposes') parser.add_argument('--nobrowse', action='store_true', help='Do not spawn browser on exit') + parser.add_argument('--nodiff', action='store_true', help='Do not generate mcplotdiff-html comparison cells against the reference column') + parser.add_argument('--diff-errors-only', dest='diff_errors_only', action='store_true', help='Only generate diff cells for rows that show a discrepancy against the reference (default: diff every row with valid data)') + parser.add_argument('--diffmax', nargs=1, help='Maximum time (s) allowed per mcplotdiff-html comparison (default 300s)') + parser.add_argument('--diffworkers', nargs=1, help='Number of mcplotdiff-html comparisons to run in parallel (default: number of available processors)') args = parser.parse_args() main(args)