mcviewtest: spawn the new mcplotdiff-html plotter

tools/Python/mccodelib/mcplotloader.py

@@ -5,6 +5,7 @@
 import glob
 import re
 import os
+import numpy as np
 from os.path import isfile, isdir, join, dirname, basename, splitext, exists
 from os import walk
 from decimal import Decimal
@@ -13,6 +14,47 @@
 from .plotgraph import PlotGraphPrint, DataHandle, PNMultiple, PNSingle
 
 
+def _parse_data_block(lines_block):
+    ''' Fast, vectorized parser for a block of whitespace-separated numeric
+        data lines (e.g. the "# Data" or "# Events" section of a 2D monitor,
+        or the data rows of a 1D monitor), returning a list of rows (each a
+        list of floats).
+
+        This replaces what used to be a per-line, per-value Python float()
+        loop in _parse_1D_monitor()/_parse_2D_monitor() - for a 2D monitor
+        that's one Python-level float() call per grid cell (e.g. 90000 for a
+        300x300 PSD monitor, doubled if an Events section is present), which
+        dominates load time for instruments with sizeable 2D monitors.
+        np.fromstring(..., sep=' ') parses the whole block in one call
+        instead of looping in Python.
+
+        Falls back to the slower, more permissive line-by-line parser (which
+        silently skips any line that fails to parse, matching the historical
+        behaviour) if the fast path doesn't cleanly apply - e.g. a genuinely
+        malformed/truncated file, blank lines within the block, or a jagged
+        block with inconsistent column counts between rows. This keeps the
+        common case fast without sacrificing correctness on the edge cases
+        the original loop happened to tolerate. '''
+    if not lines_block:
+        return []
+    try:
+        ncols = len(lines_block[0].split())
+        if ncols == 0:
+            raise ValueError('empty first line in data block')
+        flat = np.fromstring(' '.join(lines_block), dtype=float, sep=' ')
+        if flat.size != ncols * len(lines_block):
+            raise ValueError('unexpected element count (jagged or malformed data block)')
+        return flat.reshape(len(lines_block), ncols).tolist()
+    except Exception:
+        rows = []
+        for l in lines_block:
+            try:
+                rows.append([float(item) for item in l.strip().split()])
+            except Exception:
+                pass
+        return rows
+
+
 '''
 McCode simulation output data types.
 '''
@@ -193,24 +235,13 @@ def _parse_1D_monitor(text):
 
         # load the actual data
         lines = text.splitlines()
-        xvals = []
-        yvals = []
-        y_err_vals = []
-        Nvals = []
-        for l in lines:
-            if '#' in l:
-                continue
+        data_lines = [l for l in lines if '#' not in l]
+        rows = _parse_data_block(data_lines)
 
-            vals = l.split()
-            xvals.append(float(vals[0]))
-            yvals.append(float(vals[1]))
-            y_err_vals.append(float(vals[2]))
-            Nvals.append(float(vals[3]))
-
-        data.xvals = xvals
-        data.yvals = yvals
-        data.y_err_vals = y_err_vals
-        data.Nvals = Nvals
+        data.xvals      = [r[0] for r in rows]
+        data.yvals      = [r[1] for r in rows]
+        data.y_err_vals = [r[2] for r in rows]
+        data.Nvals      = [r[3] for r in rows]
 
     except Exception as e:
         print('Data1D load error.')
@@ -279,13 +310,15 @@ def _parse_2D_monitor(text):
         dat = True
         datacount=0;
         events = False
+        data_lines = []
+        events_lines = []
         for l in lines:
             if '# Data ' in l:
                 datacount=datacount+1;
                 # In case we meet mutiple # Data entries, file is probably saved via -USR2 or from 
                 # Progress_bar(). In that case, better flush earlier data entries:
                 if (datacount>1):
-                    data.zvals=[]
+                    data_lines=[]
                 dat = True
                 continue
 
@@ -301,18 +334,13 @@ def _parse_2D_monitor(text):
                 continue
 
             if dat:
-                try:
-                    vals = [float(item) for item in l.strip().split()]
-                    data.zvals.append(vals)
-                except:
-                    pass
+                data_lines.append(l)
 
             if events:
-                try:
-                    vals = [float(item) for item in l.strip().split()]
-                    data.counts.append(vals)
-                except:
-                    pass
+                events_lines.append(l)
+
+        data.zvals = _parse_data_block(data_lines)
+        data.counts = _parse_data_block(events_lines)
 
     except Exception as e:
         print('Data2D load error.')

tools/Python/mcplot/html/mcplot.py

@@ -10,6 +10,7 @@
 import base64
 import json
 import subprocess
+import concurrent.futures
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
 
@@ -23,6 +24,18 @@
 WIDTH = 1024
 HEIGHT = 768
 
+def get_default_workers():
+    ''' Number of per-monitor plot jobs to run in parallel by default: the
+        number of processors available to this process. Prefers
+        os.sched_getaffinity(0) (Linux only) over os.cpu_count(), since the
+        former respects cgroup/taskset CPU restrictions (e.g. inside a
+        container or batch job allocation) that the latter ignores. '''
+    try:
+        return len(os.sched_getaffinity(0))
+    except AttributeError:
+        # os.sched_getaffinity doesn't exist on macOS/Windows
+        return os.cpu_count() or 4
+
 def file_base_name(file_name):
     if '.' in file_name:
         separator_index = file_name.index('.')
@@ -35,13 +48,14 @@ def path_base_name(path):
     file_name = os.path.basename(path)
     return file_base_name(file_name)
 
-def get_html(template_name, params, simfile):
+def get_html(template_name, params, simfile, use_logscale):
     '''  '''
     text = open(os.path.join(os.path.dirname(__file__),template_name)).read()
     text = text.replace("@PARAMS@", params)
     text = text.replace("@DATAFILE@", simfile)
-
-    logscalestr = "true" if logscale==True else "false"
+    text = text.replace("@DATALINK@", "Click for ascii data")
+
+    logscalestr = "true" if use_logscale else "false"
     text = text.replace("@LOGSCALE@", logscalestr)
     text = text.replace("@LIBPATH@", libpath)
     return text
@@ -220,27 +234,42 @@ def browse(html_filepath):
 def plotfunc(node, filename=None):
     ''' plot a plotnode to a html file as an svg-plot using plotfuncs.js and d3.v4.min.js '''
 
-    global logscale
     if isinstance(filename, list):
         filename = filename[0]
     # get data and set file path
     if type(node) is PNSingle:
         data = node.getdata_idx(0)
-        return plotfunc_single(data, filename)
+        return plotfunc_single(data, filename, use_logscale=logscale)
 
     elif type(node) is PNMultiple:
-        data = node.getdata_lst()
-        data_lst = data
-
-        f     = []
-        f_log = []
-        count = 0
+        data_lst = node.getdata_lst()
+
+        # Each monitor's linear and log-scale pages are fully independent
+        # (own numpy arrays, own PIL image, own output file - no shared
+        # matplotlib-style global figure/axes state the way the
+        # mcplot-matplotlib tool has), so unlike that tool this is safe to
+        # parallelize directly: build the full list of (monitor, use_log)
+        # jobs up front and run them through a bounded thread pool, instead
+        # of the previous sequential loop that flipped a shared global
+        # `logscale` flag between calls (unsafe to parallelize as-is, since
+        # concurrent threads would race on that flag).
+        jobs = []
         for dat in data_lst:
-            logscale = False
-            f.append(plotfunc_single(dat))
-            logscale = True
-            f_log.append(plotfunc_single(dat))
-            count += 1
+            jobs.append((dat, False))
+            jobs.append((dat, True))
+
+        def _run_job(job):
+            dat, use_log = job
+            return plotfunc_single(dat, None, use_logscale=use_log)
+
+        max_workers = get_default_workers()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            # executor.map preserves input order regardless of completion
+            # order, so results[0::2]/[1::2] line up with data_lst as before
+            results = list(executor.map(_run_job, jobs))
+
+        f     = results[0::2]
+        f_log = results[1::2]
         # now create an overview HTML (index.html with <iframe>)
         directory  = os.path.dirname(f[0])
         baseinstr = os.path.basename(os.getcwd())
@@ -334,13 +363,13 @@ def plotfunc(node, filename=None):
             outfile.write("</body></html>\n")
         return filename
 
-def plotfunc_single(data, f = None):
+def plotfunc_single(data, f = None, use_logscale=False):
     """save the data node into given file"""
     # three cases of plot data (1D, 2D and multiple), each block should end with a fully formed 'text' variable
     text = ""
 
     if f is None:
-        if logscale:
+        if use_logscale:
             f = data.filepath + "_log.html"
         else:
             f = data.filepath + ".html"
@@ -350,11 +379,11 @@ def plotfunc_single(data, f = None):
 
     # create 1D html
     if type(data) is Data1D:
-        text = get_html('template_1d.html', get_params_str_1D(data), os.path.basename(data.filename))
+        text = get_html('template_1d.html', get_params_str_1D(data), os.path.basename(data.filename), use_logscale)
 
     # create 2D html
     elif type(data) is Data2D:
-        text = get_html('template_2d.html', get_params_str_2D(data), os.path.basename(data.filename))
+        text = get_html('template_2d.html', get_params_str_2D(data), os.path.basename(data.filename), use_logscale)
 
     # write to file
     with open(f, 'w') as fid: