mcviewtest: spawn the new mcplotdiff-html plotter

.github/workflows/mcstas-conda-testsuite.yml

@@ -133,3 +133,119 @@ jobs:
       if: always() && (inputs.manual-debugging == true)
       with:
         limit-access-to-actor: true
+
+  compare:
+    name: mcstas-conda-testsuite-comparison
+    needs: build
+    if: always()
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+      with:
+        path: src
+
+    - name: Get conda dependency list
+      id: conda-deps
+      run: |
+           ./src/devel/bin/mccode-create-conda-yml -m mcstas -n mcstas -o dependencies.yml
+
+    - uses: mamba-org/setup-micromamba@v3
+      with:
+        environment-name: mcstas
+        environment-file: dependencies.yml
+
+    - name: Configure build and install mcstas
+      id: mcstas-install
+      run: |
+           # Only the Python tooling (mcviewtest, mcplotdiff-html, mcplot-html) is
+           # actually needed here - the diff/report generation is platform-agnostic
+           # regardless of which OS originally produced the monitor data - but we
+           # reuse the exact same install path as the build job for consistency,
+           # rather than maintaining a second, lighter-weight install mechanism.
+           ./src/devel/bin/mccode-build-conda -m mcstas -s $PWD/src -b $PWD/build_mcstas
+
+    - name: 'Download all platform test artifacts'
+      id: download
+      uses: actions/download-artifact@v6
+      with:
+        # matches mcstas-artefacts-ubuntu-latest.openmpi.conda-testsuite,
+        # mcstas-artefacts-macos-latest.openmpi.conda-testsuite,
+        # mcstas-artefacts-windows-latest.msmpi.conda-testsuite - by pattern
+        # rather than hardcoded names, so this keeps working if the build
+        # matrix gains/loses a platform
+        pattern: mcstas-artefacts-*.conda-testsuite
+        path: downloaded_artifacts
+
+    - name: 'Merge platform columns into one shared testdir'
+      id: merge
+      run: |
+           set -e
+           set -u
+           set -x
+
+           mkdir combined
+           REFLABEL=""
+
+           # Each platform's tarball contains exactly one label directory
+           # directly under run_test-suite/ (e.g.
+           # mcstas-3.99.99_mpi_x_2_ubuntu-latest_openmpi_1e6_Linux_<datetime>).
+           # mcviewtest treats each such directory as one comparison "column",
+           # so merging them as siblings under one shared testdir is all that's
+           # needed to make them comparable.
+           for tgz in downloaded_artifacts/*/*.tgz; do
+             work=$(mktemp -d)
+             tar xzf "$tgz" -C "$work"
+             labeldir=$(ls -d "$work"/run_test-suite/*/)
+             mv "$labeldir" combined/
+
+             # Linux (ubuntu-latest) is the reference platform; the label
+             # name itself embeds a version string and timestamp we can't
+             # predict, so identify it via the source artifact/tarball name
+             # instead of hardcoding it.
+             if [[ "$tgz" == *"ubuntu-latest"* ]]; then
+               REFLABEL=$(basename "$labeldir")
+             fi
+             rm -rf "$work"
+           done
+
+           if [ -z "$REFLABEL" ]; then
+             echo "ERROR: could not identify the Linux/reference column (ubuntu-latest artifact missing?)"
+             exit 1
+           fi
+
+           echo "REFLABEL=$REFLABEL" >> "$GITHUB_ENV"
+           echo "Reference column: $REFLABEL"
+           ls combined
+
+    - name: 'Run mcviewtest across all platform columns'
+      id: viewtest
+      run: |
+           set -e
+           set -u
+           set -x
+           cd "$PWD/combined" && mcviewtest --reflabel "$REFLABEL" --nobrowse
+
+    - name: 'Tar combined output (all platform folders + comparison table)'
+      id: tar-package
+      if: always()
+      run: |
+           tar cvfz mcstas-conda-testsuite-comparison.tgz combined
+
+    - name: 'Upload comparison artifact'
+      id: comparison-upload
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: mcstas-testsuite-comparison
+        path: mcstas-conda-testsuite-comparison.tgz
+
+    - name: Setup tmate session for manual debugging
+      uses: mxschmitt/action-tmate@v3
+      if: always() && (inputs.manual-debugging == true)
+      with:
+        limit-access-to-actor: true

.github/workflows/mcxtrace-conda-testsuite.yml

@@ -133,3 +133,120 @@ jobs:
       if: always() && (inputs.manual-debugging == true)
       with:
         limit-access-to-actor: true
+
+  compare:
+    name: mcxtrace-conda-testsuite-comparison
+    needs: build
+    if: always()
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v6
+      with:
+        path: src
+
+    - name: Get conda dependency list
+      id: conda-deps
+      run: |
+           ./src/devel/bin/mccode-create-conda-yml -m mcxtrace -n mcxtrace -o dependencies.yml
+
+    - uses: mamba-org/setup-micromamba@v3
+      with:
+        environment-name: mcxtrace
+        environment-file: dependencies.yml
+
+
+    - name: Configure build and install mcxtrace
+      id: mcxtrace-install
+      run: |
+           # Only the Python tooling (mxviewtest, mxplotdiff-html, mxplot-html) is
+           # actually needed here - the diff/report generation is platform-agnostic
+           # regardless of which OS originally produced the monitor data - but we
+           # reuse the exact same install path as the build job for consistency,
+           # rather than maintaining a second, lighter-weight install mechanism.
+           ./src/devel/bin/mccode-build-conda -m mcxtrace -s $PWD/src -b $PWD/build_mcxtrace
+
+    - name: 'Download all platform test artifacts'
+      id: download
+      uses: actions/download-artifact@v6
+      with:
+        # matches mcxtrace-artefacts-ubuntu-latest.openmpi.conda-testsuite,
+        # mcxtrace-artefacts-macos-latest.openmpi.conda-testsuite,
+        # mcxtrace-artefacts-windows-latest.msmpi.conda-testsuite - by pattern
+        # rather than hardcoded names, so this keeps working if the build
+        # matrix gains/loses a platform
+        pattern: mcxtrace-artefacts-*.conda-testsuite
+        path: downloaded_artifacts
+
+    - name: 'Merge platform columns into one shared testdir'
+      id: merge
+      run: |
+           set -e
+           set -u
+           set -x
+
+           mkdir combined
+           REFLABEL=""
+
+           # Each platform's tarball contains exactly one label directory
+           # directly under run_test-suite/ (e.g.
+           # mcxtrace-1.99.99_mpi_x_2_ubuntu-latest_openmpi_1e6_Linux_<datetime>).
+           # mxviewtest treats each such directory as one comparison "column",
+           # so merging them as siblings under one shared testdir is all that's
+           # needed to make them comparable.
+           for tgz in downloaded_artifacts/*/*.tgz; do
+             work=$(mktemp -d)
+             tar xzf "$tgz" -C "$work"
+             labeldir=$(ls -d "$work"/run_test-suite/*/)
+             mv "$labeldir" combined/
+
+             # Linux (ubuntu-latest) is the reference platform; the label
+             # name itself embeds a version string and timestamp we can't
+             # predict, so identify it via the source artifact/tarball name
+             # instead of hardcoding it.
+             if [[ "$tgz" == *"ubuntu-latest"* ]]; then
+               REFLABEL=$(basename "$labeldir")
+             fi
+             rm -rf "$work"
+           done
+
+           if [ -z "$REFLABEL" ]; then
+             echo "ERROR: could not identify the Linux/reference column (ubuntu-latest artifact missing?)"
+             exit 1
+           fi
+
+           echo "REFLABEL=$REFLABEL" >> "$GITHUB_ENV"
+           echo "Reference column: $REFLABEL"
+           ls combined
+
+    - name: 'Run mxviewtest across all platform columns'
+      id: viewtest
+      run: |
+           set -e
+           set -u
+           set -x
+           cd "$PWD/combined" && mxviewtest --reflabel "$REFLABEL" --nobrowse
+
+    - name: 'Tar combined output (all platform folders + comparison table)'
+      id: tar-package
+      if: always()
+      run: |
+           tar cvfz mcxtrace-conda-testsuite-comparison.tgz combined
+
+    - name: 'Upload comparison artifact'
+      id: comparison-upload
+      uses: actions/upload-artifact@v6
+      if: always()
+      with:
+        name: mcxtrace-testsuite-comparison
+        path: mcxtrace-conda-testsuite-comparison.tgz
+
+    - name: Setup tmate session for manual debugging
+      uses: mxschmitt/action-tmate@v3
+      if: always() && (inputs.manual-debugging == true)
+      with:
+        limit-access-to-actor: true

tools/Python/mccodelib/mcplotloader.py

@@ -5,6 +5,7 @@
 import glob
 import re
 import os
+import numpy as np
 from os.path import isfile, isdir, join, dirname, basename, splitext, exists
 from os import walk
 from decimal import Decimal
@@ -13,6 +14,47 @@
 from .plotgraph import PlotGraphPrint, DataHandle, PNMultiple, PNSingle
 
 
+def _parse_data_block(lines_block):
+    ''' Fast, vectorized parser for a block of whitespace-separated numeric
+        data lines (e.g. the "# Data" or "# Events" section of a 2D monitor,
+        or the data rows of a 1D monitor), returning a list of rows (each a
+        list of floats).
+
+        This replaces what used to be a per-line, per-value Python float()
+        loop in _parse_1D_monitor()/_parse_2D_monitor() - for a 2D monitor
+        that's one Python-level float() call per grid cell (e.g. 90000 for a
+        300x300 PSD monitor, doubled if an Events section is present), which
+        dominates load time for instruments with sizeable 2D monitors.
+        np.fromstring(..., sep=' ') parses the whole block in one call
+        instead of looping in Python.
+
+        Falls back to the slower, more permissive line-by-line parser (which
+        silently skips any line that fails to parse, matching the historical
+        behaviour) if the fast path doesn't cleanly apply - e.g. a genuinely
+        malformed/truncated file, blank lines within the block, or a jagged
+        block with inconsistent column counts between rows. This keeps the
+        common case fast without sacrificing correctness on the edge cases
+        the original loop happened to tolerate. '''
+    if not lines_block:
+        return []
+    try:
+        ncols = len(lines_block[0].split())
+        if ncols == 0:
+            raise ValueError('empty first line in data block')
+        flat = np.fromstring(' '.join(lines_block), dtype=float, sep=' ')
+        if flat.size != ncols * len(lines_block):
+            raise ValueError('unexpected element count (jagged or malformed data block)')
+        return flat.reshape(len(lines_block), ncols).tolist()
+    except Exception:
+        rows = []
+        for l in lines_block:
+            try:
+                rows.append([float(item) for item in l.strip().split()])
+            except Exception:
+                pass
+        return rows
+
+
 '''
 McCode simulation output data types.
 '''
@@ -193,24 +235,13 @@ def _parse_1D_monitor(text):
 
         # load the actual data
         lines = text.splitlines()
-        xvals = []
-        yvals = []
-        y_err_vals = []
-        Nvals = []
-        for l in lines:
-            if '#' in l:
-                continue
+        data_lines = [l for l in lines if '#' not in l]
+        rows = _parse_data_block(data_lines)
 
-            vals = l.split()
-            xvals.append(float(vals[0]))
-            yvals.append(float(vals[1]))
-            y_err_vals.append(float(vals[2]))
-            Nvals.append(float(vals[3]))
-
-        data.xvals = xvals
-        data.yvals = yvals
-        data.y_err_vals = y_err_vals
-        data.Nvals = Nvals
+        data.xvals      = [r[0] for r in rows]
+        data.yvals      = [r[1] for r in rows]
+        data.y_err_vals = [r[2] for r in rows]
+        data.Nvals      = [r[3] for r in rows]
 
     except Exception as e:
         print('Data1D load error.')
@@ -279,13 +310,15 @@ def _parse_2D_monitor(text):
         dat = True
         datacount=0;
         events = False
+        data_lines = []
+        events_lines = []
         for l in lines:
             if '# Data ' in l:
                 datacount=datacount+1;
                 # In case we meet mutiple # Data entries, file is probably saved via -USR2 or from 
                 # Progress_bar(). In that case, better flush earlier data entries:
                 if (datacount>1):
-                    data.zvals=[]
+                    data_lines=[]
                 dat = True
                 continue
 
@@ -301,18 +334,13 @@ def _parse_2D_monitor(text):
                 continue
 
             if dat:
-                try:
-                    vals = [float(item) for item in l.strip().split()]
-                    data.zvals.append(vals)
-                except:
-                    pass
+                data_lines.append(l)
 
             if events:
-                try:
-                    vals = [float(item) for item in l.strip().split()]
-                    data.counts.append(vals)
-                except:
-                    pass
+                events_lines.append(l)
+
+        data.zvals = _parse_data_block(data_lines)
+        data.counts = _parse_data_block(events_lines)
 
     except Exception as e:
         print('Data2D load error.')