RodSizer improvements: calibration & raft-splitting fixes, batch comparison, richer stats, UX (supersedes #2)

README.md

@@ -56,12 +56,8 @@ Group website: https://murphy-group.chemistry.illinois.edu/
 
 ## Notes:
 - DO NOT double-click on RodSizer_CLEANER_MacOS.command unless you are sure that you want to clean ALL local history of data and reports.
-- First-time launching may take some time, like 5-10 mins. Most of that is
-  installing the deep-learning packages (`tensorflow` + `stardist`, ~1 GB). These
-  are kept for possible future ML-based detection but are **not used** by the
-  current pipeline (K-means + watershed), so the wait is one-time setup only —
-  later launches are fast. See `backend/requirements.txt` to remove them if you
-  want a leaner install.
+- First-time launching takes a few minutes (typically 2-3) while Python
+  packages are installed. This is one-time setup only — later launches are fast.
 - Try ask a coding agent if there's an issue with environment setup.
 - MacOS is more recommended.
 - (Windows) If Windows Defender asks, click `More Info` -> `Run Anyway`.
@@ -88,4 +84,4 @@ Group website: https://murphy-group.chemistry.illinois.edu/
 
 ## Requirements
 - macOS or Windows
-- Python 3 installed (standard on most Macs, or downloadable from `python.org`)
+- Python 3.10-3.12 (the Mac launcher installs it automatically if missing; on Windows download from `python.org`)

RodSizer_DEBUG_Windows.bat

@@ -32,9 +32,11 @@ REM Auto-Detect
 if exist "C:\Python39\python.exe" set "PY_EXE=C:\Python39\python.exe" & goto FOUND_PYTHON
 if exist "C:\Python310\python.exe" set "PY_EXE=C:\Python310\python.exe" & goto FOUND_PYTHON
 if exist "C:\Python311\python.exe" set "PY_EXE=C:\Python311\python.exe" & goto FOUND_PYTHON
+if exist "C:\Python312\python.exe" set "PY_EXE=C:\Python312\python.exe" & goto FOUND_PYTHON
 if exist "C:\Program Files\Python39\python.exe" set "PY_EXE=C:\Program Files\Python39\python.exe" & goto FOUND_PYTHON
 if exist "C:\Program Files\Python310\python.exe" set "PY_EXE=C:\Program Files\Python310\python.exe" & goto FOUND_PYTHON
 if exist "C:\Program Files\Python311\python.exe" set "PY_EXE=C:\Program Files\Python311\python.exe" & goto FOUND_PYTHON
+if exist "C:\Program Files\Python312\python.exe" set "PY_EXE=C:\Program Files\Python312\python.exe" & goto FOUND_PYTHON
 
 :ASK_USER
 echo.

RodSizer_Launcher_MacOS.command

@@ -74,65 +74,47 @@ if command -v tesseract &>/dev/null; then
 fi
 
 # =============================================================================
-# STEP 2 — Python 3.11
-#
-#  TensorFlow 2.x officially supports Python 3.8–3.11.
-#  Python 3.12+ does NOT have stable TensorFlow wheels yet.
-#  We therefore require Python 3.11 specifically.
+# STEP 2 — Python 3.10-3.12
+#  (the pinned dependency set in backend/requirements.txt targets these
+#   versions; 3.13+ lacks wheels for the pinned numpy)
 # =============================================================================
-step "Step 2/4: Checking Python 3.11"
+step "Step 2/4: Checking Python"
 
 PYTHON_CMD=""
 
-# Helper: find python3.11 by checking all known locations
-find_python311() {
-    # 1. Homebrew-managed path (most reliable — works regardless of PATH)
-    local brew_prefix
-    brew_prefix="$(brew --prefix python@3.11 2>/dev/null)"
-    if [ -n "$brew_prefix" ] && [ -x "$brew_prefix/bin/python3.11" ]; then
-        echo "$brew_prefix/bin/python3.11"; return
-    fi
-
-    # 2. Direct binary in PATH
-    if command -v python3.11 &>/dev/null; then
-        command -v python3.11; return
-    fi
-
-    # 3. Common Homebrew Cellar fallback (Apple Silicon)
-    local cellar_py
-    cellar_py=$(ls /opt/homebrew/Cellar/python@3.11/*/bin/python3.11 2>/dev/null | sort -V | tail -1)
-    if [ -x "$cellar_py" ]; then
-        echo "$cellar_py"; return
-    fi
-    # Intel
-    cellar_py=$(ls /usr/local/Cellar/python@3.11/*/bin/python3.11 2>/dev/null | sort -V | tail -1)
-    if [ -x "$cellar_py" ]; then
-        echo "$cellar_py"; return
-    fi
-
-    # 4. pyenv
-    if command -v pyenv &>/dev/null; then
-        local pyenv_ver
-        pyenv_ver=$(pyenv versions --bare 2>/dev/null | grep "^3\.11\." | sort -V | tail -1)
-        if [ -n "$pyenv_ver" ]; then
-            local pyenv_py="$(pyenv root)/versions/$pyenv_ver/bin/python3.11"
-            [ -x "$pyenv_py" ] && echo "$pyenv_py"; return
+# Helper: find a suitable Python (3.10-3.12), preferring newer versions
+find_python() {
+    local candidate
+    for candidate in python3.12 python3.11 python3.10 python3; do
+        if command -v "$candidate" &>/dev/null; then
+            if "$candidate" -c 'import sys; sys.exit(0 if (3, 10) <= sys.version_info[:2] <= (3, 12) else 1)' &>/dev/null; then
+                command -v "$candidate"; return
+            fi
         fi
-    fi
+    done
+
+    # Homebrew-managed fallback (works regardless of PATH)
+    local brew_prefix ver
+    for ver in 3.12 3.11 3.10; do
+        brew_prefix="$(brew --prefix python@$ver 2>/dev/null)"
+        if [ -n "$brew_prefix" ] && [ -x "$brew_prefix/bin/python$ver" ]; then
+            echo "$brew_prefix/bin/python$ver"; return
+        fi
+    done
 }
 
-PYTHON_CMD="$(find_python311)"
+PYTHON_CMD="$(find_python)"
 
 # If still not found, install via Homebrew then retry
 if [ -z "$PYTHON_CMD" ]; then
-    warn "Python 3.11 not found. Installing via Homebrew (needed for TensorFlow)..."
-    brew install python@3.11
-    PYTHON_CMD="$(find_python311)"
+    warn "Python 3.10-3.12 not found. Installing 3.12 via Homebrew..."
+    brew install python@3.12
+    PYTHON_CMD="$(find_python)"
 fi
 
 if [ -z "$PYTHON_CMD" ] || [ ! -x "$PYTHON_CMD" ]; then
-    error "Could not find or install Python 3.11."
-    error "Please install it manually:  brew install python@3.11"
+    error "Could not find or install Python."
+    error "Please install it manually:  brew install python@3.12"
     echo "Press Enter to close..."; read -r; exit 1
 fi
 
@@ -170,84 +152,21 @@ needs_python_packages() {
     "$VENV_PYTHON" - <<'PY' &>/dev/null
 import importlib
 
-for name in ("fastapi", "uvicorn", "tensorflow", "numpy", "cv2", "stardist"):
+for name in ("fastapi", "uvicorn", "numpy", "cv2", "skimage", "tifffile", "h5py"):
     importlib.import_module(name)
 PY
 }
 
-install_tensorflow_with_fallback() {
-    local tf_spec="tensorflow==2.21.0"
-
-    info "Installing TensorFlow..."
-    if "$VENV_PYTHON" -m pip install "$tf_spec"; then
-        return 0
-    fi
-
-    warn "pip could not resolve tensorflow from the package index."
-    warn "Retrying TensorFlow install directly from PyPI..."
-    PIP_CONFIG_FILE=/dev/null "$VENV_PYTHON" -m pip install --index-url https://pypi.org/simple "$tf_spec" && return 0
-
-    return 1
-}
-
 # Check whether a first-time install is needed
 if ! needs_python_packages; then
-    info "Installing Python packages — this may take 5–10 minutes on first run."
+    info "Installing Python packages — this may take 2–3 minutes on first run."
     info "Please do NOT close this window."
 
     # Upgrade pip / setuptools silently
     "$VENV_PYTHON" -m pip install --upgrade pip setuptools wheel --quiet
 
-    # ── TensorFlow first (sets numpy version constraint everything else must follow) ──
-    #
-    #  tensorflow-macos was deprecated at 2.13 and forces numpy<2, which conflicts
-    #  with opencv-python-headless >=4.9 and ncempy >=1.15 (both require numpy>=2).
-    #
-    #  tensorflow >= 2.16 ships a universal wheel that runs natively on Apple
-    #  Silicon (M-series) without a separate macos fork, and is compatible with
-    #  numpy 2.x.  tensorflow-metal is still the GPU-acceleration plugin for M-chips.
-    if ! install_tensorflow_with_fallback; then
-        error "TensorFlow installation failed."
-        error "This is usually a package-index or network issue, not a RodSizer code issue."
-        if [ "$IS_APPLE_SILICON" = true ]; then
-            error "If needed, try this manually inside backend/.venv:"
-            error "  python -m pip install --index-url https://pypi.org/simple tensorflow==2.21.0"
-        else
-            error "Try rerunning later, or install TensorFlow manually inside backend/.venv."
-        fi
-        echo "Press Enter to close..."; read -r; exit 1
-    fi
-
-    # Note: tensorflow-metal is NOT installed because v1.2.0 is incompatible
-    # with tensorflow >=2.16 (dlopen fails on _pywrap_tensorflow_internal.so).
-    # Modern TensorFlow already runs natively on Apple Silicon without it.
-
-    if ! "$VENV_PYTHON" -c "import tensorflow" &>/dev/null 2>&1; then
-        error "TensorFlow import failed after installation."
-        error "Try deleting backend/.venv and relaunching, or check network connectivity."
-        echo "Press Enter to close..."; read -r; exit 1
-    fi
-
-    # ── Remaining packages (numpy version is now fixed by TF above) ─────────────
-    TMP_REQ=$(mktemp /tmp/rodsizer_req_XXXX.txt)
-    grep -v "^tensorflow" "$BACKEND_DIR/requirements.txt" > "$TMP_REQ"
-
-    if ! "$VENV_PYTHON" -m pip install -r "$TMP_REQ" --quiet; then
+    if ! "$VENV_PYTHON" -m pip install -r "$BACKEND_DIR/requirements.txt" --quiet; then
         error "Failed to install some packages. Check the output above."
-        rm -f "$TMP_REQ"
-        echo "Press Enter to close..."; read -r; exit 1
-    fi
-    rm -f "$TMP_REQ"
-
-    # ── Verify tensorflow is importable ───────────────────────────────────────
-    if ! "$VENV_PYTHON" -c "import tensorflow" &>/dev/null 2>&1; then
-        error "TensorFlow installation failed."
-        error "Try running this manually inside the venv:"
-        if [ "$IS_APPLE_SILICON" = true ]; then
-            error "  python -m pip install --index-url https://pypi.org/simple tensorflow==2.21.0"
-        else
-            error "  python -m pip install tensorflow==2.21.0"
-        fi
         echo "Press Enter to close..."; read -r; exit 1
     fi
 

RodSizer_Launcher_Windows.bat

@@ -29,9 +29,11 @@ REM --- 3. Try Auto-Detect Common Paths ---
 if exist "C:\Python39\python.exe" set "PY_EXE=C:\Python39\python.exe" & goto FOUND_PYTHON
 if exist "C:\Python310\python.exe" set "PY_EXE=C:\Python310\python.exe" & goto FOUND_PYTHON
 if exist "C:\Python311\python.exe" set "PY_EXE=C:\Python311\python.exe" & goto FOUND_PYTHON
+if exist "C:\Python312\python.exe" set "PY_EXE=C:\Python312\python.exe" & goto FOUND_PYTHON
 if exist "C:\Program Files\Python39\python.exe" set "PY_EXE=C:\Program Files\Python39\python.exe" & goto FOUND_PYTHON
 if exist "C:\Program Files\Python310\python.exe" set "PY_EXE=C:\Program Files\Python310\python.exe" & goto FOUND_PYTHON
 if exist "C:\Program Files\Python311\python.exe" set "PY_EXE=C:\Program Files\Python311\python.exe" & goto FOUND_PYTHON
+if exist "C:\Program Files\Python312\python.exe" set "PY_EXE=C:\Program Files\Python312\python.exe" & goto FOUND_PYTHON
 
 REM --- 4. User Input Fallback ---
 :ASK_USER

backend/autodetect_utils.py

@@ -432,6 +432,111 @@ def dilmarkers(markers, original_shape):
     return dilated_markers, bg_image # returning image as overlay placeholder
 
 
+def split_clump_intensity(gray_crop, crop_bool, min_marker_area, expected_width_px=None):
+    """Split fused SIDE-BY-SIDE rods using brightness seams in the raw image.
+
+    Parallel rods touching along their full length merge into one solid blob in
+    the binary mask. The distance transform of that blob has no "neck", so the
+    standard ``split_clump`` watershed sees a single object. In the raw TEM
+    image, however, the contact line between adjacent rods is a slightly
+    LIGHTER seam than the dark rod cores. This routine re-thresholds the clump
+    interior at a darker cutoff so those seams drop out, leaving one connected
+    "core" per rod; the cores then seed a watershed over the smoothed intensity
+    that assigns every clump pixel to its rod.
+
+    Args:
+        gray_crop: 2-D uint8 crop of the ORIGINAL grayscale image (rods dark).
+        crop_bool: matching 2-D boolean mask of the clump.
+        min_marker_area: minimum area (px) for final fragments.
+        expected_width_px: estimated single-rod width; used to size the core
+            filter so texture specks never become seeds.
+
+    Returns:
+        List of >= 2 boolean masks if a split was found, else None (caller
+        should fall back to the distance-transform splitter).
+    """
+    crop_bool = np.ascontiguousarray(crop_bool, dtype=bool)
+    if not crop_bool.any() or gray_crop.shape != crop_bool.shape or gray_crop.ndim != 2:
+        return None
+
+    vals = gray_crop[crop_bool]
+    if vals.size < 4 * min_marker_area:
+        return None
+
+    smooth = ndi.gaussian_filter(gray_crop.astype(np.float32), 2.0)
+
+    # A rod core (dark, elongated) is far larger than noise/texture blobs.
+    if expected_width_px and expected_width_px > 4:
+        core_floor = max(min_marker_area, int(0.5 * expected_width_px ** 2))
+    else:
+        core_floor = max(min_marker_area, 100)
+
+    # Sweep a few cutoffs; keep the one that resolves the most rod-sized cores.
+    best_markers, best_count = None, 0
+    for pct in (45, 55, 65):
+        t = np.percentile(vals, pct)
+        cores = (smooth < t) & crop_bool
+        cores = morphology.binary_opening(cores, morphology.disk(3))
+        lab, n = ndi.label(cores)
+        if n == 0:
+            continue
+        sizes = ndi.sum(cores, lab, index=np.arange(1, n + 1))
+        kept_ids = np.flatnonzero(np.asarray(sizes) >= core_floor) + 1
+        if len(kept_ids) > best_count:
+            remap = np.zeros(n + 1, dtype=np.int32)
+            remap[kept_ids] = np.arange(1, len(kept_ids) + 1)
+            best_markers, best_count = remap[lab], len(kept_ids)
+
+    if best_count < 2:
+        return None
+
+    # Outside the mask counts as maximally bright so basins never leak out.
+    inside = np.where(crop_bool, smooth, 255.0)
+    ws = watershed(inside, best_markers, mask=crop_bool)
+
+    masks = []
+    for label_idx in range(1, best_count + 1):
+        mask = ws == label_idx
+        if int(mask.sum()) >= min_marker_area:
+            masks.append(mask)
+
+    return masks if len(masks) >= 2 else None
+
+
+def _mask_short_side(mask):
+    cnts, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    if not cnts:
+        return 0.0
+    (_, (rw, rh), _) = cv2.minAreaRect(max(cnts, key=cv2.contourArea))
+    return float(min(rw, rh))
+
+
+def split_raft(gray_crop, crop_bool, min_marker_area, expected_width_px, depth=0):
+    """Recursively split a raft of side-by-side rods via intensity seams.
+
+    The first pass thresholds with the WHOLE clump's intensity statistics, so
+    rods much lighter than their neighbours may not seed a core and stay fused
+    with a neighbour. Each still-too-wide child is therefore re-split using its
+    own local statistics (recursion adapts the threshold to the lighter rods).
+
+    Returns a flat list of masks (may be [crop_bool] when nothing splits).
+    """
+    children = split_clump_intensity(gray_crop, crop_bool, min_marker_area,
+                                     expected_width_px=expected_width_px)
+    if not children:
+        return [crop_bool]
+
+    result = []
+    for child in children:
+        if (depth < 2 and expected_width_px
+                and _mask_short_side(child) >= 1.6 * expected_width_px):
+            result.extend(split_raft(gray_crop, child, min_marker_area,
+                                     expected_width_px, depth + 1))
+        else:
+            result.append(child)
+    return result
+
+
 def split_clump(crop_bool, min_marker_area, separation_strength=0):
     """
     Split a fused / low-solidity binary region into individual convex particles