From f0152cd988a29f99d26369b42fed9b60e12c9bc4 Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Mon, 20 Jul 2026 12:16:46 -0500 Subject: [PATCH 01/10] Remove unused tensorflow/stardist deps; support Python 3.10-3.13 Nothing in the pipeline imports stardist or tensorflow, and they were the main cause of slow (~5-10 min, ~1 GB) and failure-prone first-time setup. Mac launcher no longer forces Python 3.11 (a TensorFlow constraint) and now accepts any Python 3.10+; Windows launchers auto-detect 3.12/3.13. Co-Authored-By: Claude Fable 5 --- README.md | 8 +- RodSizer_DEBUG_Windows.bat | 4 + RodSizer_Launcher_MacOS.command | 141 +++++++------------------------- RodSizer_Launcher_Windows.bat | 4 + backend/requirements.txt | 14 ++-- 5 files changed, 44 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 1930a81..9ab8ac5 100644 --- a/README.md +++ b/README.md @@ -56,12 +56,8 @@ Group website: https://murphy-group.chemistry.illinois.edu/ ## Notes: - DO NOT double-click on RodSizer_CLEANER_MacOS.command unless you are sure that you want to clean ALL local history of data and reports. -- First-time launching may take some time, like 5-10 mins. Most of that is - installing the deep-learning packages (`tensorflow` + `stardist`, ~1 GB). These - are kept for possible future ML-based detection but are **not used** by the - current pipeline (K-means + watershed), so the wait is one-time setup only — - later launches are fast. See `backend/requirements.txt` to remove them if you - want a leaner install. +- First-time launching takes a few minutes (typically 2-3) while Python + packages are installed. This is one-time setup only — later launches are fast. - Try ask a coding agent if there's an issue with environment setup. - MacOS is more recommended. - (Windows) If Windows Defender asks, click `More Info` -> `Run Anyway`. diff --git a/RodSizer_DEBUG_Windows.bat b/RodSizer_DEBUG_Windows.bat index 5f13165..214a8ec 100644 --- a/RodSizer_DEBUG_Windows.bat +++ b/RodSizer_DEBUG_Windows.bat @@ -32,9 +32,13 @@ REM Auto-Detect if exist "C:\Python39\python.exe" set "PY_EXE=C:\Python39\python.exe" & goto FOUND_PYTHON if exist "C:\Python310\python.exe" set "PY_EXE=C:\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Python311\python.exe" set "PY_EXE=C:\Python311\python.exe" & goto FOUND_PYTHON +if exist "C:\Python312\python.exe" set "PY_EXE=C:\Python312\python.exe" & goto FOUND_PYTHON +if exist "C:\Python313\python.exe" set "PY_EXE=C:\Python313\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python39\python.exe" set "PY_EXE=C:\Program Files\Python39\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python310\python.exe" set "PY_EXE=C:\Program Files\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python311\python.exe" set "PY_EXE=C:\Program Files\Python311\python.exe" & goto FOUND_PYTHON +if exist "C:\Program Files\Python312\python.exe" set "PY_EXE=C:\Program Files\Python312\python.exe" & goto FOUND_PYTHON +if exist "C:\Program Files\Python313\python.exe" set "PY_EXE=C:\Program Files\Python313\python.exe" & goto FOUND_PYTHON :ASK_USER echo. diff --git a/RodSizer_Launcher_MacOS.command b/RodSizer_Launcher_MacOS.command index b33ea75..07f17bc 100755 --- a/RodSizer_Launcher_MacOS.command +++ b/RodSizer_Launcher_MacOS.command @@ -74,65 +74,45 @@ if command -v tesseract &>/dev/null; then fi # ============================================================================= -# STEP 2 — Python 3.11 -# -# TensorFlow 2.x officially supports Python 3.8–3.11. -# Python 3.12+ does NOT have stable TensorFlow wheels yet. -# We therefore require Python 3.11 specifically. +# STEP 2 — Python 3.10+ # ============================================================================= -step "Step 2/4: Checking Python 3.11" +step "Step 2/4: Checking Python" PYTHON_CMD="" -# Helper: find python3.11 by checking all known locations -find_python311() { - # 1. Homebrew-managed path (most reliable — works regardless of PATH) - local brew_prefix - brew_prefix="$(brew --prefix python@3.11 2>/dev/null)" - if [ -n "$brew_prefix" ] && [ -x "$brew_prefix/bin/python3.11" ]; then - echo "$brew_prefix/bin/python3.11"; return - fi - - # 2. Direct binary in PATH - if command -v python3.11 &>/dev/null; then - command -v python3.11; return - fi - - # 3. Common Homebrew Cellar fallback (Apple Silicon) - local cellar_py - cellar_py=$(ls /opt/homebrew/Cellar/python@3.11/*/bin/python3.11 2>/dev/null | sort -V | tail -1) - if [ -x "$cellar_py" ]; then - echo "$cellar_py"; return - fi - # Intel - cellar_py=$(ls /usr/local/Cellar/python@3.11/*/bin/python3.11 2>/dev/null | sort -V | tail -1) - if [ -x "$cellar_py" ]; then - echo "$cellar_py"; return - fi - - # 4. pyenv - if command -v pyenv &>/dev/null; then - local pyenv_ver - pyenv_ver=$(pyenv versions --bare 2>/dev/null | grep "^3\.11\." | sort -V | tail -1) - if [ -n "$pyenv_ver" ]; then - local pyenv_py="$(pyenv root)/versions/$pyenv_ver/bin/python3.11" - [ -x "$pyenv_py" ] && echo "$pyenv_py"; return +# Helper: find a suitable Python (3.10 or newer), preferring newer versions +find_python() { + local candidate + for candidate in python3.13 python3.12 python3.11 python3.10 python3; do + if command -v "$candidate" &>/dev/null; then + if "$candidate" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' &>/dev/null; then + command -v "$candidate"; return + fi fi - fi + done + + # Homebrew-managed fallback (works regardless of PATH) + local brew_prefix ver + for ver in 3.13 3.12 3.11 3.10; do + brew_prefix="$(brew --prefix python@$ver 2>/dev/null)" + if [ -n "$brew_prefix" ] && [ -x "$brew_prefix/bin/python$ver" ]; then + echo "$brew_prefix/bin/python$ver"; return + fi + done } -PYTHON_CMD="$(find_python311)" +PYTHON_CMD="$(find_python)" # If still not found, install via Homebrew then retry if [ -z "$PYTHON_CMD" ]; then - warn "Python 3.11 not found. Installing via Homebrew (needed for TensorFlow)..." - brew install python@3.11 - PYTHON_CMD="$(find_python311)" + warn "Python 3.10+ not found. Installing via Homebrew..." + brew install python@3.12 + PYTHON_CMD="$(find_python)" fi if [ -z "$PYTHON_CMD" ] || [ ! -x "$PYTHON_CMD" ]; then - error "Could not find or install Python 3.11." - error "Please install it manually: brew install python@3.11" + error "Could not find or install Python." + error "Please install it manually: brew install python@3.12" echo "Press Enter to close..."; read -r; exit 1 fi @@ -170,84 +150,21 @@ needs_python_packages() { "$VENV_PYTHON" - <<'PY' &>/dev/null import importlib -for name in ("fastapi", "uvicorn", "tensorflow", "numpy", "cv2", "stardist"): +for name in ("fastapi", "uvicorn", "numpy", "cv2", "skimage", "tifffile", "h5py"): importlib.import_module(name) PY } -install_tensorflow_with_fallback() { - local tf_spec="tensorflow==2.21.0" - - info "Installing TensorFlow..." - if "$VENV_PYTHON" -m pip install "$tf_spec"; then - return 0 - fi - - warn "pip could not resolve tensorflow from the package index." - warn "Retrying TensorFlow install directly from PyPI..." - PIP_CONFIG_FILE=/dev/null "$VENV_PYTHON" -m pip install --index-url https://pypi.org/simple "$tf_spec" && return 0 - - return 1 -} - # Check whether a first-time install is needed if ! needs_python_packages; then - info "Installing Python packages — this may take 5–10 minutes on first run." + info "Installing Python packages — this may take 2–3 minutes on first run." info "Please do NOT close this window." # Upgrade pip / setuptools silently "$VENV_PYTHON" -m pip install --upgrade pip setuptools wheel --quiet - # ── TensorFlow first (sets numpy version constraint everything else must follow) ── - # - # tensorflow-macos was deprecated at 2.13 and forces numpy<2, which conflicts - # with opencv-python-headless >=4.9 and ncempy >=1.15 (both require numpy>=2). - # - # tensorflow >= 2.16 ships a universal wheel that runs natively on Apple - # Silicon (M-series) without a separate macos fork, and is compatible with - # numpy 2.x. tensorflow-metal is still the GPU-acceleration plugin for M-chips. - if ! install_tensorflow_with_fallback; then - error "TensorFlow installation failed." - error "This is usually a package-index or network issue, not a RodSizer code issue." - if [ "$IS_APPLE_SILICON" = true ]; then - error "If needed, try this manually inside backend/.venv:" - error " python -m pip install --index-url https://pypi.org/simple tensorflow==2.21.0" - else - error "Try rerunning later, or install TensorFlow manually inside backend/.venv." - fi - echo "Press Enter to close..."; read -r; exit 1 - fi - - # Note: tensorflow-metal is NOT installed because v1.2.0 is incompatible - # with tensorflow >=2.16 (dlopen fails on _pywrap_tensorflow_internal.so). - # Modern TensorFlow already runs natively on Apple Silicon without it. - - if ! "$VENV_PYTHON" -c "import tensorflow" &>/dev/null 2>&1; then - error "TensorFlow import failed after installation." - error "Try deleting backend/.venv and relaunching, or check network connectivity." - echo "Press Enter to close..."; read -r; exit 1 - fi - - # ── Remaining packages (numpy version is now fixed by TF above) ───────────── - TMP_REQ=$(mktemp /tmp/rodsizer_req_XXXX.txt) - grep -v "^tensorflow" "$BACKEND_DIR/requirements.txt" > "$TMP_REQ" - - if ! "$VENV_PYTHON" -m pip install -r "$TMP_REQ" --quiet; then + if ! "$VENV_PYTHON" -m pip install -r "$BACKEND_DIR/requirements.txt" --quiet; then error "Failed to install some packages. Check the output above." - rm -f "$TMP_REQ" - echo "Press Enter to close..."; read -r; exit 1 - fi - rm -f "$TMP_REQ" - - # ── Verify tensorflow is importable ─────────────────────────────────────── - if ! "$VENV_PYTHON" -c "import tensorflow" &>/dev/null 2>&1; then - error "TensorFlow installation failed." - error "Try running this manually inside the venv:" - if [ "$IS_APPLE_SILICON" = true ]; then - error " python -m pip install --index-url https://pypi.org/simple tensorflow==2.21.0" - else - error " python -m pip install tensorflow==2.21.0" - fi echo "Press Enter to close..."; read -r; exit 1 fi diff --git a/RodSizer_Launcher_Windows.bat b/RodSizer_Launcher_Windows.bat index 1273e89..d99c039 100644 --- a/RodSizer_Launcher_Windows.bat +++ b/RodSizer_Launcher_Windows.bat @@ -29,9 +29,13 @@ REM --- 3. Try Auto-Detect Common Paths --- if exist "C:\Python39\python.exe" set "PY_EXE=C:\Python39\python.exe" & goto FOUND_PYTHON if exist "C:\Python310\python.exe" set "PY_EXE=C:\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Python311\python.exe" set "PY_EXE=C:\Python311\python.exe" & goto FOUND_PYTHON +if exist "C:\Python312\python.exe" set "PY_EXE=C:\Python312\python.exe" & goto FOUND_PYTHON +if exist "C:\Python313\python.exe" set "PY_EXE=C:\Python313\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python39\python.exe" set "PY_EXE=C:\Program Files\Python39\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python310\python.exe" set "PY_EXE=C:\Program Files\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python311\python.exe" set "PY_EXE=C:\Program Files\Python311\python.exe" & goto FOUND_PYTHON +if exist "C:\Program Files\Python312\python.exe" set "PY_EXE=C:\Program Files\Python312\python.exe" & goto FOUND_PYTHON +if exist "C:\Program Files\Python313\python.exe" set "PY_EXE=C:\Program Files\Python313\python.exe" & goto FOUND_PYTHON REM --- 4. User Input Fallback --- :ASK_USER diff --git a/backend/requirements.txt b/backend/requirements.txt index ee2d12c..5e507ac 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -13,12 +13,8 @@ tifffile openpyxl h5py -# --- Heavy deps kept for possible future ML detection (NOT used right now) --- -# stardist + tensorflow are the main reason the FIRST-TIME setup is slow (~5-10 -# min) and large (~1 GB+): they pull in a full TensorFlow build. The current -# detection pipeline is K-means + distance-transform watershed and does NOT use -# them (they are imported nowhere in the hot path). They are kept on purpose in -# case StarDist-based detection is added later. Remove these two lines if you -# want a much faster, smaller install and don't plan to use StarDist. -stardist -tensorflow +# NOTE: stardist + tensorflow were removed (July 2026). Nothing in the current +# pipeline (K-means + distance-transform watershed) uses them, and they were the +# main reason first-time setup was slow (~5-10 min, ~1 GB) and failed on some +# machines (no TensorFlow wheels for newer Python versions). If StarDist-based +# detection is ever added, re-add: stardist and tensorflow From 1bb10fb8bb2753057bab60cf087122b818ebf296 Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Mon, 20 Jul 2026 12:17:39 -0500 Subject: [PATCH 02/10] Folder analysis: restore sortable batch ledger table Re-implements the column sorting (click header to toggle asc/desc with arrow indicator) that existed in Arda's local copy, adapted to the new XSS-safe textContent row rendering. Co-Authored-By: Claude Fable 5 --- frontend/folder_analysis.html | 53 ++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/frontend/folder_analysis.html b/frontend/folder_analysis.html index d383a03..2c019ae 100644 --- a/frontend/folder_analysis.html +++ b/frontend/folder_analysis.html @@ -278,11 +278,11 @@

Batch Ledger

- - - - - + + + + + @@ -308,6 +308,8 @@

Batch Ledger

dlBtn.href = `/folders/${encodeURIComponent(folderName)}/export_aggregate`; let lengthChart, widthChart; + let currentData = []; + let currentSort = { column: null, direction: 'asc' }; async function loadAggregateData() { try { @@ -338,14 +340,41 @@

Batch Ledger

document.getElementById('meanAR').textContent = data.stats.mean_ar || "-"; // Charts - const items = data.data; + currentData = data.data || []; + const items = currentData; const lengths = items.map(d => d.length_nm); const widths = items.map(d => d.width_nm); lengthChart = createHistogram('lengthChart', 'Length Distribution', lengths); widthChart = createHistogram('widthChart', 'Width Distribution', widths); - // Table + renderTable(); + } + + function renderTable() { + let items = currentData.slice(); + + if (currentSort.column) { + const col = currentSort.column; + const dir = currentSort.direction === 'asc' ? 1 : -1; + items.sort((a, b) => { + const va = a[col], vb = b[col]; + if (typeof va === 'string' || typeof vb === 'string') { + return String(va ?? '').localeCompare(String(vb ?? '')) * dir; + } + return ((va ?? 0) - (vb ?? 0)) * dir; + }); + } + + // Update header arrows + document.querySelectorAll('#dataTable .sort-arrow').forEach(span => { + if (span.dataset.col === currentSort.column) { + span.textContent = currentSort.direction === 'asc' ? '▲' : '▼'; + } else { + span.textContent = ''; + } + }); + const tbody = document.querySelector('#dataTable tbody'); tbody.innerHTML = ''; items.forEach(item => { @@ -366,6 +395,16 @@

Batch Ledger

}); } + function sortTable(column) { + if (currentSort.column === column) { + currentSort.direction = currentSort.direction === 'asc' ? 'desc' : 'asc'; + } else { + currentSort.column = column; + currentSort.direction = 'asc'; + } + renderTable(); + } + function createHistogram(canvasId, label, data) { const ctx = document.getElementById(canvasId).getContext('2d'); if (!data.length) return; From 2f606f6dd7ac32b488df7857766e999900868166 Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Mon, 20 Jul 2026 12:19:18 -0500 Subject: [PATCH 03/10] Pin dependency versions to a known-good set (Python 3.10-3.12) Unpinned installs meant each computer resolved different library versions - a major source of machine-to-machine differences. Versions pinned from the tested working environment (Python 3.12.5, macOS). Launchers now target Python 3.10-3.12 to match the pinned numpy. Co-Authored-By: Claude Fable 5 --- README.md | 2 +- RodSizer_DEBUG_Windows.bat | 2 -- RodSizer_Launcher_MacOS.command | 14 ++++++++------ RodSizer_Launcher_Windows.bat | 2 -- backend/requirements.txt | 32 ++++++++++++++++++-------------- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9ab8ac5..ab43559 100644 --- a/README.md +++ b/README.md @@ -84,4 +84,4 @@ Group website: https://murphy-group.chemistry.illinois.edu/ ## Requirements - macOS or Windows -- Python 3 installed (standard on most Macs, or downloadable from `python.org`) +- Python 3.10-3.12 (the Mac launcher installs it automatically if missing; on Windows download from `python.org`) diff --git a/RodSizer_DEBUG_Windows.bat b/RodSizer_DEBUG_Windows.bat index 214a8ec..e55bd80 100644 --- a/RodSizer_DEBUG_Windows.bat +++ b/RodSizer_DEBUG_Windows.bat @@ -33,12 +33,10 @@ if exist "C:\Python39\python.exe" set "PY_EXE=C:\Python39\python.exe" & goto FOU if exist "C:\Python310\python.exe" set "PY_EXE=C:\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Python311\python.exe" set "PY_EXE=C:\Python311\python.exe" & goto FOUND_PYTHON if exist "C:\Python312\python.exe" set "PY_EXE=C:\Python312\python.exe" & goto FOUND_PYTHON -if exist "C:\Python313\python.exe" set "PY_EXE=C:\Python313\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python39\python.exe" set "PY_EXE=C:\Program Files\Python39\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python310\python.exe" set "PY_EXE=C:\Program Files\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python311\python.exe" set "PY_EXE=C:\Program Files\Python311\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python312\python.exe" set "PY_EXE=C:\Program Files\Python312\python.exe" & goto FOUND_PYTHON -if exist "C:\Program Files\Python313\python.exe" set "PY_EXE=C:\Program Files\Python313\python.exe" & goto FOUND_PYTHON :ASK_USER echo. diff --git a/RodSizer_Launcher_MacOS.command b/RodSizer_Launcher_MacOS.command index 07f17bc..b5aa3e0 100755 --- a/RodSizer_Launcher_MacOS.command +++ b/RodSizer_Launcher_MacOS.command @@ -74,18 +74,20 @@ if command -v tesseract &>/dev/null; then fi # ============================================================================= -# STEP 2 — Python 3.10+ +# STEP 2 — Python 3.10-3.12 +# (the pinned dependency set in backend/requirements.txt targets these +# versions; 3.13+ lacks wheels for the pinned numpy) # ============================================================================= step "Step 2/4: Checking Python" PYTHON_CMD="" -# Helper: find a suitable Python (3.10 or newer), preferring newer versions +# Helper: find a suitable Python (3.10-3.12), preferring newer versions find_python() { local candidate - for candidate in python3.13 python3.12 python3.11 python3.10 python3; do + for candidate in python3.12 python3.11 python3.10 python3; do if command -v "$candidate" &>/dev/null; then - if "$candidate" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)' &>/dev/null; then + if "$candidate" -c 'import sys; sys.exit(0 if (3, 10) <= sys.version_info[:2] <= (3, 12) else 1)' &>/dev/null; then command -v "$candidate"; return fi fi @@ -93,7 +95,7 @@ find_python() { # Homebrew-managed fallback (works regardless of PATH) local brew_prefix ver - for ver in 3.13 3.12 3.11 3.10; do + for ver in 3.12 3.11 3.10; do brew_prefix="$(brew --prefix python@$ver 2>/dev/null)" if [ -n "$brew_prefix" ] && [ -x "$brew_prefix/bin/python$ver" ]; then echo "$brew_prefix/bin/python$ver"; return @@ -105,7 +107,7 @@ PYTHON_CMD="$(find_python)" # If still not found, install via Homebrew then retry if [ -z "$PYTHON_CMD" ]; then - warn "Python 3.10+ not found. Installing via Homebrew..." + warn "Python 3.10-3.12 not found. Installing 3.12 via Homebrew..." brew install python@3.12 PYTHON_CMD="$(find_python)" fi diff --git a/RodSizer_Launcher_Windows.bat b/RodSizer_Launcher_Windows.bat index d99c039..8a5b50e 100644 --- a/RodSizer_Launcher_Windows.bat +++ b/RodSizer_Launcher_Windows.bat @@ -30,12 +30,10 @@ if exist "C:\Python39\python.exe" set "PY_EXE=C:\Python39\python.exe" & goto FOU if exist "C:\Python310\python.exe" set "PY_EXE=C:\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Python311\python.exe" set "PY_EXE=C:\Python311\python.exe" & goto FOUND_PYTHON if exist "C:\Python312\python.exe" set "PY_EXE=C:\Python312\python.exe" & goto FOUND_PYTHON -if exist "C:\Python313\python.exe" set "PY_EXE=C:\Python313\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python39\python.exe" set "PY_EXE=C:\Program Files\Python39\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python310\python.exe" set "PY_EXE=C:\Program Files\Python310\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python311\python.exe" set "PY_EXE=C:\Program Files\Python311\python.exe" & goto FOUND_PYTHON if exist "C:\Program Files\Python312\python.exe" set "PY_EXE=C:\Program Files\Python312\python.exe" & goto FOUND_PYTHON -if exist "C:\Program Files\Python313\python.exe" set "PY_EXE=C:\Program Files\Python313\python.exe" & goto FOUND_PYTHON REM --- 4. User Input Fallback --- :ASK_USER diff --git a/backend/requirements.txt b/backend/requirements.txt index 5e507ac..1c30e9a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,17 +1,21 @@ -fastapi -uvicorn -python-multipart -numpy -opencv-python-headless -scikit-image -ncempy -matplotlib -pandas -Pillow -pytesseract -tifffile -openpyxl -h5py +# Pinned, known-good dependency set (tested on Python 3.12, July 2026). +# Pinning exact versions keeps every computer running identical code — +# unpinned installs were a major source of "works here, not there" bugs. +# Python 3.10-3.12 required (numpy 1.26 has no wheels for 3.13+). +fastapi==0.122.0 +uvicorn==0.38.0 +python-multipart==0.0.20 +numpy==1.26.4 +opencv-python-headless==4.12.0.88 +scikit-image==0.25.2 +ncempy==1.14 +matplotlib==3.10.7 +pandas==2.3.3 +Pillow==12.0.0 +pytesseract==0.3.13 +tifffile==2025.10.16 +openpyxl==3.1.5 +h5py==3.15.1 # NOTE: stardist + tensorflow were removed (July 2026). Nothing in the current # pipeline (K-means + distance-transform watershed) uses them, and they were the From eb1ac433573c2a951746878ce0f04396afa1dddd Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Wed, 5 Aug 2026 16:58:01 -0500 Subject: [PATCH 04/10] Add results cleanup, richer statistics, and batch comparison view - Results housekeeping: Excel export temps now self-delete after download, deleting a folder removes its cached results, and a startup sweep purges results whose source upload no longer exists (freed 362 MB / 541 files on first run). - Statistics: new compute_summary_stats() adds median, D10/D90, CV% and rod yield (AR >= 1.5) to Excel exports, the folder aggregate API, and a Detailed Statistics table on the folder analysis page. - New Compare Batches page (/compare): overlay normalized length / width / aspect-ratio distributions of 2-4 folders with side-by-side extended statistics; linked from the dashboard. Co-Authored-By: Claude Fable 5 --- backend/main.py | 97 +++++++-- backend/processing.py | 55 ++++- frontend/compare.html | 396 ++++++++++++++++++++++++++++++++++ frontend/folder_analysis.html | 27 +++ frontend/index.html | 5 +- 5 files changed, 556 insertions(+), 24 deletions(-) create mode 100644 frontend/compare.html diff --git a/backend/main.py b/backend/main.py index 9e0c1f6..2536307 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,5 @@ from fastapi import FastAPI, UploadFile, File, HTTPException, Form, Query, BackgroundTasks +from starlette.background import BackgroundTask from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles @@ -183,6 +184,69 @@ def _find_input_and_calibration_source(image_id: str): def _frontend_page(name: str) -> FileResponse: return FileResponse(FRONTEND_DIR / name, headers=_NO_CACHE_HEADERS) + +# --- Results housekeeping --- + +def _safe_unlink(path: Path): + try: + path.unlink() + except OSError: + pass + + +def _valid_upload_ids() -> set: + """Stems of every uploaded file (results are named '_').""" + ids = set() + for p in UPLOAD_DIR.rglob("*"): + if p.is_file() and ".analysis_cache" not in p.parts: + ids.add(p.stem) + return ids + + +def _delete_results_for_ids(image_ids) -> int: + removed = 0 + for res_file in RESULTS_DIR.iterdir(): + if not res_file.is_file(): + continue + if any(res_file.name.startswith(image_id + "_") for image_id in image_ids): + _safe_unlink(res_file) + removed += 1 + return removed + + +def cleanup_results_dir() -> dict: + """Remove result files whose source upload no longer exists, plus stale + export temp files (>1 day old). Never touches uploads/.""" + import time + valid_ids = _valid_upload_ids() + removed_orphans = 0 + removed_exports = 0 + now = time.time() + for res_file in RESULTS_DIR.iterdir(): + if not res_file.is_file() or res_file.name.startswith("."): + continue + if res_file.name.startswith("export_"): + # Export temps now self-delete after download; sweep old leftovers. + if now - res_file.stat().st_mtime > 86400: + _safe_unlink(res_file) + removed_exports += 1 + continue + if not any(res_file.name.startswith(image_id + "_") for image_id in valid_ids): + _safe_unlink(res_file) + removed_orphans += 1 + return {"orphans": removed_orphans, "stale_exports": removed_exports} + + +@app.on_event("startup") +async def _startup_cleanup(): + try: + summary = cleanup_results_dir() + if summary["orphans"] or summary["stale_exports"]: + print(f"Results cleanup: removed {summary['orphans']} orphaned files, " + f"{summary['stale_exports']} stale exports") + except Exception as e: + print(f"Results cleanup skipped: {e}") + @app.get("/") async def read_index(): return _frontend_page("index.html") @@ -195,6 +259,10 @@ async def read_analysis(): async def read_folder_analysis(): return _frontend_page("folder_analysis.html") +@app.get("/compare") +async def read_compare(): + return _frontend_page("compare.html") + # --- Folder Management --- @app.post("/folders") @@ -261,14 +329,16 @@ async def delete_folder(folder_name: str): if not folder_path.exists() or not folder_path.is_dir(): raise HTTPException(status_code=404, detail="Folder not found") + # Collect the folder's image ids first so their results can be removed too + folder_image_ids = {p.stem for p in folder_path.rglob("*") + if p.is_file() and ".analysis_cache" not in p.parts} + # Delete folder and contents shutil.rmtree(folder_path) - - # Also clean up results for images that were in this folder? - # Since we don't strictly track which result belongs to which folder in the filename (only ID), - # this is tricky unless we scan the deleted files. - # For now, let's just delete the upload folder. Orphaned results are harmless but take space. - + + if folder_image_ids: + _delete_results_for_ids(folder_image_ids) + return {"status": "success", "message": "Folder deleted"} except HTTPException: raise @@ -372,9 +442,12 @@ def get_stat(col): "mean_ar": f"{ar_m} ± {ar_s}" } + extended = _processing_function("compute_summary_stats")(combined_data) if combined_data else [] + return { "data": combined_data, "stats": stats, + "extended_stats": extended, "file_count": len(files) } @@ -416,7 +489,8 @@ async def export_aggregate_folder(folder_name: str): return FileResponse( path=temp_path, filename=f"{safe_folder_name}_analysis_report.xlsx", - media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + background=BackgroundTask(_safe_unlink, temp_path) ) except Exception as e: @@ -718,11 +792,7 @@ async def export_data(req: ExportRequest): temp_path = RESULTS_DIR / temp_name _processing_function("save_results_to_excel")(filtered_results, temp_path) - - # We should use BackgroundTasks to clean up, but simpler here: - # FileResponse can delete after? using background. - # But allow it to persist is fine for now (results dir is cache). - + # Sanitize the suggested download filename — it originates from the # uploaded filename and flows into a response header. raw_name = data.get("filename") or req.image_id @@ -731,7 +801,8 @@ async def export_data(req: ExportRequest): return FileResponse( path=temp_path, filename=f"{safe_download_stem}_detected.xlsx", - media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + background=BackgroundTask(_safe_unlink, temp_path) ) except HTTPException: diff --git a/backend/processing.py b/backend/processing.py index 01373f9..a394e1a 100644 --- a/backend/processing.py +++ b/backend/processing.py @@ -9,6 +9,50 @@ import ncempy.io as nio from autodetect_utils import image_kmeans, ruecs, dilmarkers, split_clump +# Aspect-ratio threshold above which a particle is counted as a rod in the +# "Rod yield" statistic. Reported alongside the value so readers see the cutoff. +ROD_AR_THRESHOLD = 1.5 + + +def compute_summary_stats(results): + """Distribution statistics for a set of particles. + + Returns a list of {"Metric", "Value"} rows: count, then per dimension + (length, width, aspect ratio) the mean ± SD, median, D10/D90 percentiles + and coefficient of variation, plus the rod yield at ROD_AR_THRESHOLD. + """ + df = pd.DataFrame(results) + if df.empty: + return [] + + rows = [{"Metric": "Count", "Value": len(df)}] + + dims = [("length_nm", "Length (nm)"), + ("width_nm", "Width (nm)"), + ("aspect_ratio", "Aspect Ratio")] + for col, label in dims: + if col not in df: + continue + vals = pd.to_numeric(df[col], errors="coerce").dropna() + if vals.empty: + continue + mean, std = vals.mean(), vals.std() + cv = (std / mean * 100) if mean else 0 + rows.append({"Metric": f"Mean {label}", "Value": f"{mean:.1f} ± {std:.1f}"}) + rows.append({"Metric": f"Median {label}", "Value": f"{vals.median():.1f}"}) + rows.append({"Metric": f"D10 {label}", "Value": f"{vals.quantile(0.10):.1f}"}) + rows.append({"Metric": f"D90 {label}", "Value": f"{vals.quantile(0.90):.1f}"}) + rows.append({"Metric": f"CV {label} (%)", "Value": f"{cv:.1f}"}) + + if "aspect_ratio" in df: + ar = pd.to_numeric(df["aspect_ratio"], errors="coerce").dropna() + if not ar.empty: + yield_pct = (ar >= ROD_AR_THRESHOLD).mean() * 100 + rows.append({"Metric": f"Rod yield, AR >= {ROD_AR_THRESHOLD} (%)", + "Value": f"{yield_pct:.1f}"}) + return rows + + def save_results_to_excel(results, output_path): """ Saves results to an Excel file with 'Statistics' and 'Data' sheets. @@ -22,16 +66,7 @@ def save_results_to_excel(results, output_path): cols_to_drop = [c for c in ["contour", "contour_full"] if c in df_export.columns] df_export = df_export.drop(columns=cols_to_drop) - # Calculate Stats for Excel - s_mean = df_export.mean(numeric_only=True).round(1) - s_std = df_export.std(numeric_only=True).round(1) - - stats_rows = [] - stats_rows.append({"Metric": "Count", "Value": len(df_export)}) - stats_rows.append({"Metric": "Mean Length (nm)", "Value": f"{s_mean.get('length_nm', 0)} ± {s_std.get('length_nm', 0)}"}) - stats_rows.append({"Metric": "Mean Width (nm)", "Value": f"{s_mean.get('width_nm', 0)} ± {s_std.get('width_nm', 0)}"}) - stats_rows.append({"Metric": "Mean AR", "Value": f"{s_mean.get('aspect_ratio', 0)} ± {s_std.get('aspect_ratio', 0)}"}) - stats_df = pd.DataFrame(stats_rows) + stats_df = pd.DataFrame(compute_summary_stats(results)) # Save to Excel try: diff --git a/frontend/compare.html b/frontend/compare.html new file mode 100644 index 0000000..4ab8af6 --- /dev/null +++ b/frontend/compare.html @@ -0,0 +1,396 @@ + + + + + + + Compare Batches - RodSizer + + + + + + +
+ ← Back to Folders +

Compare Batches

+

+ Overlay the size distributions of two or more synthesis folders. + Only folders with saved analysis data ("Add to Folder Analysis") can be compared. +

+ +
+
Loading folders…
+ + Select 2–4 folders +
+ +

+ +
+

Statistics

+
+
Source ImageIDL (nm)W (nm)ARSource Image ID L (nm) W (nm) AR
+ + +
+ + +
+

Length distribution (nm)

+
+
+
+

Width distribution (nm)

+
+
+
+

Aspect ratio distribution

+
+
+ + + + + + + diff --git a/frontend/folder_analysis.html b/frontend/folder_analysis.html index 2c019ae..89bb986 100644 --- a/frontend/folder_analysis.html +++ b/frontend/folder_analysis.html @@ -271,6 +271,17 @@

Folder Analysis Guide

+ +

Detailed Statistics

+ + + + + + + + +
MetricValue

Batch Ledger

@@ -339,6 +350,22 @@

Batch Ledger

document.getElementById('meanWidth').textContent = data.stats.mean_width || "-"; document.getElementById('meanAR').textContent = data.stats.mean_ar || "-"; + // Detailed statistics table (median, D10/D90, CV%, rod yield) + const statsBody = document.querySelector('#extendedStatsTable tbody'); + statsBody.innerHTML = ''; + (data.extended_stats || []).forEach(row => { + const tr = document.createElement('tr'); + const tdMetric = document.createElement('td'); + tdMetric.textContent = row.Metric; + tdMetric.style.cssText = 'padding: 5px 10px; border-bottom: 1px solid #eee;'; + const tdValue = document.createElement('td'); + tdValue.textContent = row.Value; + tdValue.style.cssText = 'padding: 5px 10px; border-bottom: 1px solid #eee; text-align: right; font-variant-numeric: tabular-nums;'; + tr.appendChild(tdMetric); + tr.appendChild(tdValue); + statsBody.appendChild(tr); + }); + // Charts currentData = data.data || []; const items = currentData; diff --git a/frontend/index.html b/frontend/index.html index ff81e99..04e4c48 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -414,7 +414,10 @@

About This App

Folders

- +
+ + +
From a09c571677093fc18612f3d0cc965dc987bede44 Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Wed, 5 Aug 2026 17:16:43 -0500 Subject: [PATCH 05/10] Replace browser confirm() popups with two-step inline delete buttons Embedded browser panes (e.g. IDE previews) block native confirm() dialogs, which made image and folder deletion silently do nothing. First click arms the button ('Delete?' / 'Really delete?'), a second click within 4s confirms; it disarms automatically otherwise. Co-Authored-By: Claude Fable 5 --- frontend/index.html | 67 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/frontend/index.html b/frontend/index.html index 04e4c48..633db8d 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -300,6 +300,23 @@ opacity: 1; } + /* First click arms the button; second click confirms. */ + .delete-btn.armed { + opacity: 1; + width: auto; + border-radius: 14px; + padding: 0 12px 2px; + font-size: 13px; + font-weight: 600; + background: #dc3545; + } + + .btn-danger.armed { + background: #a71d2a; + outline: 2px solid #dc3545; + outline-offset: 1px; + } + /* Overlay */ .processing-overlay { position: absolute; @@ -448,7 +465,7 @@

Folder Name

onclick="openUploadHelp()">Instructions - + @@ -641,7 +658,7 @@

Upload Instructions

deleteBtn.style.padding = '5px 10px'; deleteBtn.style.fontSize = '12px'; deleteBtn.textContent = 'Delete'; - deleteBtn.addEventListener('click', () => deleteFolderByName(folder)); + deleteBtn.addEventListener('click', (ev) => deleteFolderByName(folder, ev)); actionsCell.appendChild(renameBtn); actionsCell.appendChild(deleteBtn); @@ -694,10 +711,35 @@

Upload Instructions

// --- Folder Operations --- - async function deleteCurrentFolder() { + // Two-step in-page confirmation (replaces the browser confirm() popup, + // which some embedded browsers block). First click arms the button and + // shows the confirm label; a second click within 4s runs the action. + function armButton(btn, armedLabel, action) { + if (btn.dataset.armed) { + clearTimeout(btn._disarmTimer); + disarmButton(btn); + action(); + return; + } + btn.dataset.armed = '1'; + btn.dataset.origLabel = btn.textContent; + btn.textContent = armedLabel; + btn.classList.add('armed'); + btn._disarmTimer = setTimeout(() => disarmButton(btn), 4000); + } + + function disarmButton(btn) { + if (!btn.dataset.armed) return; + delete btn.dataset.armed; + btn.textContent = btn.dataset.origLabel || btn.textContent; + delete btn.dataset.origLabel; + btn.classList.remove('armed'); + } + + async function deleteCurrentFolder(event) { if (!currentFolder) return; - if (!confirm(`Permanently delete folder "${currentFolder}"?`)) return; - await performDelete(currentFolder); + const btn = event.target.closest('button'); + armButton(btn, 'Really delete?', () => performDelete(currentFolder)); } async function renameCurrentFolder() { @@ -705,9 +747,9 @@

Upload Instructions

await renameFolder(currentFolder); } - async function deleteFolderByName(name) { - if (!confirm(`Permanently delete folder "${name}"?`)) return; - await performDelete(name); + async function deleteFolderByName(name, event) { + const btn = event.target.closest('button'); + armButton(btn, 'Really delete?', () => performDelete(name)); } async function performDelete(name) { @@ -816,7 +858,10 @@

Upload Instructions

const delBtn = document.createElement('button'); delBtn.className = 'delete-btn'; delBtn.textContent = '×'; - delBtn.addEventListener('click', (ev) => deleteImage(img.id, ev)); + delBtn.addEventListener('click', (ev) => { + ev.stopPropagation(); + armButton(delBtn, 'Delete?', () => deleteImage(img.id)); + }); imgWrap.appendChild(delBtn); const info = document.createElement('div'); @@ -867,9 +912,7 @@

Upload Instructions

} } - async function deleteImage(id, event) { - event.stopPropagation(); - if (!confirm('Delete this image?')) return; + async function deleteImage(id) { try { const response = await fetch(`/images/${id}`, { method: 'DELETE' }); if (response.ok) loadImages(); From 50ff705f43c8515eddcb0f48d2615497e5be04ef Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Wed, 5 Aug 2026 17:50:22 -0500 Subject: [PATCH 06/10] Fix upload race that lost .dm3 calibration for half of batch uploads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calibration lookup ran inside the per-file save loop, so an image saved before its .dm3 partner in the same batch never found it — the decision was frozen into the queued background task. Lookup now happens after the entire batch is on disk. Co-Authored-By: Claude Fable 5 --- backend/main.py | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/backend/main.py b/backend/main.py index 2536307..1345b4e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -502,6 +502,7 @@ async def export_aggregate_folder(folder_name: str): @app.post("/upload") async def upload_images(background_tasks: BackgroundTasks, folder: str = Form(None), files: List[UploadFile] = File(...)): uploaded_files = [] + saved_paths = [] try: if len(files) > MAX_UPLOAD_FILES: raise HTTPException(status_code=400, detail=f"Too many files (max {MAX_UPLOAD_FILES} per request)") @@ -550,36 +551,33 @@ async def upload_images(background_tasks: BackgroundTasks, folder: str = Form(No # 1. Generate Immediate Preview (Sync) # This ensures the user sees something right away _processing_function("generate_preview")(file_path, RESULTS_DIR) - + uploaded_files.append({ - "id": file_path.stem, + "id": file_path.stem, "filename": safe_filename, "status": "processing" }) - - # 2. Queue Heavy Processing (Background) - # Find matching calibration file (.dm3/.dm4) - search_dir = file_path.parent + saved_paths.append(file_path) + + # 2. Queue Heavy Processing (Background) + # IMPORTANT: calibration lookup happens only after EVERY file in the + # batch is saved. Doing it inside the save loop caused a race: an image + # saved before its .dm3 partner never got calibrated (upload order is + # effectively random, so ~half of paired uploads lost their scale). + def _original_stem(name: str) -> str: + if len(name) > 37 and name[36] == '_': + return Path(name[37:]).stem + return Path(name).stem + + for file_path in saved_paths: calibration_source_path = None - original_stem = None - - if len(save_name) > 37 and save_name[36] == '_': - original_stem = Path(save_name[37:]).stem - else: - original_stem = Path(save_name).stem - - for f in search_dir.glob("*"): - if f.suffix.lower() in ['.dm3', '.dm4', '.emd']: - dm3_stem = None - if len(f.name) > 37 and f.name[36] == '_': - dm3_stem = Path(f.name[37:]).stem - else: - dm3_stem = f.stem - if dm3_stem == original_stem: + target_stem = _original_stem(file_path.name) + for f in file_path.parent.glob("*"): + if f.suffix.lower() in ['.dm3', '.dm4', '.emd'] and f != file_path: + if _original_stem(f.name) == target_stem: calibration_source_path = f break - - # Add to background tasks + background_tasks.add_task( _processing_function("process_image"), file_path, From 88e953cba33cb76e9b7d0657b45ae67941c5f84e Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Wed, 5 Aug 2026 17:50:22 -0500 Subject: [PATCH 07/10] Split side-by-side rod rafts using intensity seams (recursive) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parallel rods touching along their full length have no binary 'neck', so the distance-transform watershed measured whole rafts as one giant particle (widths 3-5x the true rod width). New split path: - Estimate single-rod width from the narrowest substantial regions. - Regions much wider than that (even convex ones that pass the solidity test) get split_clump_intensity: re-threshold the clump interior at a darker cutoff so the lighter contact seams drop out, leaving one core per rod; cores seed a watershed over the smoothed intensity. - split_raft recurses on still-wide children with LOCAL contrast, which resolves rods much lighter than their raft neighbours. - Accept a split only if most pieces are single-rod-width — rejects bogus cuts through single rods with internal diffraction bands. Worst raft image: 19 particles (9 fused monsters, widths to 175 nm) -> 67 particles, zero width outliers, median width unchanged. Co-Authored-By: Claude Fable 5 --- backend/autodetect_utils.py | 105 ++++++++++++++++++++++++++++++++++++ backend/processing.py | 84 ++++++++++++++++++++++++----- 2 files changed, 175 insertions(+), 14 deletions(-) diff --git a/backend/autodetect_utils.py b/backend/autodetect_utils.py index 00237db..f3549af 100644 --- a/backend/autodetect_utils.py +++ b/backend/autodetect_utils.py @@ -432,6 +432,111 @@ def dilmarkers(markers, original_shape): return dilated_markers, bg_image # returning image as overlay placeholder +def split_clump_intensity(gray_crop, crop_bool, min_marker_area, expected_width_px=None): + """Split fused SIDE-BY-SIDE rods using brightness seams in the raw image. + + Parallel rods touching along their full length merge into one solid blob in + the binary mask. The distance transform of that blob has no "neck", so the + standard ``split_clump`` watershed sees a single object. In the raw TEM + image, however, the contact line between adjacent rods is a slightly + LIGHTER seam than the dark rod cores. This routine re-thresholds the clump + interior at a darker cutoff so those seams drop out, leaving one connected + "core" per rod; the cores then seed a watershed over the smoothed intensity + that assigns every clump pixel to its rod. + + Args: + gray_crop: 2-D uint8 crop of the ORIGINAL grayscale image (rods dark). + crop_bool: matching 2-D boolean mask of the clump. + min_marker_area: minimum area (px) for final fragments. + expected_width_px: estimated single-rod width; used to size the core + filter so texture specks never become seeds. + + Returns: + List of >= 2 boolean masks if a split was found, else None (caller + should fall back to the distance-transform splitter). + """ + crop_bool = np.ascontiguousarray(crop_bool, dtype=bool) + if not crop_bool.any() or gray_crop.shape != crop_bool.shape or gray_crop.ndim != 2: + return None + + vals = gray_crop[crop_bool] + if vals.size < 4 * min_marker_area: + return None + + smooth = ndi.gaussian_filter(gray_crop.astype(np.float32), 2.0) + + # A rod core (dark, elongated) is far larger than noise/texture blobs. + if expected_width_px and expected_width_px > 4: + core_floor = max(min_marker_area, int(0.5 * expected_width_px ** 2)) + else: + core_floor = max(min_marker_area, 100) + + # Sweep a few cutoffs; keep the one that resolves the most rod-sized cores. + best_markers, best_count = None, 0 + for pct in (45, 55, 65): + t = np.percentile(vals, pct) + cores = (smooth < t) & crop_bool + cores = morphology.binary_opening(cores, morphology.disk(3)) + lab, n = ndi.label(cores) + if n == 0: + continue + sizes = ndi.sum(cores, lab, index=np.arange(1, n + 1)) + kept_ids = np.flatnonzero(np.asarray(sizes) >= core_floor) + 1 + if len(kept_ids) > best_count: + remap = np.zeros(n + 1, dtype=np.int32) + remap[kept_ids] = np.arange(1, len(kept_ids) + 1) + best_markers, best_count = remap[lab], len(kept_ids) + + if best_count < 2: + return None + + # Outside the mask counts as maximally bright so basins never leak out. + inside = np.where(crop_bool, smooth, 255.0) + ws = watershed(inside, best_markers, mask=crop_bool) + + masks = [] + for label_idx in range(1, best_count + 1): + mask = ws == label_idx + if int(mask.sum()) >= min_marker_area: + masks.append(mask) + + return masks if len(masks) >= 2 else None + + +def _mask_short_side(mask): + cnts, _ = cv2.findContours(mask.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + if not cnts: + return 0.0 + (_, (rw, rh), _) = cv2.minAreaRect(max(cnts, key=cv2.contourArea)) + return float(min(rw, rh)) + + +def split_raft(gray_crop, crop_bool, min_marker_area, expected_width_px, depth=0): + """Recursively split a raft of side-by-side rods via intensity seams. + + The first pass thresholds with the WHOLE clump's intensity statistics, so + rods much lighter than their neighbours may not seed a core and stay fused + with a neighbour. Each still-too-wide child is therefore re-split using its + own local statistics (recursion adapts the threshold to the lighter rods). + + Returns a flat list of masks (may be [crop_bool] when nothing splits). + """ + children = split_clump_intensity(gray_crop, crop_bool, min_marker_area, + expected_width_px=expected_width_px) + if not children: + return [crop_bool] + + result = [] + for child in children: + if (depth < 2 and expected_width_px + and _mask_short_side(child) >= 1.6 * expected_width_px): + result.extend(split_raft(gray_crop, child, min_marker_area, + expected_width_px, depth + 1)) + else: + result.append(child) + return result + + def split_clump(crop_bool, min_marker_area, separation_strength=0): """ Split a fused / low-solidity binary region into individual convex particles diff --git a/backend/processing.py b/backend/processing.py index a394e1a..fc52975 100644 --- a/backend/processing.py +++ b/backend/processing.py @@ -7,7 +7,7 @@ import math from scipy import ndimage as ndi import ncempy.io as nio -from autodetect_utils import image_kmeans, ruecs, dilmarkers, split_clump +from autodetect_utils import image_kmeans, ruecs, dilmarkers, split_clump, split_raft # Aspect-ratio threshold above which a particle is counted as a rod in the # "Rod yield" statistic. Reported alongside the value so readers see the cutoff. @@ -447,30 +447,86 @@ def read_dm3_pixel_size(dm3_path): split_labels = np.zeros_like(labels, dtype=np.int32) next_label = 1 - for region in regions: - if region.area < min_size_px: - continue + def region_short_side(region): + """Short side (px) of the rotated bounding box around a region.""" + cnts, _ = cv2.findContours(region.image.astype(np.uint8), + cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + if not cnts: + return 0.0 + (_, (rw, rh), _) = cv2.minAreaRect(max(cnts, key=cv2.contourArea)) + return float(min(rw, rh)) + + # Estimate the single-rod width from the population itself. Rafts of fused + # rods inflate the high end, so use a low percentile of the "simple" + # (high-solidity) regions — robust even when many regions are rafts. + # Estimation only looks at substantial regions (>= 8x the noise floor): + # small debris/specks would otherwise drag the estimate far below the true + # rod width and break the split-acceptance guard. + sized_regions = [r for r in regions if r.area >= min_size_px] + short_sides = {r.label: region_short_side(r) for r in sized_regions} + est_regions = [r for r in sized_regions if r.area >= 8 * min_size_px] + simple_widths = [short_sides[r.label] for r in est_regions + if r.solidity > 0.9 and short_sides[r.label] > 0] + all_widths = [short_sides[r.label] for r in est_regions if short_sides[r.label] > 0] + if len(simple_widths) >= 3: + width_est_px = float(np.percentile(simple_widths, 25)) + elif len(all_widths) >= 3: + # Raft-heavy image: barely any clean single rods. The narrowest + # substantial regions are still single rods, so a low percentile of + # ALL substantial regions works. + width_est_px = float(np.percentile(all_widths, 10)) + else: + width_est_px = None + for region in sized_regions: minr, minc, maxr, maxc = region.bbox label_crop = split_labels[minr:maxr, minc:maxc] - # Solidity > 0.9 is treated as a single rod ("simple"); lower values are - # treated as clumps and separated with a distance-transform watershed - # (autodetect_utils.split_clump). Watershed keeps a single elongated rod - # whole while breaking touching rods apart, and is far faster than the - # recursive rUECS erosion it replaces. - if region.solidity > 0.9: + short_px = short_sides[region.label] + # A raft of parallel rods fused side-by-side is much wider than a + # single rod, and can be convex enough to pass the solidity test. + is_wide = bool(width_est_px) and short_px >= 1.6 * width_est_px + + # Solidity > 0.9 and normal width: a single rod, keep whole. + if region.solidity > 0.9 and not is_wide: label_crop[region.image] = next_label next_label += 1 - else: + continue + + split_masks = None + + # Width-suspicious regions first get the intensity-seam watershed + # (split_clump_intensity): parallel touching rods have no binary + # "neck", but the lighter contact seams in the raw image do separate + # them. Accept the split only if the pieces individually look like + # single rods (short side near the estimated rod width) — this rejects + # bogus cuts through a single rod with internal diffraction bands. + if is_wide: + gray_crop = img[minr:maxr, minc:maxc] + children = split_raft(gray_crop, region.image, min_marker_area, width_est_px) + if len(children) >= 2: + child_shorts = [] + for c_mask in children: + cnts, _ = cv2.findContours(c_mask.astype(np.uint8), + cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + if cnts: + (_, (rw, rh), _) = cv2.minAreaRect(max(cnts, key=cv2.contourArea)) + child_shorts.append(min(rw, rh)) + rod_like = [s for s in child_shorts if s <= 1.8 * width_est_px] + if child_shorts and len(rod_like) / len(child_shorts) >= 0.6: + split_masks = children + + # Fallback / normal clump path: distance-transform watershed. + if split_masks is None: split_masks = split_clump( region.image, min_marker_area, separation_strength=binary_mask_tune, ) - for d_mask in split_masks: - label_crop[d_mask] = next_label - next_label += 1 + + for d_mask in split_masks: + label_crop[d_mask] = next_label + next_label += 1 labels = split_labels From 2869c3013dc87e090b0514029659f7b6355d983e Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Thu, 6 Aug 2026 11:07:11 -0500 Subject: [PATCH 08/10] Folder-analysis membership: inline save feedback, gallery tick, removal - 'Add to Folder Analysis' no longer flickers and no longer relies on a popup alert (blocked in embedded browsers, so saves looked like they did nothing): the button itself shows 'Added N particles' inline, with a stable width and inline error text on failure. - Gallery cards now show a green tick badge on images whose selection is part of the folder analysis (backend reports in_analysis). - Clicking the tick (twice, two-step confirm) removes that image's data from the folder analysis via a new DELETE endpoint. Co-Authored-By: Claude Fable 5 --- backend/main.py | 28 +++++++++++++++++++--- frontend/analysis.html | 43 +++++++++++++++++++++++---------- frontend/index.html | 54 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 16 deletions(-) diff --git a/backend/main.py b/backend/main.py index 1345b4e..469203b 100644 --- a/backend/main.py +++ b/backend/main.py @@ -397,6 +397,23 @@ async def save_folder_selection(folder_name: str, req: FolderSelectionRequest): print(f"Save Selection Error: {e}") raise HTTPException(status_code=500, detail=str(e)) +@app.delete("/folders/{folder_name}/selection/{image_id}") +async def remove_folder_selection(folder_name: str, image_id: str): + """Remove one image's saved selection from the folder analysis.""" + try: + _validate_image_id(image_id) + folder_path = _resolve_folder(folder_name) + cache_file = folder_path / ".analysis_cache" / f"{image_id}.json" + if not cache_file.exists(): + raise HTTPException(status_code=404, detail="No saved selection for this image") + cache_file.unlink() + return {"status": "success", "message": "Selection removed from folder analysis"} + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + @app.get("/folders/{folder_name}/aggregate") async def aggregate_folder(folder_name: str): try: @@ -616,12 +633,17 @@ async def list_images(folder: str = Query(None)): # If overlay exists, it's done overlay_path = RESULTS_DIR / f"{image_id}_overlay.jpg" status = "complete" if overlay_path.exists() else "processing" - + + # A saved selection in the folder's analysis cache means this image + # has been validated and added to the folder analysis. + in_analysis = (path.parent / ".analysis_cache" / f"{image_id}.json").exists() + images.append({ - "id": image_id, + "id": image_id, "filename": path.name, "display_name": display_name, - "status": status + "status": status, + "in_analysis": in_analysis }) # Sort by newest first (optional, but nice) images.sort(key=lambda x: x['display_name']) diff --git a/frontend/analysis.html b/frontend/analysis.html index 13b8fe8..f4d55a9 100644 --- a/frontend/analysis.html +++ b/frontend/analysis.html @@ -760,8 +760,9 @@

Analysis Statistics

+ @@ -1849,16 +1850,25 @@

Analysis Statistics

} }); + const btn = document.getElementById('addToFolderBtn'); + const statusEl = document.getElementById('addToFolderStatus'); + const showStatus = (msg, color) => { + statusEl.textContent = msg; + statusEl.style.color = color; + statusEl.style.display = 'block'; + }; + if (selectedIds.length === 0) { - alert("Please select at least one particle."); + showStatus('Select at least one particle first.', '#dc3545'); return; } + // Inline feedback on the button itself — popup alert() is blocked + // in some embedded browsers, and the instant text swap looked like + // a flicker. min-width on the button keeps the layout stable. try { - const btn = document.getElementById('addToFolderBtn'); - const originalText = btn.innerText; - btn.innerText = "Saving..."; btn.disabled = true; + statusEl.style.display = 'none'; const response = await fetch(`/folders/${encodeURIComponent(folderParam)}/save_selection`, { method: 'POST', @@ -1871,18 +1881,25 @@

Analysis Statistics

if (response.ok) { const res = await response.json(); - alert(`Successfully added ${res.count} particles from this image to folder analysis.`); + btn.innerText = `✓ Added ${res.count} particles`; + btn.style.backgroundColor = '#1e7e34'; } else { - const err = await response.json(); - alert("Error: " + (err.detail || "Server Error")); + const err = await response.json().catch(() => ({})); + btn.innerText = 'Failed — try again'; + btn.style.backgroundColor = '#dc3545'; + showStatus('Error: ' + (err.detail || 'Server Error'), '#dc3545'); } - - btn.innerText = "Add to Folder Analysis"; - btn.disabled = false; } catch (e) { console.error(e); - alert("Failed to save selection."); - document.getElementById('addToFolderBtn').innerText = "Add to Folder Analysis"; + btn.innerText = 'Failed — try again'; + btn.style.backgroundColor = '#dc3545'; + showStatus('Could not reach the server.', '#dc3545'); + } finally { + setTimeout(() => { + btn.innerText = 'Add to Folder Analysis'; + btn.style.backgroundColor = '#28a745'; + btn.disabled = false; + }, 2500); } } diff --git a/frontend/index.html b/frontend/index.html index 633db8d..b29423d 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -300,6 +300,35 @@ opacity: 1; } + /* Green tick on images whose selection was added to Folder Analysis. + Clicking it (twice, to confirm) removes the image from the analysis. */ + .analysis-badge { + position: absolute; + top: 10px; + left: 10px; + background: rgba(40, 167, 69, 0.92); + color: white; + border: none; + border-radius: 50%; + width: 28px; + height: 28px; + font-size: 16px; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + padding: 0; + } + + .analysis-badge.armed { + width: auto; + border-radius: 14px; + padding: 0 12px; + font-size: 13px; + font-weight: 600; + background: #dc3545; + } + /* First click arms the button; second click confirms. */ .delete-btn.armed { opacity: 1; @@ -855,6 +884,19 @@

Upload Instructions

imgWrap.appendChild(overlay); } + if (img.in_analysis) { + const badge = document.createElement('button'); + badge.className = 'analysis-badge'; + badge.textContent = '✓'; + badge.title = 'In Folder Analysis — click to remove'; + badge.addEventListener('click', (ev) => { + ev.stopPropagation(); + armButton(badge, 'Remove from analysis?', + () => removeFromAnalysis(img.id)); + }); + imgWrap.appendChild(badge); + } + const delBtn = document.createElement('button'); delBtn.className = 'delete-btn'; delBtn.textContent = '×'; @@ -912,6 +954,18 @@

Upload Instructions

} } + async function removeFromAnalysis(id) { + try { + const response = await fetch( + `/folders/${encodeURIComponent(currentFolder)}/selection/${encodeURIComponent(id)}`, + { method: 'DELETE' }); + if (response.ok) loadImages(); + else alert('Failed to remove from folder analysis'); + } catch (e) { + alert('Error removing from folder analysis'); + } + } + async function deleteImage(id) { try { const response = await fetch(`/images/${id}`, { method: 'DELETE' }); From cfb6354516c6abf81458c7533ad4ccd11ce06bb6 Mon Sep 17 00:00:00 2001 From: Arda Turk Date: Thu, 6 Aug 2026 11:19:07 -0500 Subject: [PATCH 09/10] Overlay style: thicker boxes, IDs on by default with decade colors - Box outlines thickened (base line ~2.3x) for visibility on dense rafts; excluded particles keep their orange dashed style. - Particle IDs now show by default; button starts as 'Hide IDs'. - ID numbers are color-coded by decade (0s yellow, 10s magenta, 20s green, 30s cyan, 40s red, 50s orange, 60s pink, 70s spring green, cycling) so a given ID is findable at a glance, restoring the scheme from the original pre-fork overlay renderer. Co-Authored-By: Claude Fable 5 --- frontend/analysis.html | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/frontend/analysis.html b/frontend/analysis.html index f4d55a9..e067220 100644 --- a/frontend/analysis.html +++ b/frontend/analysis.html @@ -591,7 +591,7 @@

Analysis Instructions

1. Review the image: the centre panel shows detected particles as boxes. Toggle Show Binary Mask to inspect segmentation, Zoom to look closer, and - Show IDs to label particles (hidden by default so they don't cover small rods).

+ Hide IDs to unclutter the view; ID colors cycle by decade (0s yellow, 10s magenta, 20s green, 30s cyan, 40s red, ...).

2. Check the scale: the calibration source is shown at the top right. If it is wrong or uncalibrated, click Re-scale and draw a line along the scale bar, or use the Mask slider to fine-tune how aggressively touching particles are separated.

@@ -638,7 +638,7 @@

Detected Particles

- +