From 5f118e57261922afc5ed8b0b68f1e6e3e34f9fb4 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 21:41:49 +0200 Subject: [PATCH 1/6] =?UTF-8?q?ci:=20=F0=9F=91=B7=20build=20installable=20?= =?UTF-8?q?plugin=20bundles=20for=20four=20platforms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installing a plugin required a Rust toolchain, a sibling augur-rs checkout and a working cargo, which made every measurement PC a development machine. Build all runtime plugins on each pull request and each push to main for macOS arm64/x86_64, Linux x86_64 and Windows x86_64, staged in the exact layout ~/.augur/plugins/ expects, so installing is a copy. main also publishes a rolling plugins-latest release, because workflow artifacts need a login and expire after 90 days while a bench should be able to curl a URL. The workspace depends on the host by path, so the job lays out two sibling checkouts. build-runtime-plugins.sh patches a git source whenever it finds a sibling augur-rs checkout; with path deps that patch matches nothing but still costs a fetch, so the checkout's .git is dropped right after cloning. CI calls the repository's own build and install scripts instead of restating the install layout in YAML — those scripts already own plugin discovery, library naming, A1's protocols folder and the macOS install-name rewrite. Plugins are dlopened into the host process, so pin rust-toolchain.toml to the same 1.95.0 augur-rs pins and read the channel out of that file rather than naming a version in the workflow. Every bundle carries a BUILD-INFO.txt with the augur-rs revision and rustc version behind it, which is what makes an ABI mismatch reported from the bench answerable. --- .github/workflows/build-plugins.yml | 184 ++++++++++++++++++ .gitignore | 1 + README.md | 26 +++ .../030-prebuilt-plugin-bundles-from-ci.md | 87 +++++++++ docs/features/README.md | 1 + docs/features/ci-prebuilt-plugin-bundles.md | 135 +++++++++++++ docs/installing-plugins.md | 35 ++++ rust-toolchain.toml | 8 + 8 files changed, 477 insertions(+) create mode 100644 .github/workflows/build-plugins.yml create mode 100644 docs/adr/030-prebuilt-plugin-bundles-from-ci.md create mode 100644 docs/features/ci-prebuilt-plugin-bundles.md create mode 100644 rust-toolchain.toml diff --git a/.github/workflows/build-plugins.yml b/.github/workflows/build-plugins.yml new file mode 100644 index 0000000..303e653 --- /dev/null +++ b/.github/workflows/build-plugins.yml @@ -0,0 +1,184 @@ +name: Build Plugins + +# Produces drop-in plugin folders for a machine that has no Rust toolchain: the +# bench downloads a bundle, copies its folders into ~/.augur/plugins/, and hits +# "Scan for New Plugins". Pull requests get workflow artifacts; main also +# publishes a rolling release so the download needs no GitHub login. + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + inputs: + augur_rs_ref: + description: "augur-rs ref to build against (branch, tag or SHA)" + required: false + default: main + +concurrency: + group: build-plugins-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + AUGUR_RS_REF: ${{ inputs.augur_rs_ref || 'main' }} + +jobs: + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: macos-latest + name: macOS (arm64) + bundle: macos-arm64 + - os: macos-13 + name: macOS (x86_64) + bundle: macos-x86_64 + - os: ubuntu-latest + name: Linux (x86_64) + bundle: linux-x86_64 + - os: windows-latest + name: Windows (x86_64) + bundle: windows-x86_64 + + defaults: + run: + # The repo drives its builds through two bash scripts; use the same + # shell on Windows so there is exactly one code path to reason about. + shell: bash + + steps: + # This workspace depends on the host by path (../augur-rs/augur-core), so + # CI has to reproduce the two-sibling-checkout layout, not clone one repo. + - name: Check out augur-plugins + uses: actions/checkout@v5 + with: + path: augur-plugins + + - name: Check out augur-rs + uses: actions/checkout@v5 + with: + repository: muthmann/augur-rs + ref: ${{ env.AUGUR_RS_REF }} + path: augur-rs + + - name: Pin the host revision and disarm the source patch + run: | + echo "AUGUR_RS_SHA=$(git -C augur-rs rev-parse HEAD)" >> "$GITHUB_ENV" + # build-runtime-plugins.sh patches [patch."…/augur-rs.git"] whenever a + # sibling augur-rs *git checkout* exists. This workspace already + # depends on it by path, so that patch matches nothing in the crate + # graph — it only costs cargo a fetch of the checkout. Removing .git + # makes the script's detection fail and the path deps win outright. + rm -rf augur-rs/.git + + - name: Resolve the pinned Rust toolchain + run: | + channel="$(sed -n 's/^channel *= *"\(.*\)"$/\1/p' augur-plugins/rust-toolchain.toml | head -n 1)" + if [[ -z "${channel}" ]]; then + echo "No channel found in augur-plugins/rust-toolchain.toml" >&2 + exit 1 + fi + echo "RUST_CHANNEL=${channel}" >> "$GITHUB_ENV" + + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: ${{ env.RUST_CHANNEL }} + cache-workspaces: augur-plugins + + - name: Install Linux system dependencies + if: runner.os == 'Linux' + # Reuse the host's own dependency list rather than a second copy that can + # drift; it is a superset of what the plugins need (serialport/libudev). + run: bash augur-rs/.github/scripts/install-linux-deps.sh + + - name: Build runtime plugins + working-directory: augur-plugins + run: bash scripts/build-runtime-plugins.sh --profile release + + - name: Stage installable plugin folders + working-directory: augur-plugins + run: bash scripts/install-built-plugins.sh --profile release --dest "dist/${{ matrix.bundle }}" + + - name: Write build provenance + working-directory: augur-plugins + run: | + { + echo "bundle: ${{ matrix.bundle }}" + echo "built_at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "augur_plugins: $(git rev-parse HEAD)" + echo "augur_rs_ref: ${AUGUR_RS_REF}" + echo "augur_rs_sha: ${AUGUR_RS_SHA}" + echo "rustc: $(rustc --version)" + echo + echo "Copy the plugin folders next to this file into ~/.augur/plugins/," + echo "then use Plugins -> Scan for New Plugins in augur-gui." + } > "dist/${{ matrix.bundle }}/BUILD-INFO.txt" + + - name: Upload plugin bundle + uses: actions/upload-artifact@v4 + with: + name: augur-plugins-${{ matrix.bundle }} + path: augur-plugins/dist/${{ matrix.bundle }} + if-no-files-found: error + + release: + name: Publish rolling release + needs: build + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Download every plugin bundle + uses: actions/download-artifact@v4 + with: + path: bundles + pattern: augur-plugins-* + + - name: Package one archive per platform + run: | + set -euo pipefail + mkdir -p dist + for bundle_dir in bundles/augur-plugins-*/; do + bundle="$(basename "${bundle_dir%/}")" + (cd "${bundle_dir}" && zip -qr "${GITHUB_WORKSPACE}/dist/${bundle}.zip" .) + echo "Packaged ${bundle}.zip" + done + (cd dist && sha256sum ./*.zip > SHA256SUMS.txt) + ls -l dist + + - name: Publish rolling release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + tag="plugins-latest" + # Delete and recreate rather than upload --clobber: it retags at the + # new commit and guarantees no asset from an older build survives. + gh release delete "${tag}" --yes --cleanup-tag || true + gh release create "${tag}" dist/* \ + --title "Prebuilt plugins (latest main)" \ + --notes "$(printf '%s\n' \ + "Prebuilt AugurRS plugins, rebuilt on every push to \`main\`." \ + "" \ + "- augur-plugins: \`${GITHUB_SHA}\`" \ + "- built against augur-rs \`${AUGUR_RS_REF}\`" \ + "" \ + "Download the archive for your platform, unpack it, and copy the" \ + "plugin folders inside into \`~/.augur/plugins/\`. Then open augur-gui," \ + "go to **Plugins**, and click **Scan for New Plugins**." \ + "" \ + "\`BUILD-INFO.txt\` in each archive records the exact revisions and" \ + "compiler the libraries were built with. Verify downloads against" \ + "\`SHA256SUMS.txt\`.")" diff --git a/.gitignore b/.gitignore index 39875ee..f240a43 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target +/dist Cargo.lock *.swp *.swo diff --git a/README.md b/README.md index 346be75..da10175 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,31 @@ The plugin crates under `plugins/` are under active development and not yet read ## Quick Start +### Download Prebuilt Plugins (no toolchain needed) + +Every push to `main` publishes freshly built plugins for macOS (arm64 and x86_64), +Linux and Windows to the rolling +[`plugins-latest`](https://github.com/muthmann/augur-plugins/releases/tag/plugins-latest) +release. This is the recommended route for a measurement machine. + +```bash +curl -LO https://github.com/muthmann/augur-plugins/releases/download/plugins-latest/augur-plugins-macos-arm64.zip +unzip augur-plugins-macos-arm64.zip -d augur-plugins-bundle +mkdir -p ~/.augur/plugins +cp -R augur-plugins-bundle/*/ ~/.augur/plugins/ +``` + +Pick the archive matching the machine: `macos-arm64`, `macos-x86_64`, +`linux-x86_64`, or `windows-x86_64`. Then open `augur-gui`, go to **Plugins**, and +click **Scan for New Plugins**. + +Each archive contains a `BUILD-INFO.txt` recording the `augur-rs` revision and the +`rustc` version the libraries were built against — quote it in any ABI-mismatch +report. Verify downloads against `SHA256SUMS.txt` from the same release. + +Pull requests build the same bundles as workflow artifacts. See +[CI Prebuilt Plugin Bundles](./docs/features/ci-prebuilt-plugin-bundles.md). + ### Build One Plugin ```bash @@ -145,6 +170,7 @@ augur-plugins/ - [Plugin API Notes](./docs/plugin-api.md) — repo-local summary of the current runtime contract - [Installing Plugins](./docs/installing-plugins.md) — build, copy, reload, and troubleshoot installed plugins +- [CI Prebuilt Plugin Bundles](./docs/features/ci-prebuilt-plugin-bundles.md) — how the downloadable per-platform bundles are built and published - [Architecture Notes](./docs/architecture.md) — repository role, execution model, host views, and shared settings - [augur-rs Plugin Authoring Guide](https://github.com/muthmann/augur-rs/blob/main/docs/features/plugin-authoring-guide.md) — canonical host/runtime authoring guide - [augur-rs Global Settings Guide](https://github.com/muthmann/augur-rs/blob/main/docs/features/global-settings-menu.md) — host-owned settings published to plugins diff --git a/docs/adr/030-prebuilt-plugin-bundles-from-ci.md b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md new file mode 100644 index 0000000..797edf3 --- /dev/null +++ b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md @@ -0,0 +1,87 @@ +# ADR 030 — Prebuilt plugin bundles are produced by CI, not by the bench + +**Status:** accepted +**Date:** 2026-08-04 +**Feature brief:** [CI Prebuilt Plugin Bundles](../features/ci-prebuilt-plugin-bundles.md) + +## Context + +A runtime plugin is a `cdylib` plus a `plugin.toml`. Getting one onto a machine +required a Rust toolchain, a sibling `augur-rs` checkout, and `cargo`, because +this workspace depends on the host by path. That made the measurement PC a +development machine by necessity: every plugin fix had to be compiled where it +was used. + +Two properties of the plugin model make "just compile it there" worse than it +looks. Plugins are dlopened into the host process, so the compiler that builds a +plugin and the compiler that builds `augur-gui` have to agree — and this +repository pinned no toolchain at all while `augur-rs` pinned `1.95.0`. And the +installed folder is not just a library: A1 ships operator-facing `protocols/` +examples, and macOS copies need their dylib id rewritten or Plugin Manager +reloads resolve back into Cargo's build tree. + +## Decision + +CI builds the runtime plugins on every pull request and every push to `main`, for +macOS arm64, macOS x86_64, Linux x86_64 and Windows x86_64, and publishes the +result as a folder that is copied verbatim into `~/.augur/plugins/`. + +Three things follow from that, and they are the actual decision: + +1. **This repository pins the host's toolchain.** `rust-toolchain.toml` carries + the same `1.95.0` as `augur-rs`, and the workflow reads the channel out of + that file instead of naming a version in YAML. A bundle built by a different + compiler than the host is not a bundle, it is a load failure waiting to + happen, and the pin is the only thing that makes that guarantee checkable. + +2. **CI runs the repository's own build and install scripts.** It does not + reimplement plugin discovery, library naming, the `protocols/` copy or the + macOS install-name rewrite in YAML. The scripts are the single definition of + what an installed plugin is; CI is one more caller of them, with + `--dest dist/` instead of `~/.augur/plugins`. + +3. **`main` publishes a rolling release, not just artifacts.** Workflow artifacts + need a GitHub login and expire after 90 days. The bench is the consumer, and + it should be able to `curl` a URL. The tag `plugins-latest` is deleted and + recreated on every push to `main`, so its assets can never be a mixture of two + builds. + +Every bundle carries a `BUILD-INFO.txt` recording the `augur-plugins` commit, the +`augur-rs` ref and SHA, and the exact `rustc` version. + +## Consequences + +- The measurement PC needs no toolchain, no checkout, and no `cargo`. +- An ABI-mismatch report from the bench is now answerable: the provenance file + says which host revision and compiler the installed library came from. +- Local builds in this repository move from whatever `rustc` is on `PATH` to the + pinned `1.95.0` — but only for people whose `cargo` is the rustup shim. A + Homebrew `cargo` earlier on `PATH` ignores `rust-toolchain.toml` entirely and + will keep producing plugins for a compiler the host does not use. +- The macOS bundles are per-architecture while `augur-gui` ships universal, so + the download page has one more choice on it than the host's does. +- `main` gains a permanent release tag. The repository had no releases before, so + `releases/latest` now resolves to `plugins-latest`; a future versioned release + scheme would have to account for that. + +## Alternatives considered + +**Publish only workflow artifacts.** Simplest, and rejected: it puts a GitHub +login between the bench and a fix, and the artifact disappears after 90 days. + +**Build against the newest `augur-rs` release tag instead of `main`.** Matches a +bench running a released host, but lags every unpublished API change — and the +Stage-A work in this repository routinely depends on unreleased host changes. +`main` is the default; `workflow_dispatch` takes an `augur_rs_ref` for the cases +where a specific host revision is wanted. + +**Reimplement the install layout in the workflow.** Would have avoided calling +shell scripts from YAML, at the cost of a second, silently divergent definition +of what an installed plugin contains. The `protocols/` folder and the macOS +install-name rewrite were both added to the script after the fact; a YAML copy +would have missed both. + +**`lipo` the two macOS builds into universal libraries.** Attractive, since the +host is universal, but `install-built-plugins.sh` reads `target/` only +and a cross-build lands in `target//`. Deferred rather than +special-cased in CI, since it belongs in the script if it is worth doing. diff --git a/docs/features/README.md b/docs/features/README.md index f740496..fb930c9 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -13,6 +13,7 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). - [EVE Temporal Diagnostics](./evesmlm-temporal-diagnostics.md) — temporal candidate tracking, boundary overlays, and rejected-fit datasets for the eveSMLM pipeline. +- [CI Prebuilt Plugin Bundles](./ci-prebuilt-plugin-bundles.md) — every pull request and every push to `main` builds all runtime plugins for macOS (arm64/x86_64), Linux and Windows, staged in the exact `~/.augur/plugins/` layout so a bench machine installs by copying instead of compiling. `main` publishes them as a rolling `plugins-latest` release that needs no GitHub login, each bundle carrying a `BUILD-INFO.txt` with the `augur-rs` revision and `rustc` version it was built against. CI calls the repo's own build/install scripts rather than restating the install layout in YAML, and `rust-toolchain.toml` now pins the host's `1.95.0` because plugins are dlopened into the host process (ADR 030). - [Plugin Authoring Docs Refresh](./plugin-authoring-doc-refresh.md) — repo docs synced to the current runtime-only interface, host views, and `GlobalSettings`. - [Plugin Install And Reload](./plugin-install-reload.md) — macOS dylib identity fix so installed plugins do not keep pointing back at Cargo's build tree during reloads. - [Investigation Workspace Alignment](./investigation-workspace-alignment.md) — in-tree plugins updated for stable ids, linked 2D/3D/table datasets, and candidate-stage accepted/rejected event inspection. diff --git a/docs/features/ci-prebuilt-plugin-bundles.md b/docs/features/ci-prebuilt-plugin-bundles.md new file mode 100644 index 0000000..c597487 --- /dev/null +++ b/docs/features/ci-prebuilt-plugin-bundles.md @@ -0,0 +1,135 @@ +# CI Prebuilt Plugin Bundles + +**Status:** built +**Workflow:** [`.github/workflows/build-plugins.yml`](../../.github/workflows/build-plugins.yml) +**ADR:** [030 — Prebuilt plugin bundles are produced by CI](../adr/030-prebuilt-plugin-bundles-from-ci.md) + +## Problem + +Installing a plugin used to require a Rust toolchain, a sibling `augur-rs` +checkout, and a working `cargo`. That is a reasonable ask of a contributor and an +unreasonable ask of the bench machine that actually runs the experiment. A +measurement PC should not need a development environment just to pick up a fixed +plugin. + +## What it does + +Every pull request and every push to `main` builds all runtime plugins on four +platforms and stages them in the exact layout `~/.augur/plugins/` expects: + +```text +augur-plugins-macos-arm64/ + BUILD-INFO.txt + stage-a-a1/ + plugin.toml + libaugur_plugin_stage_a_a1.dylib + protocols/ + example.csv + example.toml + stage-a-modulation/ + stage-a-photodiode/ + localization/ + … +``` + +Installing is then a copy — no build step, no toolchain. + +| Bundle | Runner | Library | +|---|---|---| +| `macos-arm64` | `macos-latest` | `.dylib` | +| `macos-x86_64` | `macos-13` | `.dylib` | +| `linux-x86_64` | `ubuntu-latest` | `.so` | +| `windows-x86_64` | `windows-latest` | `.dll` | + +Pull requests publish the bundles as workflow artifacts. Pushes to `main` +additionally publish a rolling GitHub Release tagged `plugins-latest`, one zip per +platform plus `SHA256SUMS.txt`. The release exists because artifacts require a +GitHub login and expire; a release asset can be fetched from the bench with +`curl` and no account. + +`workflow_dispatch` takes an `augur_rs_ref` input for building a bundle against a +host branch or tag other than `main`. + +## Why it is shaped this way + +**Two sibling checkouts, not one.** The workspace depends on the host by path +(`augur-core = { path = "../augur-rs/augur-core" }`), so the job checks +`augur-plugins` and `augur-rs` out next to each other under the workspace root +and builds from the former. A single-repo checkout cannot resolve the dependency +at all. + +**`augur-rs/.git` is deleted right after checkout.** `build-runtime-plugins.sh` +adds `--config patch."…augur-rs.git"…` flags whenever it finds a sibling +`augur-rs` *git checkout*. With path dependencies that patch matches nothing — +cargo reports `Patch … was not used in the crate graph` and exits 0 — but it +still costs a git fetch of the checkout. Removing `.git` makes the script's +detection fail, and the path dependencies are used directly. + +**The toolchain is pinned and read from the file.** Plugins are `cdylib`s the +host `dlopen`s into its own process, so they must be built by the same compiler +as `augur-gui`. [`rust-toolchain.toml`](../../rust-toolchain.toml) pins the same +`1.95.0` as `augur-rs`, and the workflow parses the channel out of that file +rather than repeating the version — CI cannot drift from the pin. + +**Linux system dependencies come from the host's own script.** The job runs +`augur-rs/.github/scripts/install-linux-deps.sh` from the checkout it already +has, instead of keeping a second list that can go stale. `serialport` (used by +`stage-a-modulation` and `stage-a-photodiode`) needs `libudev`, and that script +is guaranteed to be a superset of what the plugins need. + +**The build goes through the repo's own two scripts.** `build-runtime-plugins.sh` +and `install-built-plugins.sh` already know which crates are runtime plugins, +which library name each `plugin.toml` declares, that A1's `protocols/` folder has +to travel with the plugin, and that macOS copies need their dylib id rewritten to +`@loader_path/`. Re-implementing any of that in YAML would be a second +source of truth. CI runs the same commands a developer runs, only with +`--dest dist/`. + +**Archiving happens once, in the release job.** The build matrix uploads raw +folders; the Ubuntu release job zips them. `zip` is not available in the Windows +runner's bash by default, so packaging on each runner would have needed a +per-platform branch for no benefit. + +## Provenance + +Each bundle carries `BUILD-INFO.txt`: + +```text +bundle: macos-arm64 +built_at: 2026-08-04T19:38:11Z +augur_plugins: 30e677c… +augur_rs_ref: main +augur_rs_sha: d43652a… +rustc: rustc 1.95.0 (…) +``` + +That is what turns an "ABI mismatch" report from the bench into an answerable +question: it records exactly which host revision and which compiler the installed +library was built against. + +## Installing a bundle + +1. Download the archive for the platform from the + [`plugins-latest` release](https://github.com/muthmann/augur-plugins/releases/tag/plugins-latest) +2. Unpack it +3. Copy the plugin folders inside into `~/.augur/plugins/` +4. In `augur-gui`: **Plugins** → **Scan for New Plugins** → enable + +See [Installing Runtime Plugins](../installing-plugins.md) for the full +installed layout and troubleshooting. + +## Limitations + +- The macOS bundles are single-architecture, not universal. `augur-gui` ships as + a universal binary, so an Intel Mac needs `macos-x86_64` and an Apple Silicon + Mac needs `macos-arm64`; picking the wrong one fails at load, not at copy. +- `macos-13` is GitHub's last x86_64 macOS runner image. When it is retired, the + Intel bundle needs a cross-build (`--target x86_64-apple-darwin`), which the + install script does not currently look for — it only reads `target/`. +- The bundles are unsigned. macOS Gatekeeper does not quarantine libraries loaded + by `dlopen` from a user directory, so this has not needed handling, but a + downloaded archive may still need `xattr -d com.apple.quarantine` if Safari + attached the flag. +- `Cargo.lock` is gitignored, so builds are not `--locked`. A dependency + publishing a broken semver-compatible release can turn CI red without a commit + in either repository. diff --git a/docs/installing-plugins.md b/docs/installing-plugins.md index 45a79e9..0b17073 100644 --- a/docs/installing-plugins.md +++ b/docs/installing-plugins.md @@ -21,6 +21,33 @@ On Linux the library ends in `.so`. On Windows it ends in `.dll`. Host-owned built-in tools are part of `augur-gui` and are not installed from this repository. +## Install Without A Toolchain (recommended for bench machines) + +CI builds every runtime plugin on each push to `main` and publishes them as a +rolling [`plugins-latest`](https://github.com/muthmann/augur-plugins/releases/tag/plugins-latest) +release, already in the layout above. Installing is then a copy: + +```bash +curl -LO https://github.com/muthmann/augur-plugins/releases/download/plugins-latest/augur-plugins-macos-arm64.zip +unzip augur-plugins-macos-arm64.zip -d bundle +mkdir -p ~/.augur/plugins +cp -R bundle/*/ ~/.augur/plugins/ +``` + +Archives exist for `macos-arm64`, `macos-x86_64`, `linux-x86_64` and +`windows-x86_64`. The macOS libraries are per-architecture, not universal, so an +Apple Silicon machine needs `macos-arm64` even though `augur-gui` itself ships +universal — the wrong one fails at load time, not at copy time. + +Every archive carries a `BUILD-INFO.txt` naming the `augur-rs` revision and the +`rustc` version it was built with. That is the first thing to check against a +[plugin ABI mismatch](#plugin-abi-mismatch). Verify downloads against +`SHA256SUMS.txt` from the same release. + +The sections below cover building from source, which contributors still need. +See [CI Prebuilt Plugin Bundles](./features/ci-prebuilt-plugin-bundles.md) for how +the bundles are produced. + ## Build One Plugin ```bash @@ -99,6 +126,14 @@ The library was built against an older plugin interface or does not export the r The installed runtime library is stale relative to the host ABI. +If the library came from a release bundle, compare its `BUILD-INFO.txt` against the +running host first — `augur_rs_sha` says which host revision it was built for, and +`rustc` says which compiler produced it. Plugins are loaded into the host process, +so a compiler mismatch is as much a cause as a stale revision; +[`rust-toolchain.toml`](../rust-toolchain.toml) pins the same version `augur-rs` +does, but only a rustup-managed `cargo` honours it. Check with +`cargo --version` — a Homebrew or distro `cargo` earlier on `PATH` ignores the pin. + 1. Rebuild the plugin against the current sibling `augur-rs` checkout. 2. Replace the installed runtime library in `~/.augur/plugins//`. 3. On macOS, prefer `./scripts/install-built-plugins.sh --profile release` or rewrite the copied dylib id with `install_name_tool -id "@loader_path/" ...`. diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..544a2fa --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,8 @@ +[toolchain] +# Must match augur-rs/rust-toolchain.toml. Plugins are cdylibs that the host +# dlopens into its own process, so the compiler that builds them and the +# compiler that builds augur-gui have to agree — an unpinned "stable" here +# means CI can hand the bench a bundle built against a different std. +# Bump this together with augur-rs, in its own commit. +channel = "1.95.0" +components = ["clippy", "rustfmt"] From c61fa6c59e527063cadc5f0a01c25b7699c55b8b Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 21:49:25 +0200 Subject: [PATCH 2/6] =?UTF-8?q?ci:=20=F0=9F=90=9B=20stop=20the=20build=20f?= =?UTF-8?q?ailing=20on=20the=20host's=20own=20CI=20layout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defects the first run exposed, both independent of the plugin sources. The Linux job borrowed augur-rs/.github/scripts/install-linux-deps.sh from the host checkout to avoid keeping a second dependency list. That script does not exist on every augur-rs revision the job can be pointed at, so the Linux build failed on the value of augur_rs_ref rather than on anything in this repository. It was also a superset: it installs the X11/Wayland/GL stack for the GUI, which no plugin crate links. Install what the plugins actually need instead — pkg-config and libudev-dev for serialport. setup-rust-toolchain injects RUSTFLAGS="-D warnings" by default. That is right for a lint job and wrong for one that ships artifacts: a dead-code warning in one plugin would have denied the bench a bundle for all of them. Lint gating belongs in its own job. Also record what the run proved about the repository itself: these plugins do not compile against augur-rs main, which lacks the TableSchema, host-view and dataset-descriptor API they use. --- .github/workflows/build-plugins.yml | 16 ++++++++-- docs/features/ci-prebuilt-plugin-bundles.md | 33 +++++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-plugins.yml b/.github/workflows/build-plugins.yml index 303e653..499e495 100644 --- a/.github/workflows/build-plugins.yml +++ b/.github/workflows/build-plugins.yml @@ -94,12 +94,22 @@ jobs: with: toolchain: ${{ env.RUST_CHANNEL }} cache-workspaces: augur-plugins + # The action injects RUSTFLAGS="-D warnings" by default. That is right + # for a lint job and wrong here: this job ships artifacts, and a dead- + # code warning in one plugin must not deny the bench a bundle for all + # of them. Lint gating belongs in its own job, not in the build. + rustflags: "" - name: Install Linux system dependencies if: runner.os == 'Linux' - # Reuse the host's own dependency list rather than a second copy that can - # drift; it is a superset of what the plugins need (serialport/libudev). - run: bash augur-rs/.github/scripts/install-linux-deps.sh + # Only what the plugin crates actually link. augur-gui's own dependency + # script is deliberately not reused: it is a superset (X11/Wayland/GL for + # the GUI, which no plugin links) and it does not exist on every augur-rs + # revision this job can be pointed at, so borrowing it made the Linux + # build fail on the value of augur_rs_ref. serialport needs libudev. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends pkg-config libudev-dev - name: Build runtime plugins working-directory: augur-plugins diff --git a/docs/features/ci-prebuilt-plugin-bundles.md b/docs/features/ci-prebuilt-plugin-bundles.md index c597487..d2a43ec 100644 --- a/docs/features/ci-prebuilt-plugin-bundles.md +++ b/docs/features/ci-prebuilt-plugin-bundles.md @@ -71,11 +71,18 @@ as `augur-gui`. [`rust-toolchain.toml`](../../rust-toolchain.toml) pins the same `1.95.0` as `augur-rs`, and the workflow parses the channel out of that file rather than repeating the version — CI cannot drift from the pin. -**Linux system dependencies come from the host's own script.** The job runs -`augur-rs/.github/scripts/install-linux-deps.sh` from the checkout it already -has, instead of keeping a second list that can go stale. `serialport` (used by -`stage-a-modulation` and `stage-a-photodiode`) needs `libudev`, and that script -is guaranteed to be a superset of what the plugins need. +**Linux system dependencies are the plugins' own, not the host's.** The job +installs `pkg-config` and `libudev-dev`, which is what `serialport` (used by +`stage-a-modulation` and `stage-a-photodiode`) needs. Reusing +`augur-rs/.github/scripts/install-linux-deps.sh` was tried first and reverted: it +pulls the whole GUI stack that no plugin links, and it does not exist on every +`augur-rs` revision this job can be pointed at, so the Linux build failed on the +value of `augur_rs_ref` rather than on anything in this repository. + +**Warnings are not errors here.** `actions-rust-lang/setup-rust-toolchain` +injects `RUSTFLAGS="-D warnings"` by default. This job ships artifacts, so it +sets `rustflags: ""` — a dead-code warning in one plugin must not deny the bench +a bundle for all of them. Lint gating belongs in its own job. **The build goes through the repo's own two scripts.** `build-runtime-plugins.sh` and `install-built-plugins.sh` already know which crates are runtime plugins, @@ -118,6 +125,22 @@ library was built against. See [Installing Runtime Plugins](../installing-plugins.md) for the full installed layout and troubleshooting. +## Prerequisite: the host API this repo targets must be on `augur-rs` `main` + +The workflow defaults to building against `augur-rs` `main`, and that only works +once `main` actually carries the host API these plugins use. At the time this +workflow was added it did not: the eveSMLM plugins reference `TableSchema` +fields (`layer_id`, `semantic_label`, `provenance`, `column_display`, +`row_id_column`, `time_column`, `coordinate_space_3d`), +`HostViewKind::Scatter3dFromTable`, `HostDatasetDescriptor.relations` / +`.display` and `HostViewRegistry.actions`, none of which exist on `augur-rs` +`main` — they live on an unmerged host branch. + +This is a real finding rather than a CI defect: it means the repository as +checked in cannot be built by anyone who does not already have that unmerged +host branch on disk. Until it lands, point `workflow_dispatch` at a pushed +`augur_rs_ref` that carries the API. + ## Limitations - The macOS bundles are single-architecture, not universal. `augur-gui` ships as From 671b25434074bb6654c5b2aececce1010072d530 Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 21:51:57 +0200 Subject: [PATCH 3/6] =?UTF-8?q?ci:=20=F0=9F=94=A7=20build=20against=20the?= =?UTF-8?q?=20host=20branch=20that=20has=20the=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit augur-rs main still has a two-field TableSchema and no Scatter3dFromTable, HostDatasetDescriptor.relations/display or HostViewRegistry.actions, all of which the plugins in this repository already use. Defaulting AUGUR_RS_REF to main is therefore a guaranteed red build that never hands the bench a bundle. Default to the open host branch that does carry the API instead, and record the coupling in the brief and the ADR. BUILD-INFO.txt already names the exact host ref and SHA behind every library, so this stays visible rather than becoming folklore. Move the default back to main in the same commit that the host API lands there. --- .github/workflows/build-plugins.yml | 9 +++-- .../030-prebuilt-plugin-bundles-from-ci.md | 13 +++++--- docs/features/ci-prebuilt-plugin-bundles.md | 33 +++++++++++-------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-plugins.yml b/.github/workflows/build-plugins.yml index 499e495..7f8278b 100644 --- a/.github/workflows/build-plugins.yml +++ b/.github/workflows/build-plugins.yml @@ -15,7 +15,7 @@ on: augur_rs_ref: description: "augur-rs ref to build against (branch, tag or SHA)" required: false - default: main + default: fix/gui-layout-and-alignment concurrency: group: build-plugins-${{ github.ref }} @@ -26,7 +26,12 @@ permissions: env: CARGO_TERM_COLOR: always - AUGUR_RS_REF: ${{ inputs.augur_rs_ref || 'main' }} + # The host branch this repository actually compiles against. augur-rs `main` + # does not carry the TableSchema, host-view and dataset-descriptor API these + # plugins use, so defaulting to `main` would be a guaranteed red build and + # would never hand the bench a bundle. Move this back to `main` in the same + # commit that the host API lands there. + AUGUR_RS_REF: ${{ inputs.augur_rs_ref || 'fix/gui-layout-and-alignment' }} jobs: build: diff --git a/docs/adr/030-prebuilt-plugin-bundles-from-ci.md b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md index 797edf3..d00a3dc 100644 --- a/docs/adr/030-prebuilt-plugin-bundles-from-ci.md +++ b/docs/adr/030-prebuilt-plugin-bundles-from-ci.md @@ -69,11 +69,14 @@ Every bundle carries a `BUILD-INFO.txt` recording the `augur-plugins` commit, th **Publish only workflow artifacts.** Simplest, and rejected: it puts a GitHub login between the bench and a fix, and the artifact disappears after 90 days. -**Build against the newest `augur-rs` release tag instead of `main`.** Matches a -bench running a released host, but lags every unpublished API change — and the -Stage-A work in this repository routinely depends on unreleased host changes. -`main` is the default; `workflow_dispatch` takes an `augur_rs_ref` for the cases -where a specific host revision is wanted. +**Build against the newest `augur-rs` release tag, or against `main`.** Both were +rejected by fact rather than by preference: `augur-rs` `main` does not carry the +`TableSchema`, host-view or dataset-descriptor API these plugins already use, so +either choice is a guaranteed red build. The default host ref is therefore the +open host branch that does carry it, and `BUILD-INFO.txt` records the exact ref +and SHA behind every library so the coupling stays visible. This is temporary by +construction: the default moves to `main` in the same commit that the host API +lands there. **Reimplement the install layout in the workflow.** Would have avoided calling shell scripts from YAML, at the cost of a second, silently divergent definition diff --git a/docs/features/ci-prebuilt-plugin-bundles.md b/docs/features/ci-prebuilt-plugin-bundles.md index d2a43ec..bdacf01 100644 --- a/docs/features/ci-prebuilt-plugin-bundles.md +++ b/docs/features/ci-prebuilt-plugin-bundles.md @@ -48,7 +48,9 @@ GitHub login and expire; a release asset can be fetched from the bench with `curl` and no account. `workflow_dispatch` takes an `augur_rs_ref` input for building a bundle against a -host branch or tag other than `main`. +different host branch or tag — but note that GitHub only offers `workflow_dispatch` +for workflows present on the default branch, so until this lands on `main` the +`AUGUR_RS_REF` default below is the only way to retarget the host revision. ## Why it is shaped this way @@ -125,21 +127,26 @@ library was built against. See [Installing Runtime Plugins](../installing-plugins.md) for the full installed layout and troubleshooting. -## Prerequisite: the host API this repo targets must be on `augur-rs` `main` +## Which host revision the bundles are built against -The workflow defaults to building against `augur-rs` `main`, and that only works -once `main` actually carries the host API these plugins use. At the time this -workflow was added it did not: the eveSMLM plugins reference `TableSchema` -fields (`layer_id`, `semantic_label`, `provenance`, `column_display`, -`row_id_column`, `time_column`, `coordinate_space_3d`), +`AUGUR_RS_REF` currently defaults to the `augur-rs` branch +`fix/gui-layout-and-alignment` (open host PR #36), **not** to `main`. + +That is not a preference, it is the state of the two repositories. These plugins +reference `TableSchema` fields (`layer_id`, `semantic_label`, `provenance`, +`column_display`, `row_id_column`, `time_column`, `coordinate_space_3d`), `HostViewKind::Scatter3dFromTable`, `HostDatasetDescriptor.relations` / -`.display` and `HostViewRegistry.actions`, none of which exist on `augur-rs` -`main` — they live on an unmerged host branch. +`.display` and `HostViewRegistry.actions` — none of which exist on `augur-rs` +`main`, which still has a two-field `TableSchema`. Defaulting to `main` would be +a guaranteed red build that never hands the bench a bundle. + +The first CI run is what surfaced this: the repository as checked in cannot be +built by anyone who does not already have an unmerged host branch on disk. -This is a real finding rather than a CI defect: it means the repository as -checked in cannot be built by anyone who does not already have that unmerged -host branch on disk. Until it lands, point `workflow_dispatch` at a pushed -`augur_rs_ref` that carries the API. +**Move the default back to `main` in the same commit that the host API lands +there.** Until then, `BUILD-INFO.txt` is the thing that keeps this honest — it +records the exact host ref and SHA behind every library, so an installed plugin +can always be traced to the host revision it matches. ## Limitations From a24e135e0e6079808a31a3cca4fdae1eac9db05a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 22:04:37 +0200 Subject: [PATCH 4/6] =?UTF-8?q?refactor(evesmlm):=20=E2=99=BB=EF=B8=8F=20s?= =?UTF-8?q?hare=20types=20through=20a=20crate,=20not=20between=20plugins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eveSMLM chain expressed its stage dependencies directly: fitting depended on the candidates crate, post-processing on the fitting crate. Plugin crates are cdylibs that each export augur_plugin_vtable, so linking one plugin's rlib into another pulled that symbol in twice. Apple's linker tolerates the duplicate. rust-lld and MSVC's link.exe do not: rust-lld: error: duplicate symbol: augur_plugin_vtable LNK2005: augur_plugin_vtable already defined … fatal error LNK1169 That went unnoticed for as long as the only build machine was a Mac. The first CI run on four platforms found it: macOS produced a complete bundle while Linux and Windows failed to link, which also denied the bench a Windows bundle for the Stage-A plugins, since the build is all-or-nothing. Move everything that crosses a stage boundary into evesmlm-types, a plain library crate that exports no vtable — the wire contract plus the current-localization dataset and registry builders that both fitting and post-processing publish. Plugin-private state stays with its plugin: the candidate tracker's TrackedCluster moves back into the candidates crate. Each plugin still re-exports the names it used to own, so downstream use paths keep compiling. This generalizes what stage-a-plugin-contract already does for the Stage-A owners, and replaces the repo convention that shared types belong in the producing plugin's crate. --- Cargo.toml | 2 + ...031-evesmlm-plugins-share-a-types-crate.md | 86 +++ docs/features/README.md | 2 +- docs/features/evesmlm.md | 15 + evesmlm-types/Cargo.toml | 13 + .../src/candidates.rs | 31 +- evesmlm-types/src/datasets.rs | 488 ++++++++++++++++++ evesmlm-types/src/lib.rs | 35 ++ .../src/localization.rs | 0 plugins/evesmlm-candidates/Cargo.toml | 1 + plugins/evesmlm-candidates/src/lib.rs | 24 +- plugins/evesmlm-candidates/src/tracking.rs | 20 + plugins/evesmlm-fitting/Cargo.toml | 2 +- plugins/evesmlm-fitting/src/gaussian.rs | 2 +- plugins/evesmlm-fitting/src/lib.rs | 488 +----------------- plugins/evesmlm-fitting/src/log_gaussian.rs | 2 +- plugins/evesmlm-fitting/src/mean_xy.rs | 2 +- plugins/evesmlm-fitting/src/phasor.rs | 2 +- .../evesmlm-fitting/src/radial_symmetry.rs | 2 +- plugins/evesmlm-postproc/Cargo.toml | 2 +- .../evesmlm-postproc/src/drift_correction.rs | 2 +- plugins/evesmlm-postproc/src/evaluation.rs | 2 +- plugins/evesmlm-postproc/src/filtering.rs | 2 +- plugins/evesmlm-postproc/src/lib.rs | 8 +- 24 files changed, 704 insertions(+), 529 deletions(-) create mode 100644 docs/adr/031-evesmlm-plugins-share-a-types-crate.md create mode 100644 evesmlm-types/Cargo.toml rename plugins/evesmlm-candidates/src/types.rs => evesmlm-types/src/candidates.rs (87%) create mode 100644 evesmlm-types/src/datasets.rs create mode 100644 evesmlm-types/src/lib.rs rename plugins/evesmlm-fitting/src/types.rs => evesmlm-types/src/localization.rs (100%) create mode 100644 plugins/evesmlm-candidates/src/tracking.rs diff --git a/Cargo.toml b/Cargo.toml index abb6328..3d4a38f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "evesmlm-types", "stage-a-io", "stage-a-plugin-contract", "plugins/stage-a-a1", @@ -27,6 +28,7 @@ augur-core = { path = "../augur-rs/augur-core" } augur-plugin-api = { path = "../augur-rs/augur-plugin-api" } augur-plugin-types = { path = "../augur-rs/augur-plugin-types" } egui = "0.27" +evesmlm-types = { path = "evesmlm-types" } rustfft = "6" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/docs/adr/031-evesmlm-plugins-share-a-types-crate.md b/docs/adr/031-evesmlm-plugins-share-a-types-crate.md new file mode 100644 index 0000000..a75f624 --- /dev/null +++ b/docs/adr/031-evesmlm-plugins-share-a-types-crate.md @@ -0,0 +1,86 @@ +# ADR 031 — Plugins share a types crate, never each other + +**Status:** accepted +**Date:** 2026-08-04 +**Supersedes:** the "shared types are exported from the producing plugin's crate" convention +**Feature brief:** [eveSMLM Pipeline](../features/evesmlm.md) + +## Context + +The eveSMLM plugins form a chain: fitting consumes what candidates publishes, +post-processing consumes what fitting publishes. The repository convention was +that shared types are exported from the producing plugin's crate, so +`augur-plugin-evesmlm-fitting` depended on `augur-plugin-evesmlm-candidates`, and +`augur-plugin-evesmlm-postproc` depended on fitting. + +Plugin crates are built as `crate-type = ["cdylib", "rlib"]`, and each one +invokes `export_plugin!`, which emits `#[no_mangle] augur_plugin_vtable`. A +plugin that depends on another plugin therefore links that plugin's rlib — and +its vtable symbol — into its own `cdylib`. + +The Apple linker tolerates the duplicate. `rust-lld` and MSVC's `link.exe` do +not: + +``` +rust-lld: error: duplicate symbol: augur_plugin_vtable +LNK2005: augur_plugin_vtable already defined … fatal error LNK1169 +``` + +This was invisible for as long as the only build machine was a Mac. It surfaced +the first time CI built the repository on Linux and Windows (ADR 030): macOS +arm64 produced a complete bundle while both other platforms failed to link. +Every non-macOS user was locked out of the eveSMLM chain, and Stage-A users were +locked out of a Windows bundle entirely, because the build is all-or-nothing. + +## Decision + +**A plugin crate may not depend on another plugin crate.** Everything that +crosses a plugin boundary lives in a plain library crate that exports no vtable. + +For eveSMLM that crate is `evesmlm-types`, holding the wire contract +(`EveEvent`, `EveCluster`, `EveCandidates`, `EveLocalization`, +`EveLocalizationResults`, `FitMethod`, the `CTX_*` channel names) and the +current-localization dataset surface that both fitting and post-processing +publish (`current_localizations_registry_for_results`, +`current_localizations_dataset`, `localization_row_id`, +`to_localization_results`, the `CURRENT_LOCALIZATIONS_*` ids). + +Plugin-private types stay in their plugin: the candidate tracker's +`TrackedCluster` moved back out of the shared crate into +`plugins/evesmlm-candidates/src/tracking.rs`. The test is whether another plugin +names the type, not whether it happens to sit next to one that does. + +Each plugin keeps re-exporting the shared names it used to own, so downstream +`use augur_plugin_evesmlm_fitting::EveLocalization` keeps compiling. + +## Consequences + +- The eveSMLM chain links on Linux and Windows, so CI can produce bundles for all + four platforms rather than macOS only. +- Every plugin `cdylib` exports exactly one `augur_plugin_vtable`, which is what + the host's loader assumes in the first place. +- `stage-a-plugin-contract` was already built this way for the Stage-A owner + plugins (ADR 005/006). This generalizes that pattern instead of treating it as + a Stage-A peculiarity. +- The repository convention in `CLAUDE.md` and `CONTRIBUTING.md` — "shared types + between plugins should be exported from the producing plugin's crate" — is + wrong as stated and is replaced by this ADR. +- One more crate per plugin family. That is the cost of the rule, and it is + smaller than the cost of a platform-specific link failure that only shows up + on a machine nobody builds on. + +## Alternatives considered + +**Feature-gate `export_plugin!` and have dependents disable it.** Would keep the +plugin-to-plugin dependency. Rejected: cargo unifies features across a workspace +build, so the `cdylib` target and the same crate consumed as an rlib dependency +resolve to one feature set — the vtable would be on for both, or off for both. + +**Duplicate the shared type definitions in each plugin.** No new crate, and no +shared contract either: the two copies would drift, and the published JSON is +exactly what must not drift. + +**Build the eveSMLM plugins only on macOS.** Considered because the bench PC that +needed a Windows bundle runs Stage-A, not eveSMLM. Rejected: it encodes a +linker accident as a platform policy, and it leaves the bug in place for the +next plugin family that chains. diff --git a/docs/features/README.md b/docs/features/README.md index fb930c9..845eac3 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -23,4 +23,4 @@ Repository-level feature notes for larger plugin suites, interface migrations, a - [Clickable 2D Overlays via Marker `source_row`](./clickable-overlays-source-row.md) — plugin-api ABI 4 `source_dataset_id`/`source_row_id` plumbing and failed-fit click-to-select loop. - [Action Requests And Single-Cluster Refit](./action-requests-and-refit.md) — plugin-declared host actions, eveSMLM refit/commit/discard flow on the `augur.evesmlm.refit_preview` dataset. - [Reconstruction Workflow](./reconstruction.md) — accumulated localization tables rendered and exported by the host. -- [eveSMLM Pipeline](./evesmlm.md) — candidate finding, fitting, and post-processing as three chainable plugins. +- [eveSMLM Pipeline](./evesmlm.md) — candidate finding, fitting, and post-processing as three chainable plugins, chained through the shared `evesmlm-types` contract crate rather than through each other: every plugin exports `augur_plugin_vtable`, so a plugin-to-plugin rlib dependency duplicated that symbol and failed to link on Linux and Windows while macOS accepted it (ADR 031). diff --git a/docs/features/evesmlm.md b/docs/features/evesmlm.md index 8ef6cb6..f65d73f 100644 --- a/docs/features/evesmlm.md +++ b/docs/features/evesmlm.md @@ -8,6 +8,21 @@ The eveSMLM pipeline is implemented as three focused plugins so each stage can b 2. **EVE Candidate Fitting** (`DerivedData`) converts each completed candidate into one or more sub-pixel localization estimates, republishes `EveLocalizationResults` and `LocalizationResults`, and exposes both the shared host-view dataset `augur.evesmlm.current_localizations` and the rejected-fit dataset `augur.evesmlm.rejected_fits`. 3. **EVE Post-Processing** (`DerivedData`) filters, drift-corrects, and evaluates the fitted localizations, then republishes the same host-view dataset id and view ids with the same schema and metadata. +## Shared Contract Crate + +The three plugins do **not** depend on each other. Everything that crosses a +stage boundary — `EveEvent`, `EveCluster`, `EveCandidates`, `EveLocalization`, +`EveLocalizationResults`, `FitMethod`, the `CTX_*` channel names, and the +`augur.evesmlm.current_localizations` dataset/registry builders that both +fitting and post-processing publish — lives in the `evesmlm-types` crate. + +That is not a stylistic choice. Each plugin `cdylib` exports +`augur_plugin_vtable`, so a plugin that linked another plugin's rlib pulled the +symbol in twice. macOS linked it anyway; `rust-lld` and MSVC's `link.exe` +refused, which meant the chain silently only worked on macOS until CI first +built the repository on Linux and Windows (ADR 031). Each plugin still +re-exports the names it used to own, so existing `use` paths keep working. + ## Why Three Plugins - Keeps raw-event grouping separate from numerical fitting, so candidate quality can be inspected directly. diff --git a/evesmlm-types/Cargo.toml b/evesmlm-types/Cargo.toml new file mode 100644 index 0000000..ffab69b --- /dev/null +++ b/evesmlm-types/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "evesmlm-types" +version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true +description = "Shared eveSMLM contract types and current-localization dataset builders for the candidate/fitting/post-processing plugin chain" + +[dependencies] +augur-plugin-api.workspace = true +augur-plugin-types.workspace = true +serde.workspace = true diff --git a/plugins/evesmlm-candidates/src/types.rs b/evesmlm-types/src/candidates.rs similarity index 87% rename from plugins/evesmlm-candidates/src/types.rs rename to evesmlm-types/src/candidates.rs index 6bf8b1b..2320657 100644 --- a/plugins/evesmlm-candidates/src/types.rs +++ b/evesmlm-types/src/candidates.rs @@ -2,6 +2,7 @@ use augur_plugin_api::FfiCdEvent; use serde::{Deserialize, Serialize}; pub const CTX_EVE_CANDIDATES: &str = "augur.evesmlm.candidates"; +pub const ACCEPTED_CANDIDATE_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; fn default_cluster_complete() -> bool { true @@ -17,6 +18,22 @@ pub enum CandidateFindingMethod { } impl CandidateFindingMethod { + pub fn from_index(index: usize) -> Self { + match index { + 1 => Self::Eigenfeature, + 2 => Self::FrameBased, + _ => Self::Dbscan, + } + } + + pub fn index(self) -> usize { + match self { + Self::Dbscan => 0, + Self::Eigenfeature => 1, + Self::FrameBased => 2, + } + } + pub fn label(self) -> &'static str { match self { Self::Dbscan => "DBSCAN", @@ -124,17 +141,3 @@ pub struct EveCandidates { pub n_events_processed: usize, pub finding_method: CandidateFindingMethod, } - -#[derive(Debug, Clone)] -pub(crate) struct TrackedCluster { - pub id: u64, - pub centroid_x: f64, - pub centroid_y: f64, - pub event_count: usize, - pub last_seen_frame: u64, - pub last_grown_frame: u64, - pub frames_since_growth: usize, - pub complete: bool, - pub emitted: bool, - pub cluster: EveCluster, -} diff --git a/evesmlm-types/src/datasets.rs b/evesmlm-types/src/datasets.rs new file mode 100644 index 0000000..fe3d43f --- /dev/null +++ b/evesmlm-types/src/datasets.rs @@ -0,0 +1,488 @@ +//! The current-localization dataset, its schema and the host-view registry +//! built from it, plus the conversion to the standard `LocalizationResults`. +//! +//! These live here rather than in the fitting plugin because post-processing +//! republishes the same dataset — see the crate docs for why a plugin must not +//! link another plugin's rlib. + +use augur_plugin_api::{ + HostDatasetDescriptor, HostDatasetDisplayMetadata, HostDatasetKind, HostDatasetRelation, + HostMarkerShape, HostViewDescriptor, HostViewKind, HostViewPlacement, HostViewRegistry, + TableColumn, TableColumnData, TableColumnDisplayEntry, TableColumnDisplayFormat, + TableColumnDisplayMetadata, TableColumnValues, TableCoordinateSpace2d, TableCoordinateSpace3d, + TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, +}; +use augur_plugin_types::{Localization, LocalizationResults}; + +use crate::candidates::ACCEPTED_CANDIDATE_EVENTS_DATASET_ID; +use crate::localization::{EveLocalization, EveLocalizationResults}; + +pub const CURRENT_LOCALIZATIONS_DATASET_ID: &str = "augur.evesmlm.current_localizations"; +pub const CURRENT_LOCALIZATIONS_LAYER_ID: &str = "augur.layer.evesmlm.current_localizations"; +pub const CURRENT_LOCALIZATIONS_VIEW_ID: &str = "augur.evesmlm.current_localizations.compact"; +pub const CURRENT_LOCALIZATIONS_3D_VIEW_ID: &str = "augur.evesmlm.current_localizations.scatter3d"; + +pub fn current_localizations_registry() -> HostViewRegistry { + current_localizations_registry_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_registry_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> HostViewRegistry { + HostViewRegistry { + datasets: vec![HostDatasetDescriptor { + id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + title: "Current EVE localizations".into(), + kind: HostDatasetKind::TableV1(current_localizations_schema_for_results( + results, + sensor_dims, + )), + empty_message: "No EVE localizations in the current frame.".into(), + display: Some(HostDatasetDisplayMetadata { + layer_title: Some("Current EVE localizations".into()), + default_visibility: Some(true), + default_color: Some([90, 170, 255, 255]), + default_marker_shape: Some(HostMarkerShape::Cross), + default_size: Some(6.0), + }), + relations: vec![HostDatasetRelation { + target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), + via_column: "cluster_id".into(), + target_column: "cluster_id".into(), + }], + }], + views: vec![ + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), + title: "Current Localizations".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::AnalysisPanel, + kind: HostViewKind::CompactTable, + }, + HostViewDescriptor { + id: CURRENT_LOCALIZATIONS_3D_VIEW_ID.into(), + title: "Current Localizations 3D".into(), + dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), + placement: HostViewPlacement::Window, + kind: HostViewKind::Scatter3dFromTable { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + }, + }, + ], + actions: Vec::new(), + } +} + +pub fn current_localizations_schema() -> TableSchema { + current_localizations_schema_for_results(&EveLocalizationResults::default(), None) +} + +pub fn current_localizations_schema_for_results( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> TableSchema { + TableSchema { + columns: vec![ + TableColumn { + id: "row_id".into(), + title: "ID".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "cluster_id".into(), + title: "Cluster".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "timestamp_us".into(), + title: "Timestamp (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_start_us".into(), + title: "Span Start (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "span_end_us".into(), + title: "Span End (us)".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "x_px".into(), + title: "X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "y_px".into(), + title: "Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_x_px".into(), + title: "Sigma X (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "sigma_y_px".into(), + title: "Sigma Y (px)".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "n_events".into(), + title: "Events".into(), + value_type: TableValueType::U64, + }, + TableColumn { + id: "polarity_balance".into(), + title: "Polarity balance".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_residual".into(), + title: "Fit residual".into(), + value_type: TableValueType::F64, + }, + TableColumn { + id: "fit_method".into(), + title: "Fit method".into(), + value_type: TableValueType::String, + }, + ], + coordinate_space_2d: current_localizations_2d_space(results, sensor_dims), + coordinate_space_3d: current_localizations_3d_space(results, sensor_dims), + row_id_column: Some("row_id".into()), + time_column: Some("timestamp_us".into()), + layer_id: Some(CURRENT_LOCALIZATIONS_LAYER_ID.into()), + semantic_label: Some("localizations".into()), + provenance: Some(TableRowProvenance { + anchor_time_column: Some("timestamp_us".into()), + span_start_column: Some("span_start_us".into()), + span_end_column: Some("span_end_us".into()), + anchor_frame_column: None, + }), + column_display: vec![ + TableColumnDisplayEntry { + column_id: "row_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + hide_in_compact: true, + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "cluster_id".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Identifier), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "timestamp_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Time".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_start_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span start".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "span_end_us".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::TimestampMicros), + label: Some("Span end".into()), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_x_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "sigma_y_px".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_residual".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), + ..Default::default() + }, + }, + TableColumnDisplayEntry { + column_id: "fit_method".into(), + display: TableColumnDisplayMetadata { + format: Some(TableColumnDisplayFormat::Category), + ..Default::default() + }, + }, + ], + } +} + +pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { + TableDatasetV1::new(vec![ + TableColumnData { + column_id: "row_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(localization_row_id) + .collect(), + ), + }, + TableColumnData { + column_id: "cluster_id".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.cluster_id) + .collect(), + ), + }, + TableColumnData { + column_id: "timestamp_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.timestamp_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_start_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_start_us) + .collect(), + ), + }, + TableColumnData { + column_id: "span_end_us".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.span_end_us) + .collect(), + ), + }, + TableColumnData { + column_id: "x_px".into(), + values: TableColumnValues::F64( + results.localizations.iter().map(|value| value.x).collect(), + ), + }, + TableColumnData { + column_id: "y_px".into(), + values: TableColumnValues::F64( + results.localizations.iter().map(|value| value.y).collect(), + ), + }, + TableColumnData { + column_id: "sigma_x_px".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.sigma_x) + .collect(), + ), + }, + TableColumnData { + column_id: "sigma_y_px".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.sigma_y) + .collect(), + ), + }, + TableColumnData { + column_id: "n_events".into(), + values: TableColumnValues::U64( + results + .localizations + .iter() + .map(|value| value.n_events as u64) + .collect(), + ), + }, + TableColumnData { + column_id: "polarity_balance".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.polarity_balance) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_residual".into(), + values: TableColumnValues::F64( + results + .localizations + .iter() + .map(|value| value.fit_residual) + .collect(), + ), + }, + TableColumnData { + column_id: "fit_method".into(), + values: TableColumnValues::String( + results + .localizations + .iter() + .map(|value| value.fit_method.label().to_owned()) + .collect(), + ), + }, + ]) + .expect("current localization columns should stay aligned") +} + +fn current_localizations_2d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results)) + .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { + x_column: "x_px".into(), + y_column: "y_px".into(), + x_min, + x_max, + y_min, + y_max, + }) +} + +fn current_localizations_3d_space( + results: &EveLocalizationResults, + sensor_dims: Option<(u16, u16)>, +) -> Option { + let (x_min, x_max, y_min, y_max) = sensor_dims + .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) + .or_else(|| localization_xy_bounds(results))?; + let (z_min, z_max) = localization_time_bounds(results)?; + Some(TableCoordinateSpace3d { + x_column: "x_px".into(), + y_column: "y_px".into(), + z_column: "timestamp_us".into(), + x_min, + x_max, + y_min, + y_max, + z_min, + z_max, + }) +} +pub fn localization_xy_bounds(results: &EveLocalizationResults) -> Option<(f64, f64, f64, f64)> { + let mut localizations = results.localizations.iter(); + let first = localizations.next()?; + let mut x_min = first.x; + let mut x_max = first.x; + let mut y_min = first.y; + let mut y_max = first.y; + for localization in localizations { + x_min = x_min.min(localization.x); + x_max = x_max.max(localization.x); + y_min = y_min.min(localization.y); + y_max = y_max.max(localization.y); + } + Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) +} + +pub fn localization_time_bounds(results: &EveLocalizationResults) -> Option<(f64, f64)> { + if let Some(first) = results.localizations.first() { + let mut min_time = first.timestamp_us; + let mut max_time = first.timestamp_us; + for localization in &results.localizations { + min_time = min_time.min(localization.timestamp_us); + max_time = max_time.max(localization.timestamp_us); + } + return Some((min_time as f64, max_time.max(min_time) as f64)); + } + + if results.frame_window_end_us >= results.frame_window_start_us { + return Some(( + results.frame_window_start_us as f64, + results.frame_window_end_us as f64, + )); + } + + None +} + +pub fn localization_row_id(localization: &EveLocalization) -> u64 { + localization.cluster_id.rotate_left(3) + ^ localization.timestamp_us + ^ localization.x.to_bits().rotate_left(7) + ^ localization.y.to_bits().rotate_left(19) + ^ localization.sigma_x.to_bits().rotate_left(31) + ^ localization.sigma_y.to_bits().rotate_left(43) + ^ localization.fit_residual.to_bits().rotate_left(53) + ^ (localization.n_events as u64).rotate_left(11) + ^ (localization.fit_method.index() as u64).rotate_left(59) + ^ localization.span_start_us.rotate_left(17) + ^ localization.span_end_us.rotate_left(29) +} + +pub fn to_localization_results(results: &EveLocalizationResults) -> LocalizationResults { + LocalizationResults { + localizations: results + .localizations + .iter() + .map(|localization| Localization { + x: localization.x, + y: localization.y, + sigma_x: localization.sigma_x, + sigma_y: localization.sigma_y, + amplitude: 0.0, + background: 0.0, + timestamp_us: localization.timestamp_us, + fit_error: localization.fit_residual, + }) + .collect(), + frame_window_start_us: results.frame_window_start_us, + frame_window_end_us: results.frame_window_end_us, + } +} diff --git a/evesmlm-types/src/lib.rs b/evesmlm-types/src/lib.rs new file mode 100644 index 0000000..00007be --- /dev/null +++ b/evesmlm-types/src/lib.rs @@ -0,0 +1,35 @@ +//! Shared eveSMLM contract. +//! +//! The candidate, fitting and post-processing plugins form a chain: fitting +//! consumes what candidates publishes, and post-processing consumes what +//! fitting publishes. Expressing that by having one plugin crate depend on +//! another looks natural, but plugin crates are `cdylib`s that each export +//! `augur_plugin_vtable` — and a plugin that links another plugin's rlib pulls +//! that symbol in twice. The Apple linker tolerates the duplicate; `rust-lld` +//! and MSVC's `link.exe` do not, so the chain built on macOS and failed to link +//! on Linux and Windows. +//! +//! Everything that crosses a plugin boundary therefore lives here, in a plain +//! library crate that exports no vtable. Plugins depend on this crate, never on +//! each other. + +pub mod candidates; +pub mod datasets; +pub mod localization; + +pub use candidates::{ + CandidateFindingMethod, ClusterBoundary, EveCandidates, EveCluster, EveEvent, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID, CTX_EVE_CANDIDATES, +}; +pub use datasets::{ + current_localizations_dataset, current_localizations_registry, + current_localizations_registry_for_results, current_localizations_schema, + current_localizations_schema_for_results, localization_row_id, localization_time_bounds, + localization_xy_bounds, to_localization_results, CURRENT_LOCALIZATIONS_3D_VIEW_ID, + CURRENT_LOCALIZATIONS_DATASET_ID, CURRENT_LOCALIZATIONS_LAYER_ID, + CURRENT_LOCALIZATIONS_VIEW_ID, +}; +pub use localization::{ + EveLocalization, EveLocalizationResults, FitMethod, RejectedFitRow, RejectionReason, + CTX_EVE_LOCALIZATION_RESULTS, +}; diff --git a/plugins/evesmlm-fitting/src/types.rs b/evesmlm-types/src/localization.rs similarity index 100% rename from plugins/evesmlm-fitting/src/types.rs rename to evesmlm-types/src/localization.rs diff --git a/plugins/evesmlm-candidates/Cargo.toml b/plugins/evesmlm-candidates/Cargo.toml index f8e7d66..07c2377 100644 --- a/plugins/evesmlm-candidates/Cargo.toml +++ b/plugins/evesmlm-candidates/Cargo.toml @@ -11,6 +11,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true +evesmlm-types.workspace = true nalgebra = "0.33" serde.workspace = true serde_json.workspace = true diff --git a/plugins/evesmlm-candidates/src/lib.rs b/plugins/evesmlm-candidates/src/lib.rs index d51776b..4556b2e 100644 --- a/plugins/evesmlm-candidates/src/lib.rs +++ b/plugins/evesmlm-candidates/src/lib.rs @@ -6,7 +6,7 @@ pub mod dbscan; pub mod eigenfeature; -pub mod types; +mod tracking; use std::collections::{HashMap, HashSet}; @@ -23,11 +23,11 @@ use augur_plugin_api::{ }; use serde_json::{json, Value}; -use types::TrackedCluster; -pub use types::{ +pub use evesmlm_types::{ CandidateFindingMethod, ClusterBoundary, EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, }; +use tracking::TrackedCluster; const KERNEL_G1: [f64; 5] = [1.0 / 16.0, 0.25, 3.0 / 8.0, 0.25, 1.0 / 16.0]; const KERNEL_G2: [f64; 9] = [ @@ -134,24 +134,6 @@ impl PolarityMode { } } -impl CandidateFindingMethod { - fn from_index(index: usize) -> Self { - match index { - 1 => Self::Eigenfeature, - 2 => Self::FrameBased, - _ => Self::Dbscan, - } - } - - fn index(self) -> usize { - match self { - Self::Dbscan => 0, - Self::Eigenfeature => 1, - Self::FrameBased => 2, - } - } -} - #[derive(Debug, Clone)] pub struct CandidateSettings { pub finding_method: CandidateFindingMethod, diff --git a/plugins/evesmlm-candidates/src/tracking.rs b/plugins/evesmlm-candidates/src/tracking.rs new file mode 100644 index 0000000..04fd140 --- /dev/null +++ b/plugins/evesmlm-candidates/src/tracking.rs @@ -0,0 +1,20 @@ +//! Internal cluster-tracking bookkeeping. +//! +//! Not part of the cross-plugin contract — the shared eveSMLM types live in +//! the `evesmlm-types` crate. + +use evesmlm_types::EveCluster; + +#[derive(Debug, Clone)] +pub(crate) struct TrackedCluster { + pub id: u64, + pub centroid_x: f64, + pub centroid_y: f64, + pub event_count: usize, + pub last_seen_frame: u64, + pub last_grown_frame: u64, + pub frames_since_growth: usize, + pub complete: bool, + pub emitted: bool, + pub cluster: EveCluster, +} diff --git a/plugins/evesmlm-fitting/Cargo.toml b/plugins/evesmlm-fitting/Cargo.toml index 4e3e8c1..1e14aaa 100644 --- a/plugins/evesmlm-fitting/Cargo.toml +++ b/plugins/evesmlm-fitting/Cargo.toml @@ -12,7 +12,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true augur-plugin-types.workspace = true -augur-plugin-evesmlm-candidates = { path = "../evesmlm-candidates" } +evesmlm-types.workspace = true levenberg-marquardt = "0.14" nalgebra = "0.33" num-complex = "0.4" diff --git a/plugins/evesmlm-fitting/src/gaussian.rs b/plugins/evesmlm-fitting/src/gaussian.rs index 429e8dd..848113d 100644 --- a/plugins/evesmlm-fitting/src/gaussian.rs +++ b/plugins/evesmlm-fitting/src/gaussian.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use crate::{mean_xy, FitEstimate}; diff --git a/plugins/evesmlm-fitting/src/lib.rs b/plugins/evesmlm-fitting/src/lib.rs index 31b83b3..51103d9 100644 --- a/plugins/evesmlm-fitting/src/lib.rs +++ b/plugins/evesmlm-fitting/src/lib.rs @@ -10,7 +10,6 @@ pub mod log_gaussian; pub mod mean_xy; pub mod phasor; pub mod radial_symmetry; -pub mod types; use augur_plugin_api::{ export_plugin, AnalysisSeverity, EventStoreHandle, FfiColorRgba, FfiMarkerOverlayItem, @@ -26,22 +25,21 @@ use augur_plugin_api::{ TableColumnDisplayMetadata, TableColumnValues, TableCoordinateSpace2d, TableCoordinateSpace3d, TableDatasetV1, TableRowProvenance, TableSchema, TableValueType, }; -pub use augur_plugin_evesmlm_candidates::{ - EveCandidates, EveCluster, EveEvent, CTX_EVE_CANDIDATES, -}; use augur_plugin_types::{Localization, LocalizationResults, CTX_LOCALIZATION_RESULTS}; -use serde_json::{json, Value}; -pub use types::{ +pub use evesmlm_types::{ + current_localizations_dataset, current_localizations_registry, + current_localizations_registry_for_results, current_localizations_schema, + current_localizations_schema_for_results, localization_row_id, localization_time_bounds, + localization_xy_bounds, to_localization_results, EveCandidates, EveCluster, EveEvent, EveLocalization, EveLocalizationResults, FitMethod, RejectedFitRow, RejectionReason, - CTX_EVE_LOCALIZATION_RESULTS, + ACCEPTED_CANDIDATE_EVENTS_DATASET_ID, CTX_EVE_CANDIDATES, CTX_EVE_LOCALIZATION_RESULTS, + CURRENT_LOCALIZATIONS_3D_VIEW_ID, CURRENT_LOCALIZATIONS_DATASET_ID, + CURRENT_LOCALIZATIONS_LAYER_ID, CURRENT_LOCALIZATIONS_VIEW_ID, }; +use serde_json::{json, Value}; const OVERLAY_COLOR: [u8; 4] = [60, 220, 140, 220]; const CANDIDATE_DEPENDENCY: [&str; 1] = ["EVE Candidate Finding"]; -pub const CURRENT_LOCALIZATIONS_DATASET_ID: &str = "augur.evesmlm.current_localizations"; -pub const CURRENT_LOCALIZATIONS_LAYER_ID: &str = "augur.layer.evesmlm.current_localizations"; -pub const CURRENT_LOCALIZATIONS_VIEW_ID: &str = "augur.evesmlm.current_localizations.compact"; -pub const CURRENT_LOCALIZATIONS_3D_VIEW_ID: &str = "augur.evesmlm.current_localizations.scatter3d"; pub const REJECTED_FITS_DATASET_ID: &str = "augur.evesmlm.rejected_fits"; pub const REJECTED_FITS_LAYER_ID: &str = "augur.layer.evesmlm.rejected_fits"; pub const REJECTED_FITS_COMPACT_VIEW_ID: &str = "augur.evesmlm.rejected_fits.compact"; @@ -52,406 +50,10 @@ pub const REFIT_PREVIEW_DATASET_ID: &str = "augur.evesmlm.refit_preview"; pub const REFIT_PREVIEW_LAYER_ID: &str = "augur.layer.evesmlm.refit_preview"; pub const REFIT_PREVIEW_VIEW_ID: &str = "augur.evesmlm.refit_preview.compact"; -pub const ACCEPTED_CANDIDATE_EVENTS_DATASET_ID: &str = "augur.evesmlm.candidates.accepted_events"; - pub const ACTION_REFIT_CLUSTER: &str = "augur.evesmlm.refit_cluster"; pub const ACTION_COMMIT_REFIT: &str = "augur.evesmlm.commit_refit"; pub const ACTION_DISCARD_REFIT: &str = "augur.evesmlm.discard_refit"; -pub fn current_localizations_registry() -> HostViewRegistry { - current_localizations_registry_for_results(&EveLocalizationResults::default(), None) -} - -pub fn current_localizations_registry_for_results( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> HostViewRegistry { - HostViewRegistry { - datasets: vec![HostDatasetDescriptor { - id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - title: "Current EVE localizations".into(), - kind: HostDatasetKind::TableV1(current_localizations_schema_for_results( - results, - sensor_dims, - )), - empty_message: "No EVE localizations in the current frame.".into(), - display: Some(HostDatasetDisplayMetadata { - layer_title: Some("Current EVE localizations".into()), - default_visibility: Some(true), - default_color: Some([90, 170, 255, 255]), - default_marker_shape: Some(HostMarkerShape::Cross), - default_size: Some(6.0), - }), - relations: vec![HostDatasetRelation { - target_dataset_id: ACCEPTED_CANDIDATE_EVENTS_DATASET_ID.into(), - via_column: "cluster_id".into(), - target_column: "cluster_id".into(), - }], - }], - views: vec![ - HostViewDescriptor { - id: CURRENT_LOCALIZATIONS_VIEW_ID.into(), - title: "Current Localizations".into(), - dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - placement: HostViewPlacement::AnalysisPanel, - kind: HostViewKind::CompactTable, - }, - HostViewDescriptor { - id: CURRENT_LOCALIZATIONS_3D_VIEW_ID.into(), - title: "Current Localizations 3D".into(), - dataset_id: CURRENT_LOCALIZATIONS_DATASET_ID.into(), - placement: HostViewPlacement::Window, - kind: HostViewKind::Scatter3dFromTable { - x_column: "x_px".into(), - y_column: "y_px".into(), - z_column: "timestamp_us".into(), - }, - }, - ], - actions: Vec::new(), - } -} - -pub fn current_localizations_schema() -> TableSchema { - current_localizations_schema_for_results(&EveLocalizationResults::default(), None) -} - -pub fn current_localizations_schema_for_results( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> TableSchema { - TableSchema { - columns: vec![ - TableColumn { - id: "row_id".into(), - title: "ID".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "cluster_id".into(), - title: "Cluster".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "timestamp_us".into(), - title: "Timestamp (us)".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "span_start_us".into(), - title: "Span Start (us)".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "span_end_us".into(), - title: "Span End (us)".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "x_px".into(), - title: "X (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "y_px".into(), - title: "Y (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "sigma_x_px".into(), - title: "Sigma X (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "sigma_y_px".into(), - title: "Sigma Y (px)".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "n_events".into(), - title: "Events".into(), - value_type: TableValueType::U64, - }, - TableColumn { - id: "polarity_balance".into(), - title: "Polarity balance".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "fit_residual".into(), - title: "Fit residual".into(), - value_type: TableValueType::F64, - }, - TableColumn { - id: "fit_method".into(), - title: "Fit method".into(), - value_type: TableValueType::String, - }, - ], - coordinate_space_2d: current_localizations_2d_space(results, sensor_dims), - coordinate_space_3d: current_localizations_3d_space(results, sensor_dims), - row_id_column: Some("row_id".into()), - time_column: Some("timestamp_us".into()), - layer_id: Some(CURRENT_LOCALIZATIONS_LAYER_ID.into()), - semantic_label: Some("localizations".into()), - provenance: Some(TableRowProvenance { - anchor_time_column: Some("timestamp_us".into()), - span_start_column: Some("span_start_us".into()), - span_end_column: Some("span_end_us".into()), - anchor_frame_column: None, - }), - column_display: vec![ - TableColumnDisplayEntry { - column_id: "row_id".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::Identifier), - hide_in_compact: true, - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "cluster_id".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::Identifier), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "timestamp_us".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::TimestampMicros), - label: Some("Time".into()), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "span_start_us".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::TimestampMicros), - label: Some("Span start".into()), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "span_end_us".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::TimestampMicros), - label: Some("Span end".into()), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "x_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "y_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 1 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "sigma_x_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "sigma_y_px".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 2 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "fit_residual".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::FixedPrecision { digits: 3 }), - ..Default::default() - }, - }, - TableColumnDisplayEntry { - column_id: "fit_method".into(), - display: TableColumnDisplayMetadata { - format: Some(TableColumnDisplayFormat::Category), - ..Default::default() - }, - }, - ], - } -} - -pub fn current_localizations_dataset(results: &EveLocalizationResults) -> TableDatasetV1 { - TableDatasetV1::new(vec![ - TableColumnData { - column_id: "row_id".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(localization_row_id) - .collect(), - ), - }, - TableColumnData { - column_id: "cluster_id".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.cluster_id) - .collect(), - ), - }, - TableColumnData { - column_id: "timestamp_us".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.timestamp_us) - .collect(), - ), - }, - TableColumnData { - column_id: "span_start_us".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.span_start_us) - .collect(), - ), - }, - TableColumnData { - column_id: "span_end_us".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.span_end_us) - .collect(), - ), - }, - TableColumnData { - column_id: "x_px".into(), - values: TableColumnValues::F64( - results.localizations.iter().map(|value| value.x).collect(), - ), - }, - TableColumnData { - column_id: "y_px".into(), - values: TableColumnValues::F64( - results.localizations.iter().map(|value| value.y).collect(), - ), - }, - TableColumnData { - column_id: "sigma_x_px".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.sigma_x) - .collect(), - ), - }, - TableColumnData { - column_id: "sigma_y_px".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.sigma_y) - .collect(), - ), - }, - TableColumnData { - column_id: "n_events".into(), - values: TableColumnValues::U64( - results - .localizations - .iter() - .map(|value| value.n_events as u64) - .collect(), - ), - }, - TableColumnData { - column_id: "polarity_balance".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.polarity_balance) - .collect(), - ), - }, - TableColumnData { - column_id: "fit_residual".into(), - values: TableColumnValues::F64( - results - .localizations - .iter() - .map(|value| value.fit_residual) - .collect(), - ), - }, - TableColumnData { - column_id: "fit_method".into(), - values: TableColumnValues::String( - results - .localizations - .iter() - .map(|value| value.fit_method.label().to_owned()) - .collect(), - ), - }, - ]) - .expect("current localization columns should stay aligned") -} - -fn current_localizations_2d_space( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> Option { - sensor_dims - .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) - .or_else(|| localization_xy_bounds(results)) - .map(|(x_min, x_max, y_min, y_max)| TableCoordinateSpace2d { - x_column: "x_px".into(), - y_column: "y_px".into(), - x_min, - x_max, - y_min, - y_max, - }) -} - -fn current_localizations_3d_space( - results: &EveLocalizationResults, - sensor_dims: Option<(u16, u16)>, -) -> Option { - let (x_min, x_max, y_min, y_max) = sensor_dims - .map(|(width, height)| (0.0, f64::from(width), 0.0, f64::from(height))) - .or_else(|| localization_xy_bounds(results))?; - let (z_min, z_max) = localization_time_bounds(results)?; - Some(TableCoordinateSpace3d { - x_column: "x_px".into(), - y_column: "y_px".into(), - z_column: "timestamp_us".into(), - x_min, - x_max, - y_min, - y_max, - z_min, - z_max, - }) -} - pub fn refit_preview_registry_for_results( results: &EveLocalizationResults, sensor_dims: Option<(u16, u16)>, @@ -563,57 +165,6 @@ fn refit_action_param_schema() -> SettingsSchema { } } -fn localization_xy_bounds(results: &EveLocalizationResults) -> Option<(f64, f64, f64, f64)> { - let mut localizations = results.localizations.iter(); - let first = localizations.next()?; - let mut x_min = first.x; - let mut x_max = first.x; - let mut y_min = first.y; - let mut y_max = first.y; - for localization in localizations { - x_min = x_min.min(localization.x); - x_max = x_max.max(localization.x); - y_min = y_min.min(localization.y); - y_max = y_max.max(localization.y); - } - Some((x_min, x_max.max(x_min), y_min, y_max.max(y_min))) -} - -fn localization_time_bounds(results: &EveLocalizationResults) -> Option<(f64, f64)> { - if let Some(first) = results.localizations.first() { - let mut min_time = first.timestamp_us; - let mut max_time = first.timestamp_us; - for localization in &results.localizations { - min_time = min_time.min(localization.timestamp_us); - max_time = max_time.max(localization.timestamp_us); - } - return Some((min_time as f64, max_time.max(min_time) as f64)); - } - - if results.frame_window_end_us >= results.frame_window_start_us { - return Some(( - results.frame_window_start_us as f64, - results.frame_window_end_us as f64, - )); - } - - None -} - -pub fn localization_row_id(localization: &EveLocalization) -> u64 { - localization.cluster_id.rotate_left(3) - ^ localization.timestamp_us - ^ localization.x.to_bits().rotate_left(7) - ^ localization.y.to_bits().rotate_left(19) - ^ localization.sigma_x.to_bits().rotate_left(31) - ^ localization.sigma_y.to_bits().rotate_left(43) - ^ localization.fit_residual.to_bits().rotate_left(53) - ^ (localization.n_events as u64).rotate_left(11) - ^ (localization.fit_method.index() as u64).rotate_left(59) - ^ localization.span_start_us.rotate_left(17) - ^ localization.span_end_us.rotate_left(29) -} - pub fn rejected_fit_row_id(row: &RejectedFitRow) -> u64 { row.timestamp_us ^ row.cluster_id.rotate_left(7) @@ -2344,27 +1895,6 @@ fn estimate_timestamp_us(events: &[EveEvent], x: f64, y: f64, radius: f64) -> u6 } } -pub fn to_localization_results(results: &EveLocalizationResults) -> LocalizationResults { - LocalizationResults { - localizations: results - .localizations - .iter() - .map(|localization| Localization { - x: localization.x, - y: localization.y, - sigma_x: localization.sigma_x, - sigma_y: localization.sigma_y, - amplitude: 0.0, - background: 0.0, - timestamp_us: localization.timestamp_us, - fit_error: localization.fit_residual, - }) - .collect(), - frame_window_start_us: results.frame_window_start_us, - frame_window_end_us: results.frame_window_end_us, - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/plugins/evesmlm-fitting/src/log_gaussian.rs b/plugins/evesmlm-fitting/src/log_gaussian.rs index ad59c90..241e052 100644 --- a/plugins/evesmlm-fitting/src/log_gaussian.rs +++ b/plugins/evesmlm-fitting/src/log_gaussian.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use nalgebra::{DMatrix, DVector}; use crate::FitEstimate; diff --git a/plugins/evesmlm-fitting/src/mean_xy.rs b/plugins/evesmlm-fitting/src/mean_xy.rs index 95462b8..1c95852 100644 --- a/plugins/evesmlm-fitting/src/mean_xy.rs +++ b/plugins/evesmlm-fitting/src/mean_xy.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use crate::FitEstimate; diff --git a/plugins/evesmlm-fitting/src/phasor.rs b/plugins/evesmlm-fitting/src/phasor.rs index dccdbb9..cba433b 100644 --- a/plugins/evesmlm-fitting/src/phasor.rs +++ b/plugins/evesmlm-fitting/src/phasor.rs @@ -1,6 +1,6 @@ use std::f64::consts::TAU; -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use num_complex::Complex64; use crate::{mean_xy, FitEstimate}; diff --git a/plugins/evesmlm-fitting/src/radial_symmetry.rs b/plugins/evesmlm-fitting/src/radial_symmetry.rs index 6c45441..0acee29 100644 --- a/plugins/evesmlm-fitting/src/radial_symmetry.rs +++ b/plugins/evesmlm-fitting/src/radial_symmetry.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_candidates::EveCluster; +use evesmlm_types::EveCluster; use crate::FitEstimate; diff --git a/plugins/evesmlm-postproc/Cargo.toml b/plugins/evesmlm-postproc/Cargo.toml index ec547d4..b18013e 100644 --- a/plugins/evesmlm-postproc/Cargo.toml +++ b/plugins/evesmlm-postproc/Cargo.toml @@ -12,6 +12,6 @@ crate-type = ["cdylib", "rlib"] [dependencies] augur-plugin-api.workspace = true augur-plugin-types.workspace = true -augur-plugin-evesmlm-fitting = { path = "../evesmlm-fitting" } +evesmlm-types.workspace = true nalgebra = "0.33" serde_json.workspace = true diff --git a/plugins/evesmlm-postproc/src/drift_correction.rs b/plugins/evesmlm-postproc/src/drift_correction.rs index 9f16e50..ce85aba 100644 --- a/plugins/evesmlm-postproc/src/drift_correction.rs +++ b/plugins/evesmlm-postproc/src/drift_correction.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_fitting::EveLocalizationResults; +use evesmlm_types::EveLocalizationResults; pub fn estimate_correction_shift( reference_points: &[(f64, f64)], diff --git a/plugins/evesmlm-postproc/src/evaluation.rs b/plugins/evesmlm-postproc/src/evaluation.rs index eac5e72..ee0bd25 100644 --- a/plugins/evesmlm-postproc/src/evaluation.rs +++ b/plugins/evesmlm-postproc/src/evaluation.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use augur_plugin_evesmlm_fitting::{EveLocalization, EveLocalizationResults}; +use evesmlm_types::{EveLocalization, EveLocalizationResults}; const DEFAULT_PSF_SIZE: usize = 9; const TRACK_LINK_RADIUS_PX: f64 = 1.5; diff --git a/plugins/evesmlm-postproc/src/filtering.rs b/plugins/evesmlm-postproc/src/filtering.rs index 752f125..a0b2de7 100644 --- a/plugins/evesmlm-postproc/src/filtering.rs +++ b/plugins/evesmlm-postproc/src/filtering.rs @@ -1,4 +1,4 @@ -use augur_plugin_evesmlm_fitting::{EveLocalization, EveLocalizationResults}; +use evesmlm_types::{EveLocalization, EveLocalizationResults}; pub fn filter_results( results: &EveLocalizationResults, diff --git a/plugins/evesmlm-postproc/src/lib.rs b/plugins/evesmlm-postproc/src/lib.rs index 4413f9d..44b0739 100644 --- a/plugins/evesmlm-postproc/src/lib.rs +++ b/plugins/evesmlm-postproc/src/lib.rs @@ -15,13 +15,13 @@ use augur_plugin_api::{ PluginInput, SettingItem, SettingKind, SettingsSchema, SettingsSection, StatusEntry, CTX_GLOBAL_SETTINGS, }; -pub use augur_plugin_evesmlm_fitting::{ +use augur_plugin_types::CTX_LOCALIZATION_RESULTS; +use evaluation::EvaluationState; +pub use evesmlm_types::{ current_localizations_dataset, current_localizations_registry_for_results, localization_row_id, to_localization_results, EveLocalization, EveLocalizationResults, FitMethod, CTX_EVE_LOCALIZATION_RESULTS, CURRENT_LOCALIZATIONS_DATASET_ID, CURRENT_LOCALIZATIONS_LAYER_ID, }; -use augur_plugin_types::CTX_LOCALIZATION_RESULTS; -use evaluation::EvaluationState; use serde_json::{json, Value}; const OVERLAY_COLOR: [u8; 4] = [90, 170, 255, 220]; @@ -690,7 +690,7 @@ mod tests { #[test] fn current_localizations_descriptor_matches_fitting() { - use augur_plugin_evesmlm_fitting::current_localizations_registry_for_results as fitting_registry; + use evesmlm_types::current_localizations_registry_for_results as fitting_registry; let results = EveLocalizationResults::default(); let fitting = fitting_registry(&results, None); let postproc = current_localizations_registry_for_results(&results, None); From e5cd98b75010263a0338dd15e1e4dc2179c2363e Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Tue, 4 Aug 2026 23:16:13 +0200 Subject: [PATCH 5/6] =?UTF-8?q?fix(stage-a):=20=F0=9F=90=9B=20find=20the?= =?UTF-8?q?=20Teensy=20on=20Windows'=20nameless=20COM=20ports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port discovery filtered candidates by the two Unix name patterns (`cu.usbmodem`, `ttyACM`) before probing. Windows names no device — every port is `COMn` — so an attached, correctly driven Teensy was filtered out before any probe could run, and both plugins reported "no USB serial device found (looked for usbmodem/ttyACM)": the two things Windows cannot produce. Move the filter into `stage-a-io::transport::candidate_ports()`, where it is platform-aware: the callout node on macOS, `ttyACM*` on Linux, and every USB-classified port on Windows, falling back to the whole list when the OS classifies nothing. What identifies the device is still the probe (HELLO on the command port, PDA1 frames on the stream port); the filter only keeps probes off unrelated ports. Also open every port with DTR asserted. macOS and Linux do this implicitly, Windows does not, so a sketch gating on `if (Serial)` would stay silent even once the right port was found. The filter existed in four places in two implementations; it is now one function with unit tests covering both platform branches, and the failure message names the ports the OS actually enumerated. Refs ADR 032 --- ...teensy-port-discovery-is-platform-aware.md | 114 ++++++++++++ docs/features/README.md | 4 +- docs/features/stage-a-modulation.md | 11 ++ docs/features/stage-a-photodiode.md | 15 +- plugins/stage-a-modulation/README.md | 12 +- plugins/stage-a-modulation/src/lib.rs | 23 +-- plugins/stage-a-photodiode/Cargo.toml | 4 +- plugins/stage-a-photodiode/README.md | 9 +- plugins/stage-a-photodiode/src/lib.rs | 45 ++--- stage-a-io/src/transport.rs | 169 +++++++++++++++--- 10 files changed, 329 insertions(+), 77 deletions(-) create mode 100644 docs/adr/032-teensy-port-discovery-is-platform-aware.md diff --git a/docs/adr/032-teensy-port-discovery-is-platform-aware.md b/docs/adr/032-teensy-port-discovery-is-platform-aware.md new file mode 100644 index 0000000..64e57ca --- /dev/null +++ b/docs/adr/032-teensy-port-discovery-is-platform-aware.md @@ -0,0 +1,114 @@ +# ADR 032 — Teensy port discovery is platform-aware and lives in `stage-a-io` + +**Status:** accepted +**Date:** 2026-08-04 +**Feature briefs:** [Stage-A Modulation](../features/stage-a-modulation.md), [Stage-A Photodiode](../features/stage-a-photodiode.md) + +## Context + +Both Stage-A owner plugins find the Teensy by enumerating serial ports and +probing the candidates: the modulation plugin opens each and keeps the one that +answers `HELLO` (the command port), the photodiode plugin listens on each and +keeps the one streaming CRC-clean PDA1 sample frames (the stream port). The +probe is what identifies the device; enumeration only decides what gets probed. + +That candidate filter was written on a Mac and hard-coded the two Unix name +patterns: + +```rust +name.contains("cu.usbmodem") || name.contains("ttyACM") +``` + +Windows names no device. Every serial port is `COMn`, so a correctly attached, +correctly driven Teensy matched neither pattern and was filtered out *before* +any probe could run. Both plugins then reported + +``` +no USB serial device found (looked for usbmodem/ttyACM) +``` + +which names the two things the machine cannot produce, so it reads as "nothing +is attached" when the device is in fact attached and enumerated. The first +Windows bundles from CI (ADR 030) made this reachable for the first time. + +The filter existed in four places — `serial_ports()` and `port_variants()` in +each plugin — and had drifted into two implementations: modulation went through +`stage-a-io`, photodiode called `serialport` directly with `stage-a-io`'s +`hardware` feature switched off. + +## Decision + +**One platform-aware candidate filter, in `stage-a-io::transport`.** + +`available_ports()` returns `PortInfo { name, label, is_usb }` — the OS path, +the USB manufacturer/product label where the OS reports one, and whether the OS +classified the port as USB at all. `candidate_ports()` narrows that list: + +- **macOS** — `cu.usbmodem*`. Every device is listed twice (`tty.*` and `cu.*`) + and only the callout node may be opened, so the name filter is also a dedupe. +- **Linux** — `ttyACM*`, the CDC-ACM class node a Teensy enumerates as. +- **Windows** — every USB-classified port. The name carries no device + information, so USB-ness is the only signal available. If the OS classified + *no* port as USB, the whole list is probed rather than none: a missing + SetupAPI classification must not be able to hide the device the way the name + filter did. + +Both plugins call `candidate_ports()` for probing and `PortInfo::variant()` for +the settings picker, so the probed set and the listed set cannot disagree. +The photodiode crate enables `stage-a-io`'s `hardware` feature to reach it; +`serialport` was already a direct dependency there, so nothing new enters the +build. + +**The filter is a probe-cost optimisation, not the identity check.** It exists +to keep the probes off unrelated ports — notably Windows' phantom Bluetooth +`COM` entries, which can block on open. Being too permissive costs a few +hundred milliseconds of probing; being too strict makes the hardware +unreachable. When in doubt, probe. + +`no_candidate_ports_message()` replaces the fixed string with what the OS +actually enumerated, distinguishing "no serial ports found" from "no serial +port looked like a Teensy (the OS offered COM1 (Bluetooth), …)". + +The platform branch is a `windows: bool` parameter to a private +`narrow_to_candidates`, not a `#[cfg]`, so the unit tests cover both branches +from any build host — including the Windows regression that motivated this ADR. + +**Every port is opened with `dtr_on_open(true)`.** macOS and Linux assert DTR +when a tty is opened; Windows does not — `serialport` sets +`DTR_CONTROL_DISABLE` in the DCB. A Teensyduino sketch that gates its output on +`if (Serial)` (which is `usb_configuration && usb_cdc_line_rtsdtr`) therefore +stays silent on Windows even once the right port is found, and the probes would +report "no port streamed PDA1 sample frames" on a working device. Asserting DTR +on all three platforms makes the port behave the same everywhere; on macOS and +Linux it is a no-op. + +## Consequences + +- The Stage-A plugins connect on Windows: the ports are found, and the opened + port has DTR asserted the way the Unix platforms already did implicitly. +- A port picker entry and a probe candidate come from one function, so a port + that appears in the dropdown is one `auto` would also have found. +- The failure message names the enumerated ports, which is the difference + between "check the cable" and "the filter dropped my device". +- `available_port_names()` and `available_ports_with_labels()` are replaced by + `available_ports()`. Both were internal to this repository. +- Windows probes any non-USB port when the OS classifies nothing at all, which + can add probe latency on a machine with legacy `COM` hardware. Accepted: an + unreachable device is worse than a slow scan. + +## Alternatives considered + +**Match the Teensy by USB VID/PID (0x16C0).** The most precise filter, and it +would work identically on all three platforms. Rejected for now: it hard-codes +the vendor of one board revision into the discovery path, and the probes +already establish identity positively — a VID match that skipped probing would +still have to tell the two ports of the dual-serial device apart. + +**Probe every enumerated port on every platform.** Simplest possible rule, and +correct. Rejected: on macOS it would open the `tty.*` twin of each device, +which blocks waiting for carrier detect, and on Windows it would sit on +phantom Bluetooth ports. + +**Keep the filter in the plugins and add a Windows arm to each.** Rejected: it +was already four copies in two implementations, and the copy that broke was +the one that had drifted. diff --git a/docs/features/README.md b/docs/features/README.md index 845eac3..8fbad49 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -5,10 +5,10 @@ Repository-level feature notes for larger plugin suites, interface migrations, a ## Available Briefs - [Stage-A Bench Stack](./stage-a.md) — Teensy-driven Stage-A bench: two serial ports, two minimal plugins, and the shared `stage-a-io` library. -- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. The coupled `ū`/`a` controls **clamp into the achievable range instead of refusing**, so a leftover depth can no longer make an optical mode unselectable, and both live bounds are shown in the control labels (ADR 025). `V_peak` is the one operator-facing name for the lobe maximum; the half-wave span is derived and never entered. The undocumented TOML `MOD`-step protocol runner was removed — declarative recording protocols belong to A1 (ADR 027). +- [Stage-A Modulation](./stage-a-modulation.md) — orthogonal Manual/Calibrated drive methods and five waveform modes under one hard DAC ceiling, applied immediately on the command port. The coupled `ū`/`a` controls **clamp into the achievable range instead of refusing**, so a leftover depth can no longer make an optical mode unselectable, and both live bounds are shown in the control labels (ADR 025). `V_peak` is the one operator-facing name for the lobe maximum; the half-wave span is derived and never entered. The undocumented TOML `MOD`-step protocol runner was removed — declarative recording protocols belong to A1 (ADR 027). Port discovery is platform-aware and shared with the photodiode plugin, so `auto` finds the Teensy on Windows' nameless `COMn` ports too (ADR 032). - [Stage-A Optical Waveform Drive](./stage-a-optical-waveform.md) — pre-warps the DAC from measured `V_null`/`V_peak` endpoints, with target-specific headroom, Bessel-normalized cycle mean `ū`, and an explicit separation from physical flux `I_k`. - [Stage-A Pockels Transfer Calibration](./stage-a-pockels-calibration.md) — one-button sweep of settled `CONST` DAC codes against the photodiode level, fitting `V_null`/`V_peak` from the light instead of a nominal datasheet, with a transfer-curve view that makes the two parameters legible before anything is measured. Each point is a 20 ms measurement after a 0.1 s settle, and every verdict on the sweep — lobe resolved, cell drifting — is made against the fit's own residual rather than against zero (ADR 019). Applying the fit now actually reaches the panel: the measurement lives on the live worker while the settings snapshot is collected from the UI mirror, so the applied lobe used to be overwritten within one frame (ADR 026). -- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. +- [Stage-A Photodiode](./stage-a-photodiode.md) — live SMA5/A4 readout plus fail-closed excitation log-contrast `a`, computed from complete phase-marker cycles in reject-port geometry. The total-power anchor `I_tot` is **learned from the detector's own stream** — the brightest reading it has taken is where the excitation is extinguished, which the Pockels sweep drives through by construction — so there is nothing to enter and nothing to confirm; the dark level is gone because a DC offset cancels exactly out of the complement (ADR 024). A refusal publishes its reason on the contract, rail detection is span-relative so the bench's millivolt-scale detector is not read as a clipped waveform, and the published level owns a fixed measurement window instead of borrowing the chart's averaging setting. Port discovery is platform-aware and shared with the modulation plugin (ADR 032). - [Stage-A A1 Analysis](./stage-a-a1.md) — synchronized camera RAW + photodiode PDQ coordinator and fail-closed calibrated log-sine amplitude sweep, with transfer/anchor provenance, and live response quicklooks. The output folder is the only required input; ids are provenance and are filled in or recorded as `unspecified` rather than refusing a run. A **Depth `a` source** setting takes `a` from the photodiode (measured, default) or from the modulation owner's calibrated commanded drive (open loop), so a bench with no phase-0 markers can still run the workflow — and every artefact records which source it used (ADR 020). With a commanded depth there is nothing to search for, so `Find a₀` and the lock table drop out and the ladder confirms each frequency against the modulation owner instead of the camera trigger (ADR 021). Every run also records the sensor's own die temperature, pixel dead time and scene illumination (ADR 022). The frequency ladder is an outer loop: one button repeats the whole depth sweep at every planned frequency, producing the `q_p(a, f)` surface on a single lease (ADR 023). Recording, both sweeps and the a₀ workflow are one **Record** section — Record once / Sweep a / Sweep f / Sweep a × f / Stop — with Live analysis at the top of the panel rather than below the controls that read it. A **protocol** runs a whole survey from a file naming every axis for every recording — a CSV with one row per recording (per-row duration and a `normal`/`pilot`/`background` role, so a file carries its own references), or TOML blocks/ranges for a dense regular sweep — including the `I_k` axis that no button could sweep (ADR 027), and the host's sensor telemetry is compacted column-wise into the measurement folder under the run's own name (ADR 028). Every leased run heartbeats its modulation and photodiode leases against the deadline the owner actually granted, so a recording longer than the owner's TTL cap no longer loses the drive — and with it the phase-0 trigger and the photodiode's optical summary — in the middle of a point (ADR 029). - [Stage-A A1 Automation](./stage-a-a1-automation.md) — roadmap to semi-automate the amplitude sweep; the single-row sweep core is **built** (ADR 010), scout/multi-row/`a50` fit remain planned. - [Stage-A A1 Exact Event Count](./stage-a-a1-event-count.md) — per-frequency `a₀` lock: closed-loop trim of the commanded depth until the photodiode *measures* the one frozen log contrast `a₀` over whole modulation cycles, a per-frequency lock table on disk, a one-button atomic frequency point recorded at exactly `a₀` under the modulation lease, and an unattended log-spaced frequency ladder that locks and records every planned `f` on a single lease. The search exists for the *measured* depth only — with a commanded depth the ladder skips it entirely and reduces to "set `a₀`, press Record all frequencies" (ADR 021). diff --git a/docs/features/stage-a-modulation.md b/docs/features/stage-a-modulation.md index d605647..a6c8daf 100644 --- a/docs/features/stage-a-modulation.md +++ b/docs/features/stage-a-modulation.md @@ -60,6 +60,17 @@ Every accepted setting change is transferred to the Teensy **immediately** as on no Apply button, no experiment state machine. The panel shows the modulation and live DAC code the board *reports* (`MOD` reply + 2 Hz `STATUS` poll), not merely the commanded values. +## Port discovery + +`auto` opens every candidate port and keeps the one that answers `HELLO` — the probe, not the port +name, tells the command port from the photodiode stream port of the same dual-serial device. Which +ports are candidates is platform-specific and shared with the photodiode plugin through +`stage-a-io::transport::candidate_ports()`: `cu.usbmodem*` on macOS (the callout node only, since +every device is listed twice), `ttyACM*` on Linux, and every USB-classified `COMn` on Windows, +where the name carries no device information at all (ADR 032). The settings picker lists exactly +the same set with each port's USB label. When nothing qualifies, the error names the ports the OS +did enumerate. + ## Contract - Owns the Teensy **command port** exclusively (one owner per port, ADR 006). The photodiode diff --git a/docs/features/stage-a-photodiode.md b/docs/features/stage-a-photodiode.md index 14eca52..70d51a5 100644 --- a/docs/features/stage-a-photodiode.md +++ b/docs/features/stage-a-photodiode.md @@ -137,11 +137,22 @@ but a chart setting. always-false state to the worker, so it could never stay recording. The `record` boolean setting remains as a non-schema compatibility alias. +## Port discovery + +`auto` listens briefly on every candidate port and keeps the one streaming CRC-clean PDA1 sample +frames — the probe, not the port name, is what identifies the stream port. Which ports are +candidates is platform-specific and shared with the modulation plugin through +`stage-a-io::transport::candidate_ports()`: `cu.usbmodem*` on macOS (the callout node only, since +every device is listed twice), `ttyACM*` on Linux, and every USB-classified `COMn` on Windows, +where the name carries no device information at all (ADR 032). The settings picker lists exactly +the same set, so a port offered in the dropdown is one `auto` would also have probed. When nothing +qualifies, the error names the ports the OS did enumerate. + ## Contract - Owns the Teensy **stream port** exclusively (ADR 006); the port carries no commands, so the - plugin is read-only by construction. It reuses `stage-a-io` (`default-features = false`) only - for the PDA1 wire parser — no client, worker, or transport. + plugin is read-only by construction. It uses `stage-a-io` for the PDA1 wire parser and for port + discovery — no client, worker, or transport. - **Frame-independent**: connecting is a checkbox setting; the reader thread and all views work with no camera attached (the host only calls `process_frame()` while frames flow). - Garbage on the port resynchronises at the next CRC-clean frame; skipped bytes and CRC failures diff --git a/plugins/stage-a-modulation/README.md b/plugins/stage-a-modulation/README.md index a3ae241..6bac4a9 100644 --- a/plugins/stage-a-modulation/README.md +++ b/plugins/stage-a-modulation/README.md @@ -102,10 +102,14 @@ inversion they used. Full detail: [feature brief](../../docs/features/stage-a-po ## Ports -**Use `auto` (default recommendation):** it probes every attached usbmodem/ttyACM device and -connects to the one that answers `HELLO` — that is always the Teensy command port, never the -photodiode stream port. Explicit ports remain selectable; `mock` runs an in-process simulated -controller for hardware-free testing. +**Use `auto` (default recommendation):** it probes every attached USB serial port and connects to +the one that answers `HELLO` — that is always the Teensy command port, never the photodiode +stream port. Explicit ports remain selectable; `mock` runs an in-process simulated controller for +hardware-free testing. + +Which ports get probed is platform-specific: `cu.usbmodem*` on macOS, `ttyACM*` on Linux, and +every USB-classified `COMn` on Windows (ADR 032). The picker lists the same set with each port's +USB label, so the Teensy is recognisable by name. Replaying a recording disconnects the plugin defensively; live control itself needs no capture session. diff --git a/plugins/stage-a-modulation/src/lib.rs b/plugins/stage-a-modulation/src/lib.rs index 8dbf1b7..634a762 100644 --- a/plugins/stage-a-modulation/src/lib.rs +++ b/plugins/stage-a-modulation/src/lib.rs @@ -2680,7 +2680,7 @@ fn open_serial(port_hint: &str) -> Result Result Vec { - stage_a_io::transport::available_port_names() + stage_a_io::transport::candidate_ports() .into_iter() - // macOS lists each device twice; use the callout (cu.*) node only. - .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) + .map(|port| port.name) .collect() } @@ -2829,15 +2828,11 @@ fn serial_ports() -> Vec { /// only the leading path is the value. fn port_variants() -> Vec { let mut variants = vec!["auto".to_owned(), "mock".to_owned()]; - for (name, label) in stage_a_io::transport::available_ports_with_labels() { - if !(name.contains("cu.usbmodem") || name.contains("ttyACM")) { - continue; - } - variants.push(match label { - Some(label) => format!("{name} ({label})"), - None => name, - }); - } + variants.extend( + stage_a_io::transport::candidate_ports() + .iter() + .map(stage_a_io::transport::PortInfo::variant), + ); variants } @@ -3090,7 +3085,7 @@ impl Plugin for StageAModulationPlugin { key: "port".into(), label: "Port".into(), tooltip: Some( - "auto (recommended) probes the attached usbmodem ports and picks \ + "auto (recommended) probes the attached USB serial ports and picks \ the one that answers HELLO — the Teensy command port; \ mock = in-process simulated controller" .into(), diff --git a/plugins/stage-a-photodiode/Cargo.toml b/plugins/stage-a-photodiode/Cargo.toml index 3e0c8c0..5042600 100644 --- a/plugins/stage-a-photodiode/Cargo.toml +++ b/plugins/stage-a-photodiode/Cargo.toml @@ -13,5 +13,7 @@ crate-type = ["cdylib", "rlib"] augur-plugin-api.workspace = true serde_json.workspace = true serialport.workspace = true -stage-a-io = { path = "../../stage-a-io", default-features = false } +# `hardware` brings in the shared platform-aware port discovery; serialport is +# already a direct dependency here, so it adds nothing new to the build. +stage-a-io = { path = "../../stage-a-io" } stage-a-plugin-contract = { path = "../../stage-a-plugin-contract" } diff --git a/plugins/stage-a-photodiode/README.md b/plugins/stage-a-photodiode/README.md index 2406154..da03995 100644 --- a/plugins/stage-a-photodiode/README.md +++ b/plugins/stage-a-photodiode/README.md @@ -24,9 +24,12 @@ read-only by construction; the command port belongs to `stage-a-modulation`. ## Ports -**Use `auto` (default recommendation):** it listens briefly on every attached usbmodem/ttyACM -device and connects to the one actually streaming CRC-clean PDA1 sample frames — that is always -the Teensy stream port. `mock` generates a synthetic sine for hardware-free testing. +**Use `auto` (default recommendation):** it listens briefly on every attached USB serial port and +connects to the one actually streaming CRC-clean PDA1 sample frames — that is always the Teensy +stream port. `mock` generates a synthetic sine for hardware-free testing. + +Which ports get listened to is platform-specific: `cu.usbmodem*` on macOS, `ttyACM*` on Linux, +and every USB-classified `COMn` on Windows (ADR 032). ## Owner control service diff --git a/plugins/stage-a-photodiode/src/lib.rs b/plugins/stage-a-photodiode/src/lib.rs index 4d4cf3e..4f7ca91 100644 --- a/plugins/stage-a-photodiode/src/lib.rs +++ b/plugins/stage-a-photodiode/src/lib.rs @@ -556,6 +556,8 @@ impl Reader { ) -> Result { let port = serialport::new(&path, 115_200) .timeout(Duration::from_millis(50)) + // Windows opens with DTR deasserted; see ADR 032. + .dtr_on_open(true) .open() .map_err(|err| format!("open {path}: {err}"))?; let stop = Arc::new(AtomicBool::new(false)); @@ -2527,16 +2529,10 @@ fn rejected_service_reply( } fn serial_ports() -> Vec { - serialport::available_ports() - .map(|ports| { - ports - .into_iter() - .map(|p| p.port_name) - // macOS lists each device twice; use the callout (cu.*) node only. - .filter(|name| name.contains("cu.usbmodem") || name.contains("ttyACM")) - .collect() - }) - .unwrap_or_default() + stage_a_io::transport::candidate_ports() + .into_iter() + .map(|port| port.name) + .collect() } /// Finds the Teensy stream port: the dual-serial firmware free-runs PDA1 @@ -2545,7 +2541,7 @@ fn serial_ports() -> Vec { fn resolve_auto_port() -> Result { let candidates = serial_ports(); if candidates.is_empty() { - return Err("no USB serial device found (looked for usbmodem/ttyACM)".to_owned()); + return Err(stage_a_io::transport::no_candidate_ports_message()); } let mut saw_legacy_ascii = false; for path in &candidates { @@ -2589,6 +2585,8 @@ enum ProbeResult { fn probe_pd_stream(path: &str) -> ProbeResult { let Ok(mut port) = serialport::new(path, 115_200) .timeout(Duration::from_millis(100)) + // Windows opens with DTR deasserted; see ADR 032. + .dtr_on_open(true) .open() else { return ProbeResult::Nothing; @@ -2632,26 +2630,11 @@ fn probe_pd_stream(path: &str) -> ProbeResult { /// host exchanges enum settings as indices into this list. fn port_variants() -> Vec { let mut variants = vec!["mock".to_owned(), "auto".to_owned()]; - for port in serialport::available_ports().unwrap_or_default() { - if !(port.port_name.contains("cu.usbmodem") || port.port_name.contains("ttyACM")) { - continue; - } - let label = match port.port_type { - serialport::SerialPortType::UsbPort(info) => match (info.manufacturer, info.product) { - (Some(manufacturer), Some(product)) if !product.starts_with(&manufacturer) => { - Some(format!("{manufacturer} {product}")) - } - (_, Some(product)) => Some(product), - (Some(manufacturer), None) => Some(manufacturer), - (None, None) => None, - }, - _ => None, - }; - variants.push(match label { - Some(label) => format!("{} ({label})", port.port_name), - None => port.port_name, - }); - } + variants.extend( + stage_a_io::transport::candidate_ports() + .iter() + .map(stage_a_io::transport::PortInfo::variant), + ); variants } diff --git a/stage-a-io/src/transport.rs b/stage-a-io/src/transport.rs index e9c9377..5e1b820 100644 --- a/stage-a-io/src/transport.rs +++ b/stage-a-io/src/transport.rs @@ -28,6 +28,10 @@ impl SerialTransport { pub fn open(path: &str, baud: u32, poll_timeout: Duration) -> io::Result { let port = serialport::new(path, baud) .timeout(poll_timeout) + // Windows opens a port with DTR deasserted; macOS and Linux assert + // it for us. A Teensy sketch that gates on `if (Serial)` would stay + // silent there, so assert it everywhere (ADR 032). + .dtr_on_open(true) .open() .map_err(|err| io::Error::other(format!("opening {path} failed: {err}")))?; Ok(Self { port }) @@ -113,33 +117,42 @@ impl Transport for MockTransport { } } -/// Names of serial ports visible to the OS (empty without the `hardware` -/// feature). Used by plugins to offer a port picker. -#[cfg(feature = "hardware")] -pub fn available_port_names() -> Vec { - serialport::available_ports() - .map(|ports| ports.into_iter().map(|p| p.port_name).collect()) - .unwrap_or_default() +/// One serial port as the OS enumerated it. +/// +/// `label` is the human-readable USB manufacturer/product where the OS +/// reports one — e.g. `"Teensyduino Dual Serial"` — so port pickers can show +/// which entry is the Teensy. `is_usb` records whether the OS classified the +/// port as a USB device at all, which is the only device hint Windows gives. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PortInfo { + pub name: String, + pub label: Option, + pub is_usb: bool, } -#[cfg(not(feature = "hardware"))] -pub fn available_port_names() -> Vec { - Vec::new() +impl PortInfo { + /// The port as a picker entry: the path, plus the USB label in parentheses + /// where there is one. Only the leading path is the value — see + /// `variant_path` in the Stage-A plugins. + pub fn variant(&self) -> String { + match &self.label { + Some(label) => format!("{} ({label})", self.name), + None => self.name.clone(), + } + } } -/// Port names plus a human-readable USB label (manufacturer/product) where -/// the OS provides one — e.g. `("/dev/cu.usbmodem…", Some("Teensyduino Dual -/// Serial"))`. Lets port pickers show which entry is the Teensy. +/// Every serial port visible to the OS (empty without the `hardware` feature). #[cfg(feature = "hardware")] -pub fn available_ports_with_labels() -> Vec<(String, Option)> { +pub fn available_ports() -> Vec { serialport::available_ports() .map(|ports| { ports .into_iter() .map(|p| { - let label = match p.port_type { + let (label, is_usb) = match p.port_type { serialport::SerialPortType::UsbPort(info) => { - match (info.manufacturer, info.product) { + let label = match (info.manufacturer, info.product) { (Some(manufacturer), Some(product)) if !product.starts_with(&manufacturer) => { @@ -148,11 +161,16 @@ pub fn available_ports_with_labels() -> Vec<(String, Option)> { (_, Some(product)) => Some(product), (Some(manufacturer), None) => Some(manufacturer), (None, None) => None, - } + }; + (label, true) } - _ => None, + _ => (None, false), }; - (p.port_name, label) + PortInfo { + name: p.port_name, + label, + is_usb, + } }) .collect() }) @@ -160,6 +178,117 @@ pub fn available_ports_with_labels() -> Vec<(String, Option)> { } #[cfg(not(feature = "hardware"))] -pub fn available_ports_with_labels() -> Vec<(String, Option)> { +pub fn available_ports() -> Vec { Vec::new() } + +/// The enumerated ports worth probing for a Teensy. +/// +/// Unix names the device, so the name is the filter: macOS lists every device +/// twice (`tty.*` and `cu.*`) and only the callout node may be opened, and a +/// Teensy's CDC-ACM class node on Linux is `ttyACM*`. Windows names nothing — +/// every port is `COMn` — so USB-ness is the only signal there, and if the OS +/// classified no port at all the whole list is probed rather than none. What +/// actually identifies the Teensy is the probe (HELLO on the command port, +/// PDA1 sample frames on the stream port); this only keeps the probe from +/// stalling on unrelated ports such as Windows' phantom Bluetooth COM entries. +pub fn candidate_ports() -> Vec { + narrow_to_candidates(available_ports(), cfg!(windows)) +} + +fn narrow_to_candidates(ports: Vec, windows: bool) -> Vec { + if !windows { + return ports + .into_iter() + .filter(|port| port.name.contains("cu.usbmodem") || port.name.contains("ttyACM")) + .collect(); + } + if ports.iter().any(|port| port.is_usb) { + return ports.into_iter().filter(|port| port.is_usb).collect(); + } + ports +} + +/// Why there was nothing to probe, naming what the OS did enumerate — the +/// difference between "no device is attached" and "a device is attached but +/// this platform's filter dropped it" is the operator's next step. +pub fn no_candidate_ports_message() -> String { + let ports = available_ports(); + if ports.is_empty() { + return "no serial ports found — check the USB cable and that the Teensy is powered" + .to_owned(); + } + let seen = ports + .iter() + .map(PortInfo::variant) + .collect::>() + .join(", "); + format!("no serial port looked like a Teensy (the OS offered {seen})") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn port(name: &str, is_usb: bool) -> PortInfo { + PortInfo { + name: name.to_owned(), + label: None, + is_usb, + } + } + + fn names(ports: Vec) -> Vec { + ports.into_iter().map(|port| port.name).collect() + } + + #[test] + fn unix_keeps_the_callout_node_and_drops_its_tty_twin() { + let ports = vec![ + port("/dev/tty.usbmodem12345", true), + port("/dev/cu.usbmodem12345", true), + port("/dev/cu.Bluetooth-Incoming-Port", false), + ]; + assert_eq!( + names(narrow_to_candidates(ports, false)), + vec!["/dev/cu.usbmodem12345"] + ); + } + + #[test] + fn unix_keeps_the_linux_cdc_acm_node() { + let ports = vec![port("/dev/ttyACM0", true), port("/dev/ttyS0", false)]; + assert_eq!(names(narrow_to_candidates(ports, false)), vec!["/dev/ttyACM0"]); + } + + #[test] + fn windows_com_ports_survive_the_unix_name_filter() { + // The bug: COMn matches neither `cu.usbmodem` nor `ttyACM`, so the + // Teensy's two ports were filtered out before any probe could run. + let ports = vec![port("COM3", true), port("COM4", true)]; + assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM3", "COM4"]); + } + + #[test] + fn windows_drops_non_usb_ports_when_a_usb_port_exists() { + let ports = vec![port("COM1", false), port("COM7", true)]; + assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM7"]); + } + + #[test] + fn windows_probes_everything_when_the_os_classifies_nothing() { + let ports = vec![port("COM1", false), port("COM3", false)]; + assert_eq!(names(narrow_to_candidates(ports, true)), vec!["COM1", "COM3"]); + } + + #[test] + fn a_labelled_port_shows_its_usb_name_in_the_picker() { + let labelled = PortInfo { + name: "COM3".to_owned(), + label: Some("Teensyduino Dual Serial".to_owned()), + is_usb: true, + }; + assert_eq!(labelled.variant(), "COM3 (Teensyduino Dual Serial)"); + assert_eq!(port("COM4", true).variant(), "COM4"); + } +} From 0e07d88a6dd0284d1b488adf71be5a6c79ad462a Mon Sep 17 00:00:00 2001 From: Mika Uthmann Date: Wed, 5 Aug 2026 10:14:07 +0200 Subject: [PATCH 6/6] =?UTF-8?q?fix(stage-a-a1):=20=F0=9F=90=9B=20accept=20?= =?UTF-8?q?protocol=20files=20a=20spreadsheet=20saved?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Excel's "CSV UTF-8" — the obvious save format on a Windows bench — writes a UTF-8 byte-order mark. Unstripped it becomes part of the first header cell, so `mean_u` stops matching `mean_u` and the protocol is refused for missing a required column that is plainly there. The TOML form fails its parse outright. Neither message points at an invisible character. Strip the BOM once for both readers. CRLF was already handled by `str::lines()`; it now has a test so it stays that way. Refs ADR 027 --- docs/features/stage-a-a1.md | 6 ++++- plugins/stage-a-a1/README.md | 3 +++ plugins/stage-a-a1/src/protocol.rs | 43 +++++++++++++++++++++++++++--- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/docs/features/stage-a-a1.md b/docs/features/stage-a-a1.md index 8cf4b67..fe7d18c 100644 --- a/docs/features/stage-a-a1.md +++ b/docs/features/stage-a-a1.md @@ -207,7 +207,11 @@ swept at all, and what a block recorded lived in the panel rather than in anything that travels with the results. A **protocol** is a file naming every axis for every recording. The reader is -chosen by extension, and both produce the same flat list of points. +chosen by extension, and both produce the same flat list of points. Both +tolerate what a spreadsheet writes: CRLF line endings, and the UTF-8 +byte-order mark Excel's "CSV UTF-8" prepends — unstripped, the BOM becomes +part of the first header cell and the file is refused for missing a column it +visibly has. **CSV — one row per recording**, and the one to reach for: it opens in a spreadsheet, comes straight out of a script, and each row carries its own diff --git a/plugins/stage-a-a1/README.md b/plugins/stage-a-a1/README.md index 193dbd9..048df5e 100644 --- a/plugins/stage-a-a1/README.md +++ b/plugins/stage-a-a1/README.md @@ -171,6 +171,9 @@ Columns are found **by name**, so their order does not matter and one can be lef Blank lines and `#` comments are skipped, and a blank cell falls back to the default. Errors carry the **file line number**, which is what your editor and spreadsheet both show. +Files saved by a spreadsheet load as-is: Windows line endings and the byte-order mark that Excel's +"CSV UTF-8" writes are both absorbed, so the first column is not silently reported missing. + Two things the row form gives you that blocks cannot without one block per value: **a different duration per row** (1 Hz needs 40 s of cycles, 200 Hz does not), and **a `role` column**, so a file can open with its own background floor and pilot and then record the points scored against them — diff --git a/plugins/stage-a-a1/src/protocol.rs b/plugins/stage-a-a1/src/protocol.rs index 9434726..2851eba 100644 --- a/plugins/stage-a-a1/src/protocol.rs +++ b/plugins/stage-a-a1/src/protocol.rs @@ -298,10 +298,21 @@ impl Axis { } } +/// Drops a leading UTF-8 byte-order mark. +/// +/// Saving a protocol as "CSV UTF-8" in Excel — the obvious choice on a Windows +/// bench — writes a BOM. Left in place it becomes part of the first header +/// cell, so `mean_u` stops matching `mean_u` and the file is refused for +/// missing a column that is plainly there; in the TOML form it fails the parse +/// outright. Neither message would point at an invisible character. +fn strip_bom(text: &str) -> &str { + text.strip_prefix('\u{feff}').unwrap_or(text) +} + /// Parses a protocol and expands it into the points to record. pub fn parse(text: &str) -> Result { - let doc: ProtocolDoc = - toml::from_str(text).map_err(|error| ProtocolError::Toml(error.to_string()))?; + let doc: ProtocolDoc = toml::from_str(strip_bom(text)) + .map_err(|error| ProtocolError::Toml(error.to_string()))?; let default_duration = doc.defaults.duration_s.unwrap_or(10); let default_settle = doc.defaults.settle_s.unwrap_or(2.0); @@ -444,7 +455,8 @@ pub fn parse_csv(text: &str) -> Result { let mut header: Option> = None; let mut points = Vec::new(); - for (offset, raw) in text.lines().enumerate() { + // `lines()` already absorbs CRLF; the BOM is the part it leaves behind. + for (offset, raw) in strip_bom(text).lines().enumerate() { let line_no = offset + 1; let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -599,6 +611,31 @@ depth_a = 0.8 duration_s = 30 "#; + #[test] + fn a_spreadsheet_bom_does_not_hide_the_first_column() { + // Excel's "CSV UTF-8" writes a BOM. Without stripping it, `mean_u` + // reads as `\u{feff}mean_u` and the file is refused for missing the + // column it visibly has. + let csv = "\u{feff}mean_u,frequency_hz,depth_a\n0.5,10,1.0\n"; + let protocol = parse_file("survey.csv", csv).expect("BOM-prefixed CSV"); + assert_eq!(protocol.points.len(), 1); + assert_eq!(protocol.points[0].mean_u, 0.5); + } + + #[test] + fn a_bom_does_not_break_the_toml_form_either() { + let protocol = parse(&format!("\u{feff}{SAMPLE}")).expect("BOM-prefixed TOML"); + assert_eq!(protocol.name, "sample"); + } + + #[test] + fn a_spreadsheet_crlf_file_parses() { + let csv = "mean_u,frequency_hz,depth_a\r\n0.5,10,1.0\r\n"; + let protocol = parse_file("survey.csv", csv).expect("CRLF CSV"); + assert_eq!(protocol.points.len(), 1); + assert_eq!(protocol.points[0].frequency_hz, 10.0); + } + #[test] fn a_protocol_expands_to_the_product_of_its_axes() { let protocol = parse(SAMPLE).expect("valid protocol");