diff --git a/.cursor/debug-previous.log b/.cursor/debug-previous.log new file mode 100644 index 0000000000..f47f07577a --- /dev/null +++ b/.cursor/debug-previous.log @@ -0,0 +1,13 @@ +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99968,"max_alloc":81908,"internal_free":99968,"spiram_free":2079119},"timestamp":60038} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":186636,"max_alloc":81908,"internal_free":186636,"spiram_free":2079887},"timestamp":120040} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":94328,"max_alloc":81908,"internal_free":94328,"spiram_free":2079119},"timestamp":180078} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99564,"max_alloc":81908,"internal_free":99564,"spiram_free":2079119},"timestamp":240093} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99564,"max_alloc":81908,"internal_free":99564,"spiram_free":2079119},"timestamp":300130} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203620,"max_alloc":81908,"internal_free":203620,"spiram_free":2079927},"timestamp":360131} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203620,"max_alloc":81908,"internal_free":203620,"spiram_free":2079927},"timestamp":420131} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99732,"max_alloc":81908,"internal_free":99732,"spiram_free":2079119},"timestamp":480150} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":164132,"max_alloc":69620,"internal_free":164132,"spiram_free":2079119},"timestamp":540153} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":90808,"max_alloc":79860,"internal_free":90808,"spiram_free":2079119},"timestamp":600166} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99676,"max_alloc":81908,"internal_free":99676,"spiram_free":2079119},"timestamp":660181} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99708,"max_alloc":81908,"internal_free":99708,"spiram_free":2079119},"timestamp":720195} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99720,"max_alloc":81908,"internal_free":99720,"spiram_free":2079119},"timestamp":780217} diff --git a/.cursor/debug.log b/.cursor/debug.log new file mode 100644 index 0000000000..4ecf534182 --- /dev/null +++ b/.cursor/debug.log @@ -0,0 +1,40 @@ +{"sessionId":"debug-session","location":"MQTTBridge.cpp:2142","message":"after_new_analyzer_us_client","hypothesisId":"H4","data":{"free":219116,"max_alloc":208884,"internal_free":219116,"spiram_free":2080343},"timestamp":4420} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:2182","message":"after_new_analyzer_eu_client","hypothesisId":"H4","data":{"free":218704,"max_alloc":208884,"internal_free":218704,"spiram_free":2080343},"timestamp":4422} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":96768,"max_alloc":81908,"internal_free":96768,"spiram_free":2079119},"timestamp":60033} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163244,"max_alloc":71668,"internal_free":163244,"spiram_free":2079119},"timestamp":120034} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":166716,"max_alloc":71668,"internal_free":166716,"spiram_free":2079119},"timestamp":180036} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163240,"max_alloc":71668,"internal_free":163240,"spiram_free":2079119},"timestamp":240035} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163372,"max_alloc":71668,"internal_free":163372,"spiram_free":2079119},"timestamp":300035} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163728,"max_alloc":71668,"internal_free":163728,"spiram_free":2079119},"timestamp":360037} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":169512,"max_alloc":71668,"internal_free":169512,"spiram_free":2079119},"timestamp":420038} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":169512,"max_alloc":71668,"internal_free":169512,"spiram_free":2079119},"timestamp":480040}MQTT: Stored raw radio data: 74 bytes, SNR=11.2, RSSI=-59.0 +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163500,"max_alloc":71668,"internal_free":163500,"spiram_free":2079119},"timestamp":540040} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163320,"max_alloc":71668,"internal_free":163320,"spiram_free":2079119},"timestamp":600041} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163352,"max_alloc":71668,"internal_free":163352,"spiram_free":2079119},"timestamp":660043} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163340,"max_alloc":71668,"internal_free":163340,"spiram_free":2079119},"timestamp":720044} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163912,"max_alloc":71668,"internal_free":163912,"spiram_free":2079119},"timestamp":780045} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":169512,"max_alloc":71668,"internal_free":169512,"spiram_free":2079119},"timestamp":840044} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:716","message":"critical_memory_check","hypothesisId":"H1_H4","data":{"free":169512,"max_alloc":71668,"internal_free":169512,"spiram_free":2079119},"timestamp":900003} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":169512,"max_alloc":71668,"internal_free":169512,"spiram_free":2079119},"timestamp":900045} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":169512,"max_alloc":71668,"internal_free":169512,"spiram_free":2079119},"timestamp":960046} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":166480,"max_alloc":71668,"internal_free":166480,"spiram_free":2079119},"timestamp":1020050} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":163324,"max_alloc":71668,"internal_free":163324,"spiram_free":2079119},"timestamp":1080052} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203612,"max_alloc":83956,"internal_free":203612,"spiram_free":2079927},"timestamp":1140052} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203612,"max_alloc":83956,"internal_free":203612,"spiram_free":2079927},"timestamp":1200056} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1260059} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1320062} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1380064} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1440067} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1500069} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203700,"max_alloc":83956,"internal_free":203700,"spiram_free":2079927},"timestamp":1560072} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203652,"max_alloc":83956,"internal_free":203652,"spiram_free":2079927},"timestamp":1620076} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203748,"max_alloc":83956,"internal_free":203748,"spiram_free":2079887},"timestamp":1680080} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1740082} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:716","message":"critical_memory_check","hypothesisId":"H1_H4","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1800004} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1800086} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1860087} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1920090} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203648,"max_alloc":83956,"internal_free":203648,"spiram_free":2079927},"timestamp":1980092} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":203700,"max_alloc":83956,"internal_free":203700,"spiram_free":2079927},"timestamp":2040094} +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":92832,"max_alloc":83956,"internal_free":92612,"spiram_free":2074463},"timestamp":2100108}02:24:28 - 5/2/2026 U RAW: 09054F86AB7EE0223B896D851B3704309F +{"sessionId":"debug-session","location":"MQTTBridge.cpp:505","message":"mqtt_loop_60s","hypothesisId":"H5","data":{"free":99732,"max_alloc":81908,"internal_free":99732,"spiram_free":2079119},"timestamp":2160133} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 3a1c2f0548..b45ab649f0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -12,15 +12,14 @@ "runArgs": [ "--privileged", "--network=host", - "--device=/dev/bus/usb", - // arch linux tty* is owned by uucp (986) - "--group-add=986", + "--volume=/dev/bus/usb:/dev/bus/usb:ro", + // arch tty* is owned by uucp (986) // debian tty* is owned by dialout (20) - "--group-add=20" + "--group-add=20", + "--group-add=986" ], "postCreateCommand": { - "platformio": "pipx install platformio", - "opencode": "curl -fsSL https://opencode.ai/install | bash" + "platformio": "pipx install platformio" }, "customizations": { "vscode": { diff --git a/.github/workflows/build-observer-firmwares-beta.yml b/.github/workflows/build-observer-firmwares-beta.yml new file mode 100644 index 0000000000..8618da4b99 --- /dev/null +++ b/.github/workflows/build-observer-firmwares-beta.yml @@ -0,0 +1,377 @@ +name: Build MQTT Observer Firmwares (BETA channel) + +permissions: + contents: write + +# Push-triggered on the dev line, mirroring build-observer-firmwares.yml. +# +# This is NOT the original design — dispatch-only was, so that publishing to real +# nodes stayed an explicit act. That does not work in this repo: this fork's +# default branch is `dev` (an upstream mirror that carries none of the observer +# workflows), and GitHub only surfaces `workflow_dispatch` for workflows present +# on the DEFAULT branch. A dispatch-only workflow here would never appear in the +# Actions UI. Adding fork-specific workflows to `dev` would pollute the upstream +# mirror and conflict on every upstream sync, so the push trigger is the correct +# mechanism — the same one production already relies on. +# +# workflow_dispatch is kept as well: harmless now, and it starts working if the +# default branch ever changes. +# +# Consequence to be aware of: every push to `observer-firmware-dev` publishes a +# dev-channel build. That is defensible for a channel users opt into, but if you +# want staging commits without publishing, work on a side branch and fast-forward +# `observer-firmware-dev` when you intend to release. +on: + workflow_dispatch: + push: + branches: + - observer-firmware-dev + # Same rationale as production: docs/CI-only changes do not alter binaries. + paths-ignore: + - '**.md' + - 'docs/**' + - 'scripts/gen_changelog.py' + - '.github/**' + - '.gitignore' + - '.gitattributes' + - '.editorconfig' + - 'LICENSE' + - '.vscode/**' + - '.claude/**' + # Throwaway build worktrees never affect the binaries. + - '.build-wt-*' + - '.build-wt-*/**' + +# Own group: serialize beta builds against each other only. Concurrency groups are +# repo-wide, not per-branch, so sharing one with the production workflows let an +# in-flight beta build force a production build into the pending state, where the +# next run queued into the group cancelled it outright. Concurrent flasher pushes +# are instead made safe by the rebase-retry in "Commit & Push Beta Artifacts". +concurrency: + group: observer-firmware-beta-build + cancel-in-progress: false + +env: + # MUST stay equal to the production channel's FIRMWARE_VERSION. The observer's + # OTA comparison treats a different base version as "always an update", so a + # distinct base here would make every beta node think it is permanently behind. + # Channels are separated by manifest URL, not by base version. + FIRMWARE_VERSION: v1.17.1 + + # Beta-only rolling release. A separate tag is required, not cosmetic: the + # publish step prunes all but the KEEP_BUILDS most recent build hashes within + # its tag, so sharing production's tag would make each channel delete the + # other's assets. + RELEASE_TAG: observer-mqtt-beta-latest + + # The channel itself. Firmware fetches /.json, + # so this URL is what keeps beta nodes on beta. + OTA_MANIFEST_BASE_URL: https://observer.gessaman.com/beta/v + # Marks the embedded version, e.g. v1.16.0.3-observer-beta-dev-abc1234, so `ver` + # (and the MQTT firmware_version / SNMP) identify BOTH the channel and its + # provenance: this channel is built from the upstream-dev-merged line, so "dev" + # is carried in the string rather than left to be inferred from the branch name. + # Does not affect OTA version parsing: ota_parseVersion() reads to the first '-' + # and ota_extractHash() takes the token after the last, so tags in between are + # transparent. + OTA_CHANNEL_TAG: beta-dev + # Marks the asset *filenames*, e.g. -v1.16.0-dev-abc1234.bin, so a + # downloaded file identifies its channel at a glance. Lowercase letters only — + # the filename parsers (gen-slim ASSET_RE, the /releases Worker, flasher.js) + # accept exactly (?:-[a-z]+)? between version and hash. + FILENAME_CHANNEL_TAG: "-dev" + + # Beta's own build counter, so the two channels' build numbers never interleave. + COUNTER_URL: https://observer.gessaman.com/observer-beta-build-counter.json + COUNTER_FILE: observer-beta-build-counter.json + + # Where beta artifacts live in the flasher repo. MANIFEST_DIR must correspond to + # OTA_MANIFEST_BASE_URL's path, and STATIC_PATH must be a host/route serving the + # beta GitHub release (see cloudflare-worker). + MANIFEST_DIR: beta/v + STATIC_PATH: https://observer-fw-beta.gessaman.com + +jobs: + + enumerate: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.split.outputs.matrix }} + build_number: ${{ steps.buildnum.outputs.n }} + steps: + - name: Clone Repo + uses: actions/checkout@v4 + + - name: Split observer envs into shards + id: split + shell: bash + run: | + SHARDS=14 + ENVS=$(grep -rhoE '^\[env:[^]]*observer_mqtt\]' platformio.ini variants/*/platformio.ini \ + | sed -E 's/^\[env:(.*)\]$/\1/' | sort -u) + echo "Discovered envs:"; echo "$ENVS" + MATRIX=$(echo "$ENVS" | awk -v n="$SHARDS" ' + { shard[NR % n] = shard[NR % n] " " $0 } + END { for (i = 0; i < n; i++) { sub(/^ /, "", shard[i]); + printf "{\"idx\":%d,\"envs\":\"%s\"}\n", i, shard[i] } }' \ + | jq -cs .) + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + - name: Compute beta build number + id: buildnum + shell: bash + run: | + # Same scheme as production but off the BETA counter, so the channels + # increment independently. + CUR=$(curl -fsSL "$COUNTER_URL" 2>/dev/null || echo '{}') + PREV_BASE=$(echo "$CUR" | jq -r '.baseVersion // ""') + PREV_BUILD=$(echo "$CUR" | jq -r '.build // 0') + if [ "$PREV_BASE" = "$FIRMWARE_VERSION" ]; then + N=$((PREV_BUILD + 1)) + else + N=1 + fi + echo "Base $FIRMWARE_VERSION; previous beta build $PREV_BUILD (base $PREV_BASE) -> N=$N" + echo "n=$N" >> "$GITHUB_OUTPUT" + + build: + needs: enumerate + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.enumerate.outputs.matrix) }} + steps: + - name: Clone Repo + uses: actions/checkout@v4 + + - name: Cache PlatformIO Toolchains + uses: actions/cache@v4 + with: + path: | + ~/.platformio/packages + ~/.platformio/platforms + key: pio-toolchains-${{ runner.os }}-${{ hashFiles('platformio.ini') }} + restore-keys: | + pio-toolchains-${{ runner.os }}- + + - name: Setup Build Environment + uses: ./.github/actions/setup-build-environment + + - name: Build Shard ${{ matrix.shard.idx }} + env: + FIRMWARE_BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} + # OTA_MANIFEST_BASE_URL and OTA_CHANNEL_TAG (what actually make this a + # beta build) come from the workflow-level env: above, which every step + # inherits. Do NOT redeclare them as ${{ env.X }} here — that is a + # self-reference, and if it resolved empty it would blank the channel. + run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} + + - name: Verify beta channel is baked in + shell: bash + run: | + # Fail fast rather than publish firmware that would OTA itself onto the + # production channel. Checks one built binary actually carries the beta + # manifest URL and does NOT carry the production one. + BIN=$(find .pio/build -name firmware.elf | head -1) + if [ -z "$BIN" ]; then echo "no ELF found to verify" >&2; exit 1; fi + if ! strings "$BIN" | grep -qF "$OTA_MANIFEST_BASE_URL"; then + echo "ERROR: beta manifest base missing from $BIN" >&2; exit 1 + fi + if strings "$BIN" | grep -qE 'https://observer\.gessaman\.com/v"?$'; then + echo "ERROR: production manifest base present in a beta build" >&2; exit 1 + fi + echo "OK: $BIN carries $OTA_MANIFEST_BASE_URL" + + - name: Upload Shard Artifact + uses: actions/upload-artifact@v4 + with: + name: fw-${{ matrix.shard.idx }} + path: out + if-no-files-found: error + + release: + needs: [enumerate, build] + runs-on: ubuntu-latest + steps: + - name: Clone Repo + uses: actions/checkout@v4 + # Shallow on purpose — see the production workflow: `git rev-parse --short` + # must produce the same abbreviation build.sh used for the asset filenames. + + - name: Download All Shard Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Flatten into out/ + run: | + mkdir -p out + find artifacts -type f -name '*.bin' -exec cp -f {} out/ \; + find artifacts -type f -name '*.partsig' -exec cp -f {} out/ \; + echo "Collected binaries:"; ls -1 out + + - name: Compute Short SHA + id: sha + run: echo "short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + + - name: Publish to Beta Rolling Release + env: + GH_TOKEN: ${{ github.token }} + run: | + # Retry wrapper for GitHub API calls. This job runs AFTER ~15 minutes of + # building across 14 runners, and every call below is an API write; with + # `bash -e`, a single transient 5xx throws all of that away. Observed + # 2026-07-19: `gh release create` got HTTP 503 during a GitHub incident + # and killed a run whose builds had all passed. + # `until` in a condition does not trip `-e`, so this is safe here. + gh_retry() { + local n=0 max=5 delay=10 + until "$@"; do + n=$((n + 1)) + if [ "$n" -ge "$max" ]; then + echo "::error::gh failed after $max attempts: $*" >&2 + return 1 + fi + echo "gh call failed (attempt $n/$max), retrying in ${delay}s: $*" >&2 + sleep "$delay" + delay=$((delay * 2)) + done + } + + # Deliberately NOT retried: a plain "release does not exist" is the + # expected answer on the first run, and retrying it would just burn the + # backoff. A 5xx here instead makes us fall through to create, which is + # then tolerated below if the release actually did already exist. + if ! gh release view "$RELEASE_TAG" >/dev/null 2>&1; then + gh_retry gh release create "$RELEASE_TAG" --prerelease \ + --title "MQTT Observer Firmwares (BETA)" \ + --notes "Rolling BETA build. Separate channel from observer-mqtt-latest; beta nodes only OTA within this channel." \ + || gh release view "$RELEASE_TAG" >/dev/null 2>&1 \ + || { echo "::error::could not create or confirm $RELEASE_TAG" >&2; exit 1; } + fi + + gh_retry gh release upload "$RELEASE_TAG" $(find out -maxdepth 1 -type f ! -name '*.partsig') --clobber + + # Keep the release body in sync with the repo's notes source, with the + # dev-channel warning prepended: the /releases feed serves the body as + # this channel's dropdown changelog. Non-fatal — stale notes beat a + # red build whose binaries are already live. + { + printf '%s' '

⚠ DEV/BETA CHANNEL: pre-release firmware. A node flashed from this channel keeps receiving OTA updates from this channel until re-flashed by cable.

' + cat firmware-notes.html + } > /tmp/beta-notes.html + gh_retry gh release edit "$RELEASE_TAG" --notes-file /tmp/beta-notes.html \ + || echo "WARNING: release notes sync failed" >&2 + + # Pruning is housekeeping and runs AFTER the upload has succeeded. If the + # API is flaky here, skip it rather than fail the job — old assets simply + # linger until the next run, which is strictly better than reporting + # failure for a build whose binaries are already published. + KEEP_BUILDS=2 + if ! asset_list=$(gh_retry gh release view "$RELEASE_TAG" --json assets \ + -q '.assets[] | "\(.createdAt) \(.name)"'); then + echo "::warning::could not list assets; skipping prune this run" + exit 0 + fi + keep_hashes=$(printf '%s\n' "$asset_list" \ + | sort -r \ + | while read -r _ts name; do + printf '%s' "$name" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' + done \ + | awk '!seen[$0]++' | head -n "$KEEP_BUILDS") + echo "Retaining build hashes:"; echo "$keep_hashes" + # Reuse asset_list rather than making a second API call (its lines are + # " ", so the name is field 2). + printf '%s\n' "$asset_list" | awk '{print $2}' \ + | while read -r asset; do + ah=$(printf '%s' "$asset" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' || true) + if [ -n "$ah" ] && grep -qxF "$ah" <<<"$keep_hashes"; then + continue + fi + gh release delete-asset "$RELEASE_TAG" "$asset" --yes || true + done + + - name: Checkout Flasher Repo + uses: actions/checkout@v4 + with: + repository: agessaman/flasher.meshcore.io + token: ${{ secrets.FLASHER_DISPATCH_TOKEN }} + path: flasher + + - name: Generate Beta Manifests + env: + BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} + run: | + # config-beta.json is no longer derived here: the flasher's Version + # dropdown is feed-driven (/releases on the firmware-proxy Worker + # lists both channels), so the beta channel needs no config of its + # own and this workflow's flasher commit touches only beta/v/ and the + # counter. + mkdir -p "flasher/$MANIFEST_DIR" + # Slim manifests come from the build output in out/ (the assets + # actually uploaded to the release), not from config-beta.json — see + # gen-slim-manifests.py's --bin-dir mode (flasher repo PR #1). + python3 flasher/scripts/gen-slim-manifests.py \ + --bin-dir out \ + --static-path "$STATIC_PATH" \ + --out-dir "flasher/$MANIFEST_DIR" \ + --base-version "$FIRMWARE_VERSION" \ + --build "$BUILD_NUMBER" \ + --partsig-dir out + + printf '{\n "baseVersion": "%s",\n "build": %s\n}\n' \ + "$FIRMWARE_VERSION" "$BUILD_NUMBER" > "flasher/$COUNTER_FILE" + echo "Beta build $FIRMWARE_VERSION.$BUILD_NUMBER" + + - name: Verify beta manifests point at the beta channel + run: | + # Guards against a beta manifest handing out a production download URL. + SAMPLE=$(find "flasher/$MANIFEST_DIR" -name '*.json' | head -1) + echo "sample: $SAMPLE"; cat "$SAMPLE" + if ! grep -qF "$STATIC_PATH" "$SAMPLE"; then + echo "ERROR: beta manifest does not use $STATIC_PATH" >&2; exit 1 + fi + + # NOTE: production's "Generate Changelog" and "Sync Docs into Flasher" steps + # are deliberately omitted. Those rewrite site-wide content (CHANGELOG.md, + # MQTT_IMPLEMENTATION.md, ...) that the production channel owns; a beta build + # must not overwrite them. + + - name: Commit & Push Beta Artifacts + working-directory: flasher + run: | + # Scoped add: beta only ever touches its own manifest dir and counter, + # so a stray edit elsewhere in the flasher checkout (in particular + # production's config.json) can never be published by this workflow. + git add -A "$MANIFEST_DIR" "$COUNTER_FILE" + if git diff --cached --quiet; then + echo "No beta changes to commit." + exit 0 + fi + git config user.name "meshcore-bot" + git config user.email "noreply@gessaman.com" + git commit -m "Update BETA observer firmware to ${{ steps.sha.outputs.short }} (build ${FIRMWARE_VERSION}.${{ needs.enumerate.outputs.build_number }})" + # The production workflows can be pushing to the flasher repo right now + # (no shared concurrency group any more), so a non-fast-forward is expected + # rather than fatal: rebase onto their commit and retry. The scoped add + # above keeps this commit inside the beta paths, so a conflict with + # production's files is not possible. + br=$(git rev-parse --abbrev-ref HEAD) + for attempt in 1 2 3; do + if git push origin "HEAD:$br"; then + exit 0 + fi + echo "push rejected (attempt $attempt); rebasing onto origin/$br" + git fetch origin "$br" + # --autostash: the scoped add above leaves unrelated edits in the + # checkout unstaged on purpose, and plain rebase refuses to run with a + # dirty tree. They stay unstaged, so they still never get published. + git rebase --autostash "origin/$br" || { + git rebase --abort || true + echo "::error::flasher rebase conflicted; re-run this workflow to republish" + exit 1 + } + done + echo "::error::could not push beta changes after 3 attempts" + exit 1 diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml new file mode 100644 index 0000000000..13d9cda0fb --- /dev/null +++ b/.github/workflows/build-observer-firmwares.yml @@ -0,0 +1,335 @@ +name: Build MQTT Observer Firmwares + +permissions: + contents: write + +on: + workflow_dispatch: + push: + branches: + - observer-firmware + # Only rebuild firmware when something that affects the binaries changes. + # Docs, the changelog, the changelog generator, CI files, and repo-meta + # files do not change firmware output — docs/changelog are handled by + # sync-flasher-content.yml instead. NOTE: a push is skipped only if EVERY + # changed file matches a pattern below; one stray unlisted file (e.g. an + # earlier .gitignore edit) triggers a full build, so keep meta files listed. + paths-ignore: + - '**.md' + - 'docs/**' + - 'scripts/gen_changelog.py' + - '.github/**' + - '.gitignore' + - '.gitattributes' + - '.editorconfig' + - 'LICENSE' + - '.vscode/**' + - '.claude/**' + # Throwaway build worktrees never affect the binaries. + - '.build-wt-*' + - '.build-wt-*/**' + +# Own group: serialize builds against each other only. Do NOT share this group +# with sync-flasher-content.yml — a mixed code+docs commit triggers both, and a +# run that is still pending when its sibling is queued gets cancelled, which +# silently skipped the firmware build. Concurrent flasher pushes are instead made +# safe by the rebase-retry in "Commit & Push Flasher Config". +concurrency: + group: observer-firmware-build + cancel-in-progress: false + +env: + # Version embedded in firmware filenames; must match the version key in the + # flasher's config.json. Bump here when the observer version changes. + FIRMWARE_VERSION: v1.17.1 + # Rolling release tag that hosts the latest observer binaries. + RELEASE_TAG: observer-mqtt-latest + # Download host serving RELEASE_TAG's assets (cloudflare-worker in the flasher + # repo). Baked into the slim OTA manifests' file URLs; must stay consistent + # with config.json's staticPath. + STATIC_PATH: https://observer-fw.gessaman.com + +jobs: + + # Discover the *_observer_mqtt envs and split them into SHARDS groups so the + # build fans out across runners (pio builds envs sequentially within a job). + enumerate: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.split.outputs.matrix }} + build_number: ${{ steps.buildnum.outputs.n }} + steps: + - name: Clone Repo + uses: actions/checkout@v4 + + - name: Split observer envs into shards + id: split + shell: bash + run: | + SHARDS=14 + ENVS=$(grep -rhoE '^\[env:[^]]*observer_mqtt\]' platformio.ini variants/*/platformio.ini \ + | sed -E 's/^\[env:(.*)\]$/\1/' | sort -u) + echo "Discovered envs:"; echo "$ENVS" + MATRIX=$(echo "$ENVS" | awk -v n="$SHARDS" ' + { shard[NR % n] = shard[NR % n] " " $0 } + END { for (i = 0; i < n; i++) { sub(/^ /, "", shard[i]); + printf "{\"idx\":%d,\"envs\":\"%s\"}\n", i, shard[i] } }' \ + | jq -cs .) + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + + - name: Compute observer build number + id: buildnum + shell: bash + run: | + # Per-base published-build counter. N increments once per release and + # resets to 1 when FIRMWARE_VERSION (the MeshCore base version) changes. + # Read-only here (off the published counter); the release job is the sole + # writer and only writes on a successful build, so a failed build doesn't + # burn a number. First run / 404 -> empty -> N=1. + COUNTER_URL="https://observer.gessaman.com/observer-build-counter.json" + CUR=$(curl -fsSL "$COUNTER_URL" 2>/dev/null || echo '{}') + PREV_BASE=$(echo "$CUR" | jq -r '.baseVersion // ""') + PREV_BUILD=$(echo "$CUR" | jq -r '.build // 0') + if [ "$PREV_BASE" = "$FIRMWARE_VERSION" ]; then + N=$((PREV_BUILD + 1)) + else + N=1 + fi + echo "Base $FIRMWARE_VERSION; previous build $PREV_BUILD (base $PREV_BASE) -> N=$N" + echo "n=$N" >> "$GITHUB_OUTPUT" + + # Build one shard (several envs) per runner. build.sh emits both the app + # .bin and the ESP32 -merged.bin into out/ for each env. + build: + needs: enumerate + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON(needs.enumerate.outputs.matrix) }} + steps: + - name: Clone Repo + uses: actions/checkout@v4 + + # Cache the installed PlatformIO platforms + toolchains (the espressif32 + # toolchain + Arduino framework are pinned in platformio.ini, so they are + # identical across commits). This skips the multi-hundred-MB download AND + # re-extraction that otherwise happens on every fresh runner's first build. + # Note: compiled objects are not worth caching here because build.sh injects + # a changing -DFIRMWARE_VERSION/-DFIRMWARE_BUILD_DATE into the global build + # flags, which busts every object's compile-command checksum each build. + - name: Cache PlatformIO Toolchains + uses: actions/cache@v4 + with: + path: | + ~/.platformio/packages + ~/.platformio/platforms + key: pio-toolchains-${{ runner.os }}-${{ hashFiles('platformio.ini') }} + restore-keys: | + pio-toolchains-${{ runner.os }}- + + - name: Setup Build Environment + uses: ./.github/actions/setup-build-environment + + - name: Build Shard ${{ matrix.shard.idx }} + env: + # Stamp the per-base build number into the embedded version (v1.16.0.N). + FIRMWARE_BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} + run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} + + - name: Upload Shard Artifact + uses: actions/upload-artifact@v4 + with: + name: fw-${{ matrix.shard.idx }} + path: out + if-no-files-found: error + + # Collect all shard outputs, publish the rolling release, then point the + # flasher at the new build by bumping the hash + notes in its config.json. + release: + needs: [enumerate, build] + runs-on: ubuntu-latest + steps: + - name: Clone Repo + uses: actions/checkout@v4 + # Intentionally shallow (default depth 1). "Compute Short SHA" below must + # run on the same shallow clone the build job used, because + # `git rev-parse --short` auto-extends its abbreviation as the object + # count grows (7 chars shallow, 8 chars with full history). build.sh names + # the firmware assets from a shallow clone, so a full-history release job + # would write an 8-char hash into config.json that no asset matches. + # History is deepened later, only for the changelog step. + + - name: Download All Shard Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + + - name: Flatten into out/ + run: | + mkdir -p out + find artifacts -type f -name '*.bin' -exec cp -f {} out/ \; + # Per-env partition-table signatures (for the slim manifest's OTA gate). + find artifacts -type f -name '*.partsig' -exec cp -f {} out/ \; + echo "Collected binaries:"; ls -1 out + + - name: Compute Short SHA + id: sha + run: echo "short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + + - name: Publish to Rolling Release (tag pinned; assets replaced in place) + env: + GH_TOKEN: ${{ github.token }} + run: | + # Create the release + tag once, then never move the tag again. Moving a + # rolling tag on every build causes "conflicting tag" errors in local + # clones (and a brief 404 window from delete/recreate). + if ! gh release view "$RELEASE_TAG" >/dev/null 2>&1; then + gh release create "$RELEASE_TAG" --prerelease \ + --title "MQTT Observer Firmwares" \ + --notes "Rolling build of all *_observer_mqtt firmwares. The git short hash is embedded in each asset filename." + fi + + # Upload this build, overwriting same-named assets. Exclude the .partsig + # files — those are consumed locally by the slim-manifest generator below, + # not published as release assets (and they'd break the hash-based prune). + gh release upload "$RELEASE_TAG" $(find out -maxdepth 1 -type f ! -name '*.partsig') --clobber + + # Keep the release body in sync with the repo's notes source: the + # /releases feed serves it as the flasher dropdown's changelog + # (config.json no longer carries per-version notes). Non-fatal — + # stale notes beat a red build whose binaries are already live. + gh release edit "$RELEASE_TAG" --notes-file firmware-notes.html \ + || echo "WARNING: release notes sync failed" >&2 + + # Prune old assets, but RETAIN the most recent KEEP_BUILDS build hashes + # (not just the current one). The flasher SPA reads config.json once at + # page load and the embedded git short-hash is what links it to assets; + # if we pruned down to only the current build, a tab opened during the + # previous build cycle would request an already-deleted binary and 404. + # Keeping the previous build covers that window. + # Pick the KEEP_BUILDS most-recent distinct build hashes: list assets as + # " ", sort newest-first (ISO timestamps sort lexically), + # pull the git short-hash out of each filename, de-dup preserving order. + KEEP_BUILDS=2 + keep_hashes=$(gh release view "$RELEASE_TAG" --json assets \ + -q '.assets[] | "\(.createdAt) \(.name)"' \ + | sort -r \ + | while read -r _ts name; do + printf '%s' "$name" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' + done \ + | awk '!seen[$0]++' | head -n "$KEEP_BUILDS") + echo "Retaining build hashes:"; echo "$keep_hashes" + gh release view "$RELEASE_TAG" --json assets -q '.assets[].name' \ + | while read -r asset; do + # `|| true`: a non-firmware asset (no embedded hash) makes grep exit + # non-zero, which under `bash -e` would abort the step. Empty ah then + # falls through to delete-asset, cleaning up any stray non-.bin asset. + ah=$(printf '%s' "$asset" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' || true) + if [ -n "$ah" ] && grep -qxF "$ah" <<<"$keep_hashes"; then + continue + fi + gh release delete-asset "$RELEASE_TAG" "$asset" --yes || true + done + + - name: Checkout Flasher Repo + uses: actions/checkout@v4 + with: + repository: agessaman/flasher.meshcore.io + token: ${{ secrets.FLASHER_DISPATCH_TOKEN }} + path: flasher + + # update-firmware.py is no longer called here: config.json's observer + # entries are github release defs (no embedded filenames to rewrite), the + # flasher's versions come from the Worker's /releases feed, and the + # changelog notes ride the release body (see the publish step). + + - name: Generate Slim Per-Variant Manifests + Persist Build Counter + env: + BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} + run: | + # Derive the slim per-variant manifests (flasher/v/.json) that the + # firmware fetches for `ota check`/`ota update`, from the build output + # in out/ — the assets actually uploaded to the release — stamping this + # build's number. Then persist the counter so the next run increments + # from here. + python3 flasher/scripts/gen-slim-manifests.py \ + --bin-dir out \ + --static-path "$STATIC_PATH" \ + --out-dir flasher/v \ + --base-version "$FIRMWARE_VERSION" \ + --build "$BUILD_NUMBER" \ + --partsig-dir out + printf '{\n "baseVersion": "%s",\n "build": %s\n}\n' \ + "$FIRMWARE_VERSION" "$BUILD_NUMBER" > flasher/observer-build-counter.json + echo "Build $FIRMWARE_VERSION.$BUILD_NUMBER" + + - name: Unshallow for Changelog + run: | + # gen_changelog.py needs the full branch history; deepen only now, AFTER + # Compute Short SHA ran on the shallow clone (so its 7-char abbreviation + # matches build.sh's firmware filenames). Guarded so a non-shallow clone + # (e.g. a manual full checkout) doesn't error. + if [ "$(git rev-parse --is-shallow-repository)" = "true" ]; then + git fetch --unshallow --quiet + fi + + - name: Generate Changelog + run: | + # Append any new branch commits to the flasher's CHANGELOG.md. This is + # append-only and idempotent: the hand-curated history and the hash + # manifest already in flasher/CHANGELOG.md are preserved, and only + # commits not yet listed are added. The flasher repo is the persistent + # store of changelog state across builds; this checkout's git history + # (fetch-depth: 0) is the source. changelog.html renders this file. + python3 scripts/gen_changelog.py flasher/CHANGELOG.md + + - name: Sync Docs into Flasher + run: | + # docs.html on the flasher site serves these raw .md files and renders + # them client-side; keep this list in sync with LOCAL_DOCS in + # flasher/docs.html. + for f in MQTT_IMPLEMENTATION.md MQTT_SNMP.md ALERTS.md; do + if [ -f "$f" ]; then + cp -f "$f" "flasher/$f" + echo "synced $f" + else + echo "WARNING: source doc $f not found" >&2 + fi + done + + - name: Commit & Push Flasher Config + working-directory: flasher + run: | + # Stage everything: the slim manifests (v/*.json) and the build counter + # can be NEW files, which `commit -am` would miss — so add -A and check + # the staged diff. (config.json is no longer rewritten per build.) + git add -A + if git diff --cached --quiet; then + echo "No flasher changes to commit." + exit 0 + fi + git config user.name "meshcore-bot" + git config user.email "noreply@gessaman.com" + git commit -m "Update observer firmware to ${{ steps.sha.outputs.short }} (build ${FIRMWARE_VERSION}.${{ needs.enumerate.outputs.build_number }})" + # sync-flasher-content.yml can be pushing the same docs/changelog right + # now (a mixed code+docs commit triggers both), so a non-fast-forward is + # expected rather than fatal: rebase onto the sibling's commit and retry. + # If the rebase leaves nothing, git reports up-to-date and we exit clean. + br=$(git rev-parse --abbrev-ref HEAD) + for attempt in 1 2 3; do + if git push origin "HEAD:$br"; then + exit 0 + fi + echo "push rejected (attempt $attempt); rebasing onto origin/$br" + git fetch origin "$br" + # --autostash so a dirty tree can never make the rebase refuse to run; + # nothing unstaged gets published either way. + git rebase --autostash "origin/$br" || { + git rebase --abort || true + echo "::error::flasher rebase conflicted; re-run this workflow to republish" + exit 1 + } + done + echo "::error::could not push flasher changes after 3 attempts" + exit 1 diff --git a/.github/workflows/check-mqtt-preset-parity.yml b/.github/workflows/check-mqtt-preset-parity.yml new file mode 100644 index 0000000000..e1039a40e2 --- /dev/null +++ b/.github/workflows/check-mqtt-preset-parity.yml @@ -0,0 +1,90 @@ +name: Check MQTT Preset Name Parity + +# Ensures observer-firmware and observer-firmware-dev share the same built-in +# MQTT preset *names* (config details may differ). See scripts/check_mqtt_preset_parity.py. + +permissions: + contents: read + +on: + workflow_dispatch: + pull_request: + branches: + - observer-firmware + - observer-firmware-dev + paths: + - 'src/helpers/MQTTPresets.h' + - 'scripts/check_mqtt_preset_parity.py' + - '.github/workflows/check-mqtt-preset-parity.yml' + push: + branches: + - observer-firmware + - observer-firmware-dev + paths: + - 'src/helpers/MQTTPresets.h' + - 'scripts/check_mqtt_preset_parity.py' + - '.github/workflows/check-mqtt-preset-parity.yml' + +jobs: + parity: + runs-on: ubuntu-latest + steps: + - name: Clone Repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch channel branches + run: | + git fetch --no-tags origin observer-firmware observer-firmware-dev + + - name: Materialize presets from both channels + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.ref }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + PRESET_PATH=src/helpers/MQTTPresets.h + mkdir -p /tmp/preset-parity + + if [ "$EVENT_NAME" = "pull_request" ]; then + # Ensure the PR head commit is reachable (merge checkout may not keep it). + git fetch --no-tags origin "$PR_HEAD_SHA" + # Proposed state of the PR's base channel vs current tip of the sibling. + git show "${PR_HEAD_SHA}:${PRESET_PATH}" > /tmp/preset-parity/pr-head.h + case "$PR_BASE" in + observer-firmware-dev) + cp /tmp/preset-parity/pr-head.h /tmp/preset-parity/dev.h + git show "origin/observer-firmware:${PRESET_PATH}" > /tmp/preset-parity/prod.h + ;; + observer-firmware) + git show "origin/observer-firmware-dev:${PRESET_PATH}" > /tmp/preset-parity/dev.h + cp /tmp/preset-parity/pr-head.h /tmp/preset-parity/prod.h + ;; + *) + echo "::error::unexpected PR base branch: $PR_BASE" >&2 + exit 1 + ;; + esac + else + # push / workflow_dispatch: compare current channel tips. + git show "origin/observer-firmware:${PRESET_PATH}" > /tmp/preset-parity/prod.h + git show "origin/observer-firmware-dev:${PRESET_PATH}" > /tmp/preset-parity/dev.h + fi + + echo "=== production (observer-firmware) head preview ===" + head -n 5 /tmp/preset-parity/prod.h + echo "=== dev (observer-firmware-dev) head preview ===" + head -n 5 /tmp/preset-parity/dev.h + + - name: Self-test checker + run: python3 scripts/check_mqtt_preset_parity.py --self-test + + - name: Compare preset names + run: | + python3 scripts/check_mqtt_preset_parity.py \ + /tmp/preset-parity/prod.h \ + /tmp/preset-parity/dev.h \ + --label-a observer-firmware \ + --label-b observer-firmware-dev diff --git a/.github/workflows/pr-build-check.yml b/.github/workflows/pr-build-check.yml index cebf0cfe5c..788390c57a 100644 --- a/.github/workflows/pr-build-check.yml +++ b/.github/workflows/pr-build-check.yml @@ -5,17 +5,35 @@ on: branches: [main, dev] paths: - 'src/**' + - 'include/**' + - 'lib/**' - 'examples/**' - 'variants/**' + - 'boards/**' + - 'arch/**' + - 'scripts/**' + - 'ssl_certs/**' + - 'webui/**' + - 'default_8MB.csv' - 'platformio.ini' + - '.github/actions/setup-build-environment/**' - '.github/workflows/pr-build-check.yml' push: branches: [main, dev] paths: - 'src/**' + - 'include/**' + - 'lib/**' - 'examples/**' - 'variants/**' + - 'boards/**' + - 'arch/**' + - 'scripts/**' + - 'ssl_certs/**' + - 'webui/**' + - 'default_8MB.csv' - 'platformio.ini' + - '.github/actions/setup-build-environment/**' - '.github/workflows/pr-build-check.yml' jobs: @@ -29,6 +47,9 @@ jobs: - Heltec_v3_companion_radio_ble - Heltec_v3_repeater - Heltec_v3_room_server + # MQTT observer smoke builds: internal RAM and PSRAM coverage. + - Heltec_v3_repeater_observer_mqtt + - T_Beam_S3_Supreme_SX1262_repeater_observer_mqtt # nRF52 - RAK_4631_companion_radio_ble - RAK_4631_companion_radio_ethernet diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 5d48f4c67b..826c4cb37c 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -19,6 +19,11 @@ jobs: - name: Setup Build Environment uses: ./.github/actions/setup-build-environment + - name: Verify ArduinoJson pin + run: | + python3 -B scripts/check_arduinojson_pin.py --self-test + python3 -B scripts/check_arduinojson_pin.py + - name: Run Unit Tests run: pio test -e native -e native_kiss_modem -vv diff --git a/.github/workflows/sync-flasher-content.yml b/.github/workflows/sync-flasher-content.yml new file mode 100644 index 0000000000..7c4c409d2b --- /dev/null +++ b/.github/workflows/sync-flasher-content.yml @@ -0,0 +1,116 @@ +name: Sync Flasher Docs & Changelog + +# Pushes the observer docs and the regenerated changelog to the flasher site +# WITHOUT rebuilding firmware. Companion to build-observer-firmwares.yml: +# - code changes -> build-observer-firmwares.yml (builds + syncs everything) +# - doc / markdown edits -> this workflow (fast sync only, no firmware rebuild) +# Keep the synced file list and trigger paths in step with the build workflow's +# "Sync Docs into Flasher" step and with LOCAL_DOCS in flasher/docs.html. + +permissions: + contents: read + +on: + workflow_dispatch: + push: + branches: + - observer-firmware + paths: + - '**.md' + - 'docs/**' + - 'scripts/gen_changelog.py' + # /webconfig frames the real portal page against a browser simulator, so + # the page itself is a synced doc asset like the .md files above. + - 'webui/index.html' + +# Own group: serialize doc syncs against each other only. Deliberately NOT shared +# with build-observer-firmwares.yml — a shared group let one workflow cancel the +# other's pending run (a mixed code+docs commit triggers both). Concurrent flasher +# pushes are handled by the rebase-retry in "Commit & Push Flasher Content". +concurrency: + group: flasher-docs-sync + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Clone Repo + uses: actions/checkout@v4 + with: + # full history so scripts/gen_changelog.py can read the branch commit log + fetch-depth: 0 + + - name: Checkout Flasher Repo + uses: actions/checkout@v4 + with: + repository: agessaman/flasher.meshcore.io + token: ${{ secrets.FLASHER_DISPATCH_TOKEN }} + path: flasher + + - name: Sync Docs into Flasher + run: | + # docs.html on the flasher site serves these raw .md files and renders + # them client-side; keep this list in sync with LOCAL_DOCS in + # flasher/docs.html and the build workflow's sync step. + for f in MQTT_IMPLEMENTATION.md MQTT_SNMP.md ALERTS.md; do + if [ -f "$f" ]; then + cp -f "$f" "flasher/$f" + echo "synced $f" + else + echo "WARNING: source doc $f not found" >&2 + fi + done + + - name: Sync the WebConfig Portal Page into Flasher + run: | + # flasher/webconfig.html embeds the REAL portal page in a device + # surround and runs it against lib/webconfig-sim.js, which intercepts + # fetch() so no device is needed. Syncing the page is what keeps that + # demo honest — the alternative is screenshots, which go stale without + # anyone noticing. The build script only injects a " in low): + mode = "html" + + return "\n".join(out) + "\n" + + +def check_stripped(raw, stripped): + """Guard against a stripper bug silently shipping a broken portal to the + fleet. Returns a reason string when the output looks wrong, else None.""" + for tag in ("", "", ""): + if raw.count(tag) != stripped.count(tag): + return "%s count changed" % tag + if len(stripped) < len(raw) * 0.5: + return "output shrank by more than half (%d -> %d)" % (len(raw), len(stripped)) + # Only comments and whitespace may go, so no structural token may appear + # that the source did not already have. + for token in ("{", "}", "(", ")", " raw.count(token): + return "gained a %s" % token + return None diff --git a/scripts/webconfig_mock_server.py b/scripts/webconfig_mock_server.py new file mode 100644 index 0000000000..c38d804e3f --- /dev/null +++ b/scripts/webconfig_mock_server.py @@ -0,0 +1,1069 @@ +#!/usr/bin/env python3 +"""Local mock of the WebConfig portal backend, for iterating on webui/index.html +in a real browser with no firmware, no flashing, and no paid emulator account. + +It serves the real webui/index.html and implements the same /api/* contract as +src/helpers/esp32/WebConfigServer.cpp — including the 202+reqid handshake, the +pending -> done result polling, aggregate-success reboot gating, secret masking +(********), and the IATA / owner-key / length validation the firmware enforces. +So the browser drives the actual portal JS (wizard, save/poll/reqid, effective +value handling, reboot overlay, stats, scan) against realistic responses. + +/api/cli is the CLI terminal's backend and has no firmware counterpart yet: it +is the proposed contract (202 + reqid, streamed per-command results) executed +against a CommonCLI-shaped interpreter, so the terminal UI can be designed +against realistic single- and multi-line replies before any of it goes on-device. + +It does NOT run the C++ handlers (that's what test/ gtest covers) or the +AsyncTCP transport — it's a frontend + contract harness. + +Usage: + python3 scripts/webconfig_mock_server.py # LAN mode (login: password) + python3 scripts/webconfig_mock_server.py --setup # first-boot setup wizard + python3 scripts/webconfig_mock_server.py --port 9000 --active-slots 2 +Then open http://localhost:8080/ (or the chosen port). Editing index.html and +refreshing shows changes immediately — the page is re-read per request. + +Stdlib only; no pip install. +""" + +import argparse +import copy +import json +import os +import re +import secrets +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlsplit + +HERE = os.path.dirname(os.path.abspath(__file__)) +INDEX_HTML = os.path.join(HERE, "..", "webui", "index.html") + +sys.path.insert(0, HERE) +# The build-time comment stripper, shared so --minify serves byte-for-byte what +# the generator embeds rather than a second implementation that could drift. +from webconfig_minify import strip_source # noqa: E402 + +MINIFY = False +# Overridable with --fw-version to exercise the console's channel labelling: +# v1.16.0.5-observer-a1b2c3d release +# v1.16.0.5-observer-beta-dev-a1b2c3d dev +# v1.16.0 local build, no OTA +FW_VERSION = "v1.16.0.5-observer-a1b2c3d" + +SENTINEL = "********" +ADMIN_PASSWORD = "password" # matches the default ADMIN_PASSWORD build flag +BATCH_PENDING_SECS = 0.8 # how long POST->done takes, to exercise polling +SCAN_SECS = 0.8 + +# Destination buffer sizes (chars, minus the NUL) — mirrors the MQTTPrefs fields +# the firmware validates in CommonCLI_Observer.cpp. +LEN_LIMITS = { + "name": 31, "wifi.ssid": 31, "wifi.pwd": 63, "mqtt.origin": 31, + "mqtt.email": 63, "mqtt.ntp": 63, "timezone": 31, "snmp.community": 23, +} +# "filter" is absent on purpose: it is a bitmask, not a text buffer, so it has +# no destination-buffer limit. It is still bounded by the shared CLI command +# budget below, like every other key. +SLOT_LEN_LIMITS = {"server": 63, "username": 31, "password": 63, + "token": 47, "topic": 95, "audience": 63} + +# BatchEntry::cmd[160] in WebConfigServer.cpp holds "set " plus a +# NUL. Over-long values are rejected there rather than truncated, because a +# clipped value can still be valid and would persist as a different setting. +BATCH_CMD_SIZE = 160 + +# Preset names + what the UI must collect (mirrors handlePresets()). +PRESETS = ( + [(n, "none") for n in ( + "analyzer-us", "analyzer-eu", "nz-analyzer", "meshmapper", "waev", + "meshomatic", "cascadiamesh", "tennmesh", "nashmesh", "ctmesh", "chimesh", + "meshat.se", "eastidahomesh", "coloradomesh", "dutchmeshcore-1", + "dutchmeshcore-2", "meshcore-ca-1", "meshcore-ca-2", "meshcore-fi", + "bostonmesh", "rflab", "ipnt.uk", "flmesh", "corecomms")] + + [("meshrank", "token"), ("inwmesh", "userpass")] +) + +SCAN_NETWORKS = [ + {"ssid": "Wokwi-GUEST", "rssi": -42, "enc": False}, + {"ssid": "HomeNet", "rssi": -55, "enc": True}, + {"ssid": "HomeNet-5G", "rssi": -61, "enc": True}, + {"ssid": "Neighbor 2.4", "rssi": -78, "enc": True}, + {"ssid": "OpenGuest", "rssi": -83, "enc": False}, +] + + +def default_config(setup_mode): + return { + "radio": { + "freq": 910.525, "bw": 62.5, "sf": 7, "cr": 5, "tx": 22, "af": 1.0, + "rxdelay": 0.0, "txdelay": 0.5, "cad": False, "rxgain": True, + "repeat": True, "flood_max": 64, "flood_max_advert": 8, + "flood_max_unscoped": 8, "loop_detect": "moderate", + "name": "MockNode", "lat": 39.7392, "lon": -104.9903, + "advert_interval": 240, "flood_advert_interval": 6, + }, + "wifi": { + # setup mode = unconfigured (empty ssid -> wizard); LAN mode = joined + "ssid": "" if setup_mode else "HomeNet", + "pwd": "" if setup_mode else "secretpw", # stored raw; masked on GET + "powersave": "min", + }, + "mqtt": { + "origin": "" if setup_mode else "MockNode", "iata": "" if setup_mode else "DEN", + "status": True, "packets": True, "raw": False, "tx": "advert", "rx": True, + "interval": 5, "timezone": "MST7MDT,M3.2.0,M11.1.0", "timezone_offset": -7, + "ntp": "pool.ntp.org", "owner": "", "email": "", "snmp": False, + "snmp_community": "public", + "neighbors": False, "neighbors_interval": 24, + "slots": [_slot() for _ in range(6)], + }, + # Settings the CLI reaches but no portal form does, so they are absent + # from /api/config (see config_json) and live only here. Without them + # the terminal answers "unknown config key" for perfectly real commands. + "cli": { + "radio.watchdog": 0, "int.thresh": 0, "agc.reset.interval": 0, + "direct.txdelay": 0.0, "multi.acks": 0, "allow.read.only": False, + "path.hash.mode": 0, "owner.info": "", "guest.password": "", + "adc.multiplier": 1.0, + "alert": False, "alert.psk": "", "alert.hashtag": "", + "alert.region": "", "alert.interval": 15, + "alert.mqtt": False, "alert.wifi": False, + "bridge.enabled": False, "bridge.source": "rx", "bridge.baud": 115200, + "bridge.delay": 0, "bridge.channel": 0, "bridge.secret": "", + }, + } + + +def _slot(): + return {"preset": "none", "server": "", "port": 8883, "username": "", + "password": "", "token": "", "topic": "", "audience": "", + "filter": "all"} + + +class State: + def __init__(self, args): + self.lock = threading.Lock() + self.setup_mode = args.setup + self.active_slots = args.active_slots + self.cfg = default_config(args.setup) + # latched at AP start, like WebConfigServer::_initial_setup + self.initial_setup = args.setup and self.cfg["wifi"]["ssid"] == "" + self.start = time.time() + self.session = None # cookie token when logged in (LAN mode) + self.batch = {"state": "idle"} + self.cli = {"state": "idle"} # deferred CLI sequence, see /api/cli + self.admin_pwd_set = False # satisfies the initial-setup invariant + self.scan_started = None + + # ---- auth ------------------------------------------------------------- + def is_authed(self, headers): + if self.setup_mode: + return True # setup mode: proximity trust, no auth + if not self.session: + return False + cookie = headers.get("Cookie", "") + m = re.search(r"wcs=([0-9a-f]+)", cookie) + return bool(m and m.group(1) == self.session) + + # ---- config serialization (masks secrets, like handleConfigGet) ------- + def config_json(self): + c = copy.deepcopy(self.cfg) + c.pop("cli") # CLI-only settings: not part of this contract + c["wifi"]["pwd"] = SENTINEL if self.cfg["wifi"]["pwd"] else "" + for s in c["mqtt"]["slots"]: + s["password"] = SENTINEL if s["password"] else "" + s["token"] = SENTINEL if s["token"] else "" + return c + + def status_json(self, authed): + return { + "mode": "setup" if self.setup_mode else "lan", + "auth": authed, + "needs_setup": self.cfg["wifi"]["ssid"] == "", + "name": self.cfg["radio"]["name"], "node_id": "a1b2c3d4e5f60718", + # Shaped like build.sh's EMBEDDED_VERSION_STRING + # (base[.build][-observer][-channel]-hash) so the console's channel + # labelling is exercised against a real version, not "v1.x-mock". + "fw": FW_VERSION, "build_date": "6 Jun 2026", + "role": "Repeater", "board": "Heltec V3 (mock)", + "uptime_s": int(time.time() - self.start), + "runtime_slots": 6, "max_slots": 6, "active_slots": self.active_slots, + "max_cmds": CLI_MAX_CMDS, + } + + +# --------------------------------------------------------------------------- +# set-command application + validation (mirrors the firmware's setters enough +# to produce realistic per-field OK / Error replies for the UI chips). +# --------------------------------------------------------------------------- +BOOL_KEYS = {"cad": ("radio", "cad"), "radio.rxgain": ("radio", "rxgain"), + "repeat": ("radio", "repeat"), "mqtt.status": ("mqtt", "status"), + "mqtt.packets": ("mqtt", "packets"), "mqtt.raw": ("mqtt", "raw"), + "mqtt.rx": ("mqtt", "rx"), "snmp": ("mqtt", "snmp"), + "mqtt.neighbors": ("mqtt", "neighbors")} +INT_KEYS = {"tx": ("radio", "tx"), "flood.max": ("radio", "flood_max"), + "flood.max.advert": ("radio", "flood_max_advert"), + "flood.max.unscoped": ("radio", "flood_max_unscoped"), + "advert.interval": ("radio", "advert_interval"), + "flood.advert.interval": ("radio", "flood_advert_interval"), + "mqtt.interval": ("mqtt", "interval"), + "mqtt.neighbors.interval": ("mqtt", "neighbors_interval"), + "timezone.offset": ("mqtt", "timezone_offset")} +FLOAT_KEYS = {"lat": ("radio", "lat"), "lon": ("radio", "lon"), + "af": ("radio", "af"), "rxdelay": ("radio", "rxdelay"), + "txdelay": ("radio", "txdelay")} +STR_KEYS = {"name": ("radio", "name"), "wifi.ssid": ("wifi", "ssid"), + "wifi.powersave": ("wifi", "powersave"), "loop.detect": ("radio", "loop_detect"), + "mqtt.origin": ("mqtt", "origin"), "mqtt.ntp": ("mqtt", "ntp"), + "mqtt.email": ("mqtt", "email"), "timezone": ("mqtt", "timezone"), + "snmp.community": ("mqtt", "snmp_community"), "mqtt.tx": ("mqtt", "tx")} +SECRET_STR_KEYS = {"wifi.pwd": ("wifi", "pwd")} + +# The CLI-only settings, typed the same way so apply_set/cli_read_key reach them +# through the existing lookups rather than a parallel code path. +for _k, _v in default_config(False)["cli"].items(): + _t = {bool: BOOL_KEYS, int: INT_KEYS, float: FLOAT_KEYS, str: STR_KEYS}[type(_v)] + _t[_k] = ("cli", _k) +SECRET_STR_KEYS.update({k: ("cli", k) for k in + ("guest.password", "alert.psk", "bridge.secret")}) +for _k in SECRET_STR_KEYS: + STR_KEYS.pop(_k, None) + + +def _hex64(v): + return len(v) == 64 and all(c in "0123456789abcdefABCDEF" for c in v) + + +def apply_set(cfg, key, val): + """Return (ok, reply) and mutate cfg. Mirrors the firmware's validation for + the fields where it matters (length, IATA, owner key, port, radio combo).""" + # length guard for the plain string fields + if key in LEN_LIMITS and len(val) > LEN_LIMITS[key]: + return False, "Error: %s too long (max %d chars)" % (key, LEN_LIMITS[key]) + + if key == "password": + # Stored outside cfg: it must never appear in the /api/config GET. The + # firmware overwrites the CLI's "password now: " echo, so the + # reply carries no secret either. + global ADMIN_PASSWORD + ADMIN_PASSWORD = val + return True, "OK" + + if key == "radio.fem.rxgain": + return False, "Error: unsupported" # no FEM on the mock board, see GETTERS + + if key == "dutycycle": + try: + dc = float(val) + except ValueError: + return False, "Error: expected a number" + if not 0 < dc <= 100: + return False, "Error, must be 1-100" + cfg["radio"]["af"] = 100.0 / dc - 1 # the CLI stores it as airtime_factor + return True, "OK" + + if key in ("freq", "bw", "sf", "cr"): + # single-component radio setters, reachable from the CLI but not from + # the form batch (which always sends the whole `radio` combo) + try: + cfg["radio"][key] = int(val) if key in ("sf", "cr") else float(val) + except ValueError: + return False, "Error: expected a number" + return True, "OK - reboot to apply" + + if key == "radio": + try: + f, bw, sf, cr = val.split(",") + f, bw, sf, cr = float(f), float(bw), int(sf), int(cr) + except ValueError: + return False, "Error, invalid radio params" + if not (150 <= f <= 2500 and 7 <= bw <= 500 and 5 <= sf <= 12 and 5 <= cr <= 8): + return False, "Error, invalid radio params" + cfg["radio"].update(freq=f, bw=bw, sf=sf, cr=cr) + return True, "OK - reboot to apply" + + if key == "mqtt.iata": + if val == "": + cfg["mqtt"]["iata"] = "" + return True, "OK - IATA cleared" + if len(val) != 3 or not val.isalnum() or not val.isascii(): + return False, "Error: IATA code must be exactly 3 letters/digits (e.g. DEN)" + cfg["mqtt"]["iata"] = val.upper() + return True, "OK" + + if key == "prv.key": + # write-only by design: the identity goes in, nothing reads it back + if not _hex64(val): + return False, "Error: private key must be 64 hex characters" + return True, "OK - identity restored, reboot to apply" + + if key == "mqtt.owner": + if val == "": + cfg["mqtt"]["owner"] = "" + return True, "OK - owner key cleared" + if not _hex64(val): + return False, "Error: public key must be 64 hex characters (32 bytes)" + cfg["mqtt"]["owner"] = val + return True, "OK" + + m = re.match(r"^mqtt([1-6])\.(\w+)$", key) + if m: + return apply_slot_set(cfg, int(m.group(1)) - 1, m.group(2), val) + + if key in BOOL_KEYS: + sec, f = BOOL_KEYS[key] + cfg[sec][f] = (val == "on") + return True, "OK" + if key in INT_KEYS: + sec, f = INT_KEYS[key] + try: + cfg[sec][f] = int(val) + except ValueError: + return False, "Error: expected a number" + return True, "OK" + if key in FLOAT_KEYS: + sec, f = FLOAT_KEYS[key] + try: + cfg[sec][f] = float(val) + except ValueError: + return False, "Error: expected a number" + return True, "OK" + if key in SECRET_STR_KEYS: + sec, f = SECRET_STR_KEYS[key] + cfg[sec][f] = val + return True, "OK" + if key in STR_KEYS: + sec, f = STR_KEYS[key] + cfg[sec][f] = val + return True, "OK" + # Strict fallthrough: this function is the single authority on what can be + # set, for the batch and the CLI alike. Accepting unknown keys here once hid + # the fact that the CLI could not reach `dutycycle` or `radio.fem.rxgain`. + # Verbatim shape from CommonCLI::handleSetCmd's fallthrough. + return False, "unknown config: %s" % key + + +# Payload-type names accepted alongside the decimal form. Mirrors +# namedPacketTypes() in src/helpers/MQTTPacketFilter.h; 12-14 are reserved +# upstream and stay reachable by number only. +PACKET_TYPE_NAMES = { + "req": 0, "response": 1, "txt_msg": 2, "ack": 3, "advert": 4, + "grp_txt": 5, "grp_data": 6, "anon_req": 7, "path": 8, "trace": 9, + "multipart": 10, "control": 11, "raw_custom": 15, +} + + +def packet_filter_mask(text): + """Canonical filter text -> bitmask, for the stats payload.""" + if text == "all": + return 0xFFFF + if text == "none": + return 0 + mask = 0 + for token in text.split(","): + mask |= 1 << int(token) + return mask + + +def canonical_packet_filter(val): + """Mirror of MQTTPacketFilter::parse + ::format. Returns None if invalid.""" + stripped = val.strip() + if stripped == "" or stripped == "all": + return "all" + if stripped == "none": + return "none" + mask = 0 + for part in stripped.split(","): + token = part.strip() + if re.fullmatch(r"[0-9]+", token): + packet_type = int(token) + if packet_type > 15: + return None + elif token in PACKET_TYPE_NAMES: + packet_type = PACKET_TYPE_NAMES[token] + else: + return None + mask |= 1 << packet_type + if mask == 0xFFFF: + return "all" + return ",".join(str(i) for i in range(16) if mask & (1 << i)) + + +def apply_slot_set(cfg, idx, field, val): + slot = cfg["mqtt"]["slots"][idx] + if field in SLOT_LEN_LIMITS and len(val) > SLOT_LEN_LIMITS[field]: + return False, "Error: %s too long (max %d chars)" % (field, SLOT_LEN_LIMITS[field]) + if field == "port": + try: + p = int(val) + except ValueError: + return False, "Error: port must be between 1 and 65535" + if not (1 <= p <= 65535): + return False, "Error: port must be between 1 and 65535" + slot["port"] = p + return True, "OK" + if field == "filter": + canonical = canonical_packet_filter(val) + if canonical is None: + return False, ("Error: filter must be all, none, or a CSV of " + "types 0-15 / names (advert,txt_msg,...)") + slot["filter"] = canonical + return True, "OK - slot %d packet types: %s" % (idx + 1, canonical) + if field in ("preset", "server", "username", "password", "token", "topic", "audience"): + slot[field] = val + if field == "token": + return True, "OK - slot %d token set" % (idx + 1) + return True, "OK" + return False, "Error: unknown slot field" + + +def is_secret_key(key): + # The serial console prints these back; the portal is reachable over the + # LAN, so it masks them in `get` replies the way /api/config already does. + return key in SECRET_STR_KEYS or bool(re.match(r"^mqtt[1-6]\.(password|token)$", key)) + + +# --------------------------------------------------------------------------- +# CLI command execution (backs /api/cli), mirroring CommonCLI enough to give +# the terminal UI realistic single- and multi-line replies. +# +# The portal's `set` batch is allowlisted (WebConfigKeys.h) because it is driven +# by form fields; the CLI is deliberately NOT, since its whole point is reaching +# the same surface the serial console reaches. Auth is the boundary — exactly +# as it is for the serial console and for remote admin over the mesh. +# --------------------------------------------------------------------------- +# MAX_BATCH in WebConfigServer.h: the CLI shares the config batch's fixed slot, +# so this is the real cap, reported to the page as status.max_cmds. +CLI_MAX_CMDS = 24 +CLI_RESULT_PAGE = 8 # WebConfigBatch::kCliResultPage +CLI_CMD_SECS = 0.25 # simulated per-command execution time +# Board::reboot() does not return, so the firmware answers `reboot` itself and +# arms the deferred reboot once results have been read (see wcIsDeferredReboot). +CLI_DEFERRED_REBOOT = "reboot" # matched as a PREFIX, like CommonCLI does + +# Commands the CLI reaches but the portal cannot honestly serve; rejected at +# POST. Mirrors wcCliUnavailable() in WebConfigServer.cpp. +CLI_UNAVAILABLE = [ + ("start ota", True, "start ota needs port 80, which this portal is using. " + "Run it from the serial console, or use `ota update`."), + ("clock sync", True, "clock sync takes its time from the caller, which a web request " + "has no way to supply. Use `time ` instead."), + ("log", False, "log writes the packet log to the serial console, not here, and " + "blocks the radio while it does. Use `log start` / `log stop`."), + ("get acl", False, "get acl writes to the serial console, not here."), +] + + +def cli_unavailable(cmd): + for token, is_prefix, why in CLI_UNAVAILABLE: + if cmd.startswith(token) if is_prefix else cmd == token: + return why + return None + + +# Failure replies CommonCLI emits that do NOT start with "Err" — the shapes that +# made a naive prefix test call them success. Mirrors +# WebConfigBatch::cliReplyIsFailure. +def cli_reads_secret(cmd): + """Commands that READ a secret. CommonCLI gates these on the caller being + the serial console; the portal is not, so the value is masked here the way + CommonCLI masks it for remote callers. Mirrors wcCliReadsSecret().""" + if not cmd.startswith("get "): + return False + key = cmd[4:].strip() + return key in ("prv.key", "guest.password", "alert.psk", "bridge.secret") \ + or is_secret_key(key) + + +def cli_reply_is_failure(reply): + if not reply: + return False + if reply.startswith(("Err", "ERR", "err", "(ERR", "Unknown command", + "unknown config", "??", "Can't find")): + return True + return ": Err" in reply + +# Commands the device answers but that have no config-key equivalent. +GETTERS = { + "freq": lambda c: "%.3f" % c["radio"]["freq"], + "bw": lambda c: "%.2f" % c["radio"]["bw"], + "sf": lambda c: str(c["radio"]["sf"]), + "cr": lambda c: str(c["radio"]["cr"]), + "public.key": lambda c: "a1b2c3d4" * 8, + "wifi.status": lambda c: ( + "SSID: %s\nIP: 192.168.1.42\nRSSI: -58 dBm\nUptime: %dm" + % (c["wifi"]["ssid"] or "(not set)", int(time.time() - ST.start) // 60)), + "mqtt.status": lambda c: cli_mqtt_status(c), + "mqtt.presets": lambda c: "\n".join( + "%2d. %s%s" % (i + 1, n, "" if nd == "none" else " (needs %s)" % nd) + for i, (n, nd) in enumerate(PRESETS)), + "role": lambda c: "Repeater", + "acl": lambda c: "a1b2c3d4e5f60718 perms 3\n1122334455667788 perms 1", + # not its own pref: the CLI derives it from airtime_factor both ways + "dutycycle": lambda c: "%.1f" % (100.0 / (c["radio"]["af"] + 1)), + "mqtt.config.valid": lambda c: ( + "yes" if any(s["preset"] != "none" for s in c["mqtt"]["slots"]) else "no - no slot configured"), + "mqtt.ntp.diag": lambda c: "last sync: 42s ago via %s (offset +0.011s)" % (c["mqtt"]["ntp"] or "none"), + "mqtt.stats": lambda c: ("published: %d\ndropped: 0\nqueue: 0/24\nreconnects: 1" + % (100 + int(time.time() - ST.start))), + # Runtime-gated on the real device (Board::canControlLoRaFemLna), not + # compiled out — the command exists everywhere and the board answers for + # itself. The mock board is a Heltec V3, which has no FEM. + "radio.fem.rxgain": lambda c: None, +} + + +def cli_mqtt_status(cfg): + lines = [] + for i, s in enumerate(cfg["mqtt"]["slots"][:ST.active_slots]): + if s["preset"] == "none": + lines.append("slot %d: unconfigured" % (i + 1)) + else: + lines.append("slot %d: %-16s connected tx=%d err=0" + % (i + 1, s["preset"], 100 + int(time.time() - ST.start))) + return "\n".join(lines) + + +def cli_get(cfg, key): + """Reply to `get `. + + CommonCLI::handleGetCmd answers `> value` — the marker sets the value apart + on the serial console. Reproduced here because it is load-bearing: a reply + that starts with "> " does not start with "OK", which is what made the + firmware's first cut mark every getter a failure. + """ + ok, val = _cli_get_value(cfg, key) + return (ok, "> " + val) if ok else (ok, val) + + +def _cli_get_value(cfg, key): + if key in GETTERS: + val = GETTERS[key](cfg) + return (True, val) if val is not None else (False, "Error: unsupported") + if is_secret_key(key): + # The serial console prints these; the portal is reachable over the LAN, + # so it masks them the same way /api/config does. + return True, SENTINEL if cli_read_key(cfg, key) else "(not set)" + val = cli_read_key(cfg, key) + if val is None: + return False, "??: %s" % key # CommonCLI::handleGetCmd fallthrough + return True, str(val) + + +def cli_read_key(cfg, key): + """Current value of a `set` key, or None when the key is unknown.""" + m = re.match(r"^mqtt([1-6])\.(\w+)$", key) + if m: + slot = cfg["mqtt"]["slots"][int(m.group(1)) - 1] + return slot.get(m.group(2)) + for table in (BOOL_KEYS, INT_KEYS, FLOAT_KEYS, STR_KEYS, SECRET_STR_KEYS): + if key in table: + sec, f = table[key] + v = cfg[sec][f] + return ("on" if v else "off") if key in BOOL_KEYS else v + # keys apply_set() special-cases, so they appear in none of the tables above + r = cfg["radio"] + return { + "name": r["name"], "lat": r["lat"], "lon": r["lon"], + "radio": "%.3f,%.2f,%d,%d" % (r["freq"], r["bw"], r["sf"], r["cr"]), + "bw": r["bw"], "sf": r["sf"], "cr": r["cr"], + "mqtt.iata": cfg["mqtt"]["iata"], "mqtt.owner": cfg["mqtt"]["owner"], + }.get(key) + + +def run_cli(cfg, line): + """Execute one command line. Returns (ok, reply); reply may be multi-line.""" + cmd = line.strip() + if cmd == "": + return True, "" + if cmd == "ver": + # Same source as /api/status's fw on the device: both are + # FIRMWARE_VERSION, so they must not disagree here either. + return True, "%s (Build: 6 Jun 2026)" % FW_VERSION + if cmd == "board": + return True, "Heltec V3 (mock)" + if cmd == "clock": + return True, time.strftime("%d/%m/%Y %H:%M:%S", time.gmtime()) + " UTC" + if cmd == "advert": + return True, "OK - Advert sent (zero hop)" + if cmd == "advert.zerohop": + return True, "OK - Advert sent (zero hop)" + if cmd in ("reboot", "clkreboot"): + return True, "OK - rebooting" + if cmd in ("poweroff", "shutdown"): + return True, "OK - powering off" + if cmd == "erase": + return True, "File system erase: OK" + if cmd == "memory": + return True, ("heap free: 142000\nheap min: 118000\n" + "largest block: 96000\npsram free: 3980000") + if cmd == "neighbors": + return True, ("d4e5f60718 -71 dBm snr 9.5 2m ago\n" + "1122334455 -94 dBm snr 2.0 14m ago") + # Handled by MyMesh::handleCommand before it delegates to CommonCLI. + if cmd == "discover.neighbors": + return True, "OK - Discover sent" + if cmd == "discover.scopes": + return True, "OK - scopes queued (18s discovery remaining)" + if cmd.startswith("setperm "): + parts = cmd[8:].split() + if len(parts) != 2 or not _hex64(parts[0]): + return False, "Err - bad params" + return True, "OK" + if cmd.startswith("clock sync"): + # Rejected at POST, but modelled anyway: over the web the caller's + # timestamp is 0, so CommonCLI always takes this branch. + return False, "(ERR: clock cannot go backwards)" + if cmd == "region": + return True, "US915" + if cmd == "sensor list": + return True, "0: battery (mV)\n1: temperature (C)\n2: humidity (%)" + if cmd.startswith("sensor get "): + return True, "> 22.4" + if cmd.startswith("sensor set "): + return True, "OK" + if cmd.startswith("gps advert "): + mode = cmd[11:] + if mode not in ("none", "share", "prefs"): + return False, "Error, must be none, share or prefs" + return True, "OK - advert position: %s" % mode + if cmd in ("gps on", "gps off"): + return True, "OK - GPS %s" % cmd[4:] + if cmd == "gps sync": + return True, "OK - clock and location set from GPS" + if cmd == "gps setloc": + return True, "OK - lat/lon set from the current fix" + if cmd == "gps": + return True, "GPS: no fix (0 satellites)" + if cmd in ("powersaving on", "powersaving off"): + return True, "OK - power saving %s" % cmd[12:] + if cmd == "powersaving": + return True, "off" + if cmd.startswith("alert test"): + if not ST.cfg["cli"]["alert.psk"]: + return False, "Error: alert channel not configured (set alert.psk or set alert.hashtag)" + return True, "OK - test alert sent" + if cmd.startswith("ota "): + return True, ("v1.7.2 available (current v1.7.1-mock)" if cmd == "ota check" + else "OK - downloading v1.7.2, will reboot when flashed") + if cmd.startswith("start webconfig"): + return True, "OK - already running (you are using it)" + if cmd == "stop webconfig": + return True, "OK - portal stopping" + if cmd == "start ota": + return True, "OK - upload AP raised at 192.168.4.1" + if cmd.startswith("neighbor.remove "): + return (True, "OK") if _hex64(cmd[16:]) else (False, "ERR: bad pubkey") + if cmd.startswith("tempradio "): + return True, "OK - temporary radio params applied (not saved)" + if cmd == "clear stats": + return True, "OK - stats cleared" + if cmd.startswith("stats-"): + return True, "recv=512 sent=88 rx_err=3 airtime=41s" + if cmd == "log": + return True, "packet log: 128 entries, 14 KB" + if cmd.startswith("log "): + return True, "OK" + if cmd.startswith("password "): + global ADMIN_PASSWORD + ADMIN_PASSWORD = cmd[9:] + return True, "OK - password changed" + if cmd.startswith("time "): + return True, "OK - clock set" + if cmd.startswith("get "): + return cli_get(cfg, cmd[4:].strip()) + if cmd.startswith("set "): + rest = cmd[4:].strip() + key, _, val = rest.partition(" ") + if not key: + return False, "Error: set what?" + # apply_set owns the "is this settable" decision; gating on whether the + # key is *readable* rejected write-only and computed ones (`dutycycle`, + # `prv.key`, `radio.fem.rxgain`). + return apply_set(cfg, key, val.strip()) + return False, "Unknown command" + + +def valid_reqid(reqid): + return isinstance(reqid, str) and bool(re.fullmatch(r"[0-9A-Fa-f]{16}", reqid)) + + +# --------------------------------------------------------------------------- +# HTTP handler +# --------------------------------------------------------------------------- +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, fmt, *args): # concise one-line log + print(" %s %s" % (self.command, self.path)) + + # -- helpers -- + def _json(self, code, obj, extra_headers=None): + body = json.dumps(obj).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + for k, v in (extra_headers or {}): + self.send_header(k, v) + self.end_headers() + self.wfile.write(body) + + def _read_body(self): + n = int(self.headers.get("Content-Length", 0)) + return self.rfile.read(n) if n else b"" + + def _need_auth(self): + if not ST.is_authed(self.headers): + self._json(401, {"error": "auth"}) + return True + return False + + # -- GET -- + def do_GET(self): + path = self.path.split("?", 1)[0] + if path == "/": + return self._serve_index() + if path == "/api/status": + return self._json(200, ST.status_json(ST.is_authed(self.headers))) + if path == "/api/presets": + return self._json(200, {"presets": [{"name": n, "needs": nd} for n, nd in PRESETS]}) + if path == "/api/config": + if self._need_auth(): + return + with ST.lock: + return self._json(200, ST.config_json()) + if path == "/api/config/result": + if self._need_auth(): + return + return self._config_result() + if path == "/api/cli/result": + if self._need_auth(): + return + return self._cli_result() + if path == "/api/stats": + if self._need_auth(): + return + return self._json(200, self._stats()) + if path == "/api/scan": + if self._need_auth(): + return + return self._scan() + return self._json(404, {"error": "not found"}) + + # -- POST -- + def do_POST(self): + path = self.path.split("?", 1)[0] + if path == "/api/login": + return self._login() + if path == "/api/logout": + ST.session = None + return self._json(200, {"ok": True}, [("Set-Cookie", "wcs=; Max-Age=0; Path=/")]) + if path == "/api/config": + if self._need_auth(): + return + return self._config_post() + if path == "/api/cli": + if self._need_auth(): + return + return self._cli_post() + if path == "/api/reboot": + if self._need_auth(): + return + return self._json(200, {"ok": True}) + if path == "/api/portal/exit": + return self._json(200, {"ok": True, "url": "http://localhost:%d/" % PORT}) + return self._json(404, {"error": "not found"}) + + # -- endpoint impls -- + def _serve_index(self): + try: + with open(INDEX_HTML, "rb") as f: # re-read each time -> live edits + html = f.read() + except OSError: + self.send_error(500, "webui/index.html not found") + return + if MINIFY: + # Serve what the device actually serves. The generator strips + # comments and indentation before compressing, so --minify is how + # you exercise those bytes in a browser rather than trusting that + # stripping a 100 KB page never changes its behaviour. + html = strip_source(html.decode("utf-8")).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(html))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(html) + + def _login(self): + if ST.setup_mode: + return self._json(200, {"ok": True}) + try: + body = json.loads(self._read_body() or b"{}") + except ValueError: + return self._json(400, {"error": "bad request"}) + if body.get("password") != ADMIN_PASSWORD: + return self._json(401, {"error": "wrong password"}) + ST.session = secrets.token_hex(16) + return self._json(200, {"ok": True}, + [("Set-Cookie", "wcs=%s; HttpOnly; SameSite=Lax; Path=/" % ST.session)]) + + def _config_post(self): + raw = self._read_body() + if len(raw) > 4096: + return self._json(413, {"error": "body too large"}) + try: + body = json.loads(raw or b"{}") + except ValueError: + return self._json(400, {"error": "bad json"}) + reqid = body.get("reqid", "") + if not valid_reqid(reqid): + return self._json(400, {"error": "bad reqid"}) + reboot = bool(body.get("reboot", False)) + setmap = body.get("set", {}) or {} + + # `password` maps to the top-level CLI command rather than a setter. It + # is accepted in both modes (LAN already required a login), but first + # onboarding cannot finish without it. + if "password" in setmap: + pwd = str(setmap["password"]) + if not 0 < len(pwd) <= 15 or "\r" in pwd or "\n" in pwd: + return self._json(400, {"error": "admin password must be 1-15 characters with no line breaks"}) + elif ST.setup_mode and ST.initial_setup and (reboot or "wifi.ssid" in setmap): + return self._json(400, {"error": "admin password required for initial setup"}) + + with ST.lock: + if ST.batch.get("state") != "idle" and ST.batch.get("reqid") == reqid: + return self._json(202, { + "state": ST.batch["state"], "count": len(ST.batch.get("results", [])), + "reqid": reqid, + }) + if ST.batch.get("state") == "pending": + return self._json(409, {"error": "busy", "reqid": ST.batch.get("reqid", "")}) + # drop unchanged secrets (sentinel), like the firmware does + entries = [(k, v) for k, v in setmap.items() + if not (is_secret_key(k) and v == SENTINEL)] + # Same command-budget rejection the firmware applies while building + # BatchEntry::cmd; CR/LF are stripped there and don't count. + for k, v in entries: + prefix = "password " if k == "password" else "set %s " % k + stripped = str(v).replace("\r", "").replace("\n", "") + if len(prefix) + len(stripped) > BATCH_CMD_SIZE - 1: + return self._json(400, {"error": "value too long", "key": k[:32]}) + if not entries and not reboot: + return self._json(400, {"error": "no changes"}) + # apply now, but expose as pending->done to exercise polling + results, all_ok = [], True + for k, v in entries: + ok, reply = apply_set(ST.cfg, k, str(v)) + if not ok: + all_ok = False + results.append({"key": k, "reply": reply}) + ST.batch = {"state": "pending", "reqid": reqid, "results": results, + "all_ok": all_ok, "reboot": reboot, + "done_at": time.time() + BATCH_PENDING_SECS} + return self._json(202, {"state": "pending", "count": len(entries), "reqid": reqid}) + + def _config_result(self): + query = parse_qs(urlsplit(self.path).query) + reqid = query.get("reqid", [""])[0] + if not valid_reqid(reqid): + return self._json(400, {"error": "bad reqid"}) + with ST.lock: + b = ST.batch + if b.get("state") == "idle": + return self._json(200, {"state": "idle", "reqid": reqid}) + if b.get("reqid") != reqid: + return self._json(404, {"error": "unknown request"}) + if b["state"] == "pending" and time.time() < b["done_at"]: + return self._json(200, {"state": "pending", "reqid": b["reqid"]}) + b["state"] = "done" # stays readable until next POST + return self._json(200, { + "state": "done", "reqid": b["reqid"], "all_ok": b["all_ok"], + "reboot": b["reboot"] and b["all_ok"], "results": b["results"], + }) + + # ---- CLI --------------------------------------------------------------- + # Same 202 + reqid + poll shape as /api/config, for the same reason: the + # commands have to run on the main loop, not the web server's task. The + # difference is that results stream -- a pasted sequence fills the terminal + # command by command instead of appearing all at once at the end. + def _cli_post(self): + raw = self._read_body() + if len(raw) > 8192: + return self._json(413, {"error": "body too large"}) + try: + body = json.loads(raw or b"{}") + except ValueError: + return self._json(400, {"error": "bad json"}) + reqid = body.get("reqid", "") + if not valid_reqid(reqid): + return self._json(400, {"error": "bad reqid"}) + cmds = body.get("cmds") + if not isinstance(cmds, list) or not cmds: + return self._json(400, {"error": "no commands"}) + if len(cmds) > CLI_MAX_CMDS: + return self._json(413, {"error": "too many commands", "max": CLI_MAX_CMDS}) + cmds = [str(c).replace("\r", "").replace("\n", "").strip() for c in cmds] + cmds = [c for c in cmds if c] + if not cmds: + return self._json(400, {"error": "no commands"}) + for c in cmds: + if len(c) > BATCH_CMD_SIZE - 1: + return self._json(400, {"error": "command too long"}) + why = cli_unavailable(c) + if why: + return self._json(400, {"error": why}) + # Same invariant handleConfigPost enforces (see wcCliUnavailable's + # neighbour in WebConfigServer.cpp): first onboarding is committed by the + # reboot, and must not commit the factory password onto someone's LAN. + if (ST.setup_mode and ST.initial_setup and not ST.admin_pwd_set + and not any(c.startswith("password ") for c in cmds) + and (any(c.startswith(CLI_DEFERRED_REBOOT) for c in cmds) + or any(c.startswith("set wifi.ssid ") for c in cmds))): + return self._json(400, {"error": "admin password required for initial setup — " + "run `password ` first"}) + + with ST.lock: + self._cli_advance(ST.cli) + if ST.cli.get("state") != "idle" and ST.cli.get("reqid") == reqid: + return self._json(202, {"state": ST.cli["state"], "reqid": reqid, + "total": len(ST.cli["cmds"])}) + if ST.cli.get("state") == "running": + return self._json(409, {"error": "busy", "reqid": ST.cli.get("reqid", "")}) + ST.cli = {"state": "running", "reqid": reqid, "cmds": cmds, "results": [], + "all_ok": True, + "reboot": any(c.startswith(CLI_DEFERRED_REBOOT) for c in cmds), + "next_at": time.time() + CLI_CMD_SECS} + return self._json(202, {"state": "running", "reqid": reqid, "total": len(cmds)}) + + @staticmethod + def _cli_advance(job): + """Run whichever queued commands are now due. Execution belongs to the + node's loop, not to the client's polling — a client that walks away must + not leave the executor claimed forever.""" + now = time.time() + while (job.get("state") == "running" and len(job["results"]) < len(job["cmds"]) + and now >= job["next_at"]): + cmd = job["cmds"][len(job["results"])] + if cmd.startswith(CLI_DEFERRED_REBOOT): + reply = "OK - reboot queued" + else: + _, reply = run_cli(ST.cfg, cmd) + if cmd.startswith("password "): + reply = "OK" # never echo the new password back + ST.admin_pwd_set = True + elif cli_reads_secret(cmd): + val = reply[2:] if reply.startswith("> ") else reply + reply = ("> (not set)" if val in ("", "(not set)") + else "> ******** (serial only)") + ok = not cli_reply_is_failure(reply) + # Only writes gate the reboot, and only on the "OK" convention every + # setter keeps (WebConfigBatch::cliReplyGatesReboot). + if cmd.startswith(("set ", "password ")): + job["all_ok"] = job.get("all_ok", True) and reply.startswith("OK") + # The command is NOT echoed: it may carry a password or token, and + # the client matches results to its own sequence by index. + job["results"].append({"ok": ok, "reply": reply}) + job["next_at"] = now + CLI_CMD_SECS + if job.get("state") == "running" and len(job["results"]) == len(job["cmds"]): + job["state"] = "done" # stays readable until the next POST + + def _cli_result(self): + query = parse_qs(urlsplit(self.path).query) + reqid = query.get("reqid", [""])[0] + if not valid_reqid(reqid): + return self._json(400, {"error": "bad reqid"}) + # `from` lets the client ask only for results it has not rendered yet, + # so a long sequence isn't re-sent on every poll. + try: + frm = max(0, int(query.get("from", ["0"])[0])) + except ValueError: + frm = 0 + with ST.lock: + j = ST.cli + if j.get("state") == "idle": + return self._json(200, {"state": "idle", "reqid": reqid}) + if j.get("reqid") != reqid: + return self._json(404, {"error": "unknown request"}) + self._cli_advance(j) # one command per CLI_CMD_SECS + # Results stream, capped per read so the device's JSON document + # stays small; a longer sequence pages across reads. "done" means + # the client has been handed everything, not just that execution + # finished — a client that stops polling at "done" must lose nothing. + page = j["results"][frm:frm + CLI_RESULT_PAGE] + final = j["state"] == "done" and frm + len(page) >= len(j["cmds"]) + body = {"state": "done" if final else "running", "reqid": reqid, + "total": len(j["cmds"]), "from": frm, "results": page} + if final: + body["all_ok"] = j["all_ok"] + body["reboot"] = j["reboot"] and j["all_ok"] + if j["reboot"] and not j["all_ok"]: + body["reboot_withheld"] = True + return self._json(200, body) + + def _scan(self): + rescan = "rescan=1" in self.path + now = time.time() + if rescan or ST.scan_started is None: + ST.scan_started = now + return self._json(200, {"state": "scanning"}) + if now - ST.scan_started < SCAN_SECS: + return self._json(200, {"state": "scanning"}) + return self._json(200, {"state": "done", "networks": SCAN_NETWORKS}) + + def _stats(self): + up = int(time.time() - ST.start) + slots = [] + for i, s in enumerate(ST.cfg["mqtt"]["slots"]): + if s["preset"] == "none": + continue + row = {"n": i + 1, "name": s["preset"], "state": "ok", + "ok": 100 + up, "err": 0} + # Mirrors buildStatsJson(): "filt" carries the raw mask and is + # omitted entirely for the all-types default. + mask = packet_filter_mask(s.get("filter", "all")) + if mask != 0xFFFF: + row["filt"] = mask + slots.append(row) + return { + "uptime_s": up, "batt_mv": 4020, "heap_free": 142000, "heap_min": 118000, + "heap_max_alloc": 96000, "noise": -98, "rssi": -71, "snr": 9.5, + "airtime_s": up // 20, "rx_airtime_s": up // 8, "recv": 512 + up, + "sent": 88 + up // 3, "rx_err": 3, "sent_flood": 40, "sent_direct": 48, + "recv_flood": 300, "recv_direct": 212, "tx_queue": 0, "mqtt_queue": 0, + "wifi_rssi": -58, "ip": "192.168.1.42", "slots": slots, + } + + +def main(): + global ST, PORT, MINIFY, FW_VERSION + ap = argparse.ArgumentParser(description="Mock WebConfig portal backend") + ap.add_argument("--port", type=int, default=8080) + ap.add_argument("--setup", action="store_true", help="first-boot setup wizard mode") + ap.add_argument("--active-slots", type=int, default=5, help="server slots to expose (2 or 5)") + ap.add_argument("--fw-version", default=FW_VERSION, + help="version string to report, shaped like build.sh's embedded one") + ap.add_argument("--minify", action="store_true", + help="serve the comment-stripped page the device ships, not the source") + args = ap.parse_args() + ST, PORT, MINIFY = State(args), args.port, args.minify + FW_VERSION = args.fw_version + + srv = ThreadingHTTPServer(("127.0.0.1", args.port), Handler) + mode = "SETUP (wizard)" if args.setup else "LAN (login: %s)" % ADMIN_PASSWORD + print("WebConfig mock backend — %s%s" % (mode, " [minified]" if MINIFY else "")) + print(" open http://localhost:%d/ (Ctrl-C to stop)" % args.port) + try: + srv.serve_forever() + except KeyboardInterrupt: + print("\nstopped") + + +if __name__ == "__main__": + main() diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index c0610b7f8a..2a491a610d 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -63,6 +63,12 @@ uint32_t Dispatcher::getCADFailMaxDuration() const { return 4000; // 4 seconds } +#ifdef WITH_MQTT_BRIDGE +uint32_t Dispatcher::getRadioWatchdogMillis() const { + return RADIO_WATCHDOG_MS; +} +#endif + void Dispatcher::loop() { if (millisHasNowPassed(next_floor_calib_time)) { _radio->triggerNoiseFloorCalibrate(getInterferenceThreshold()); @@ -83,6 +89,32 @@ void Dispatcher::loop() { _err_flags |= ERR_EVENT_STARTRX_TIMEOUT; } + // Radio watchdog: detect radio stuck in RX mode but not receiving any packets. + // Observer-only feature (gated behind WITH_MQTT_BRIDGE); configured via the + // MQTTPrefs radio_watchdog_minutes setting. +#ifdef WITH_MQTT_BRIDGE + { + const uint32_t watchdog_ms = getRadioWatchdogMillis(); + if (watchdog_ms > 0) { + unsigned long last_recv = _radio->getLastRecvMillis(); + unsigned long last_irq = _radio->getLastRadioInterruptMillis(); + unsigned long last_active = (last_recv > last_irq ? last_recv : last_irq); + if (last_radio_active_ms > last_active) last_active = last_radio_active_ms; + if (is_recv && last_active > 0) { + unsigned long silent_ms = _ms->getMillis() - last_active; + unsigned long since_recovery = _ms->getMillis() - last_watchdog_recovery; + if (silent_ms > watchdog_ms && since_recovery > watchdog_ms) { + _err_flags |= ERR_EVENT_RADIO_WATCHDOG; + MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", silent_ms, _radio->getRadioState()); + _radio->idle(); + _radio->startRecv(); + last_watchdog_recovery = _ms->getMillis(); + } + } + } + } +#endif // WITH_MQTT_BRIDGE (radio watchdog) + if (outbound) { // waiting for outbound send to be completed if (_radio->isSendComplete()) { long t = _ms->getMillis() - outbound_start; @@ -106,6 +138,7 @@ void Dispatcher::loop() { } _radio->onSendFinished(); + last_radio_active_ms = _ms->getMillis(); // TX success → radio is alive logTx(outbound, 2 + outbound->getPathByteLen() + outbound->payload_len); if (outbound->isRouteFlood()) { n_sent_flood++; diff --git a/src/Dispatcher.h b/src/Dispatcher.h index aad6cba3ec..7edb7c7fe5 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -61,6 +61,9 @@ class Radio { */ virtual void loop() { } + virtual void idle() { } + virtual void startRecv() { } + virtual int getNoiseFloor() const { return 0; } virtual void triggerNoiseFloorCalibrate(int threshold) { } @@ -69,6 +72,10 @@ class Radio { virtual void resetAGC() { } + virtual uint8_t getRadioState() const { return 0; } + virtual unsigned long getLastRecvMillis() const { return 0; } + virtual unsigned long getLastRadioInterruptMillis() const { return 0; } + virtual bool isInRecvMode() const = 0; /** @@ -78,6 +85,11 @@ class Radio { virtual float getLastRSSI() const { return 0; } virtual float getLastSNR() const { return 0; } + + /** + * \returns number of receive errors (e.g. CRC failures) since last reset; 0 if not tracked. + */ + virtual uint32_t getPacketsRecvErrors() const { return 0; } }; /** @@ -110,6 +122,11 @@ typedef uint32_t DispatcherAction; #define ERR_EVENT_FULL (1 << 0) #define ERR_EVENT_CAD_TIMEOUT (1 << 1) #define ERR_EVENT_STARTRX_TIMEOUT (1 << 2) +#define ERR_EVENT_RADIO_WATCHDOG (1 << 3) + +#ifndef RADIO_WATCHDOG_MS + #define RADIO_WATCHDOG_MS 300000 // 5 minutes +#endif /** * \brief The low-level task that manages detecting incoming Packets, and the queueing @@ -118,6 +135,8 @@ typedef uint32_t DispatcherAction; class Dispatcher { Packet* outbound; // current outbound packet unsigned long outbound_expiry, outbound_start, total_air_time, rx_air_time; + unsigned long last_watchdog_recovery; + unsigned long last_radio_active_ms; // updated on any TX or RX event; used by watchdog unsigned long next_tx_time; unsigned long cad_busy_start; unsigned long radio_nonrx_start; @@ -152,6 +171,8 @@ class Dispatcher { tx_budget_ms = 0; last_budget_update = 0; duty_cycle_window_ms = 3600000; + last_watchdog_recovery = 0; + last_radio_active_ms = 0; } virtual DispatcherAction onRecvPacket(Packet* pkt) = 0; @@ -171,6 +192,9 @@ class Dispatcher { virtual bool getCADEnabled() const { return false; } // hardware CAD disabled by default virtual int getAGCResetInterval() const { return 0; } // disabled by default virtual unsigned long getDutyCycleWindowMs() const { return 3600000; } +#ifdef WITH_MQTT_BRIDGE + virtual uint32_t getRadioWatchdogMillis() const; // observer-only radio recovery +#endif public: void begin(); @@ -187,6 +211,9 @@ class Dispatcher { uint32_t getNumSentDirect() const { return n_sent_direct; } uint32_t getNumRecvFlood() const { return n_recv_flood; } uint32_t getNumRecvDirect() const { return n_recv_direct; } + uint16_t getErrFlags() const { return _err_flags; } // Get error flags + bool hasOutbound() const { return outbound != NULL; } + bool isCurrentOutbound(const Packet* packet) const { return outbound == packet; } void resetStats() { n_sent_flood = n_sent_direct = n_recv_flood = n_recv_direct = 0; _err_flags = 0; diff --git a/src/MeshCore.h b/src/MeshCore.h index e67371ef17..15ccdc1562 100644 --- a/src/MeshCore.h +++ b/src/MeshCore.h @@ -24,15 +24,15 @@ #if MESH_DEBUG && ARDUINO #include - #define MESH_DEBUG_PRINT(F, ...) Serial.printf("DEBUG: " F, ##__VA_ARGS__) - #define MESH_DEBUG_PRINTLN(F, ...) Serial.printf("DEBUG: " F "\n", ##__VA_ARGS__) + #define MESH_DEBUG_PRINT(F, ...) do { if (Serial.availableForWrite() > 0) { Serial.printf("DEBUG: " F, ##__VA_ARGS__); } } while(0) + #define MESH_DEBUG_PRINTLN(F, ...) do { if (Serial.availableForWrite() > 0) { Serial.printf("DEBUG: " F "\n", ##__VA_ARGS__); } } while(0) #else #define MESH_DEBUG_PRINT(...) {} #define MESH_DEBUG_PRINTLN(...) {} #endif #if BRIDGE_DEBUG && ARDUINO -#define BRIDGE_DEBUG_PRINTLN(F, ...) Serial.printf("%s BRIDGE: " F, getLogDateTime(), ##__VA_ARGS__) +#define BRIDGE_DEBUG_PRINTLN(F, ...) do { if (Serial.availableForWrite() > 0) { Serial.printf("%s BRIDGE: " F, getLogDateTime(), ##__VA_ARGS__); } } while(0) #else #define BRIDGE_DEBUG_PRINTLN(...) {} #endif @@ -63,7 +63,14 @@ class MainBoard { virtual void setGpio(uint32_t values) {} virtual uint8_t getStartupReason() const = 0; virtual bool getBootloaderVersion(char* version, size_t max_len) { return false; } - virtual bool startOTAUpdate(const char* id, char reply[]) { return false; } // not supported + virtual bool startOTAUpdate(const char* id, char reply[], bool force_ap = false) { return false; } // not supported + // Pull-based OTA: fetch the firmware build for this variant from a baked-in manifest and flash it. + // current_ver is the running firmware version string (used to skip if already up to date); when + // dry_run is true the build is only reported, not flashed. Observer (ESP32+WiFi) builds only. + virtual bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { return false; } + + // LoRa front-end-module LNA (RX gain) control. Only FEM-equipped boards override + // these; others report they can't control it. Driven by NodePrefs.radio_fem_rxgain. virtual bool setLoRaFemLnaEnabled(bool enable) { return false; } virtual bool canControlLoRaFemLna() const { return false; } virtual bool isLoRaFemLnaEnabled() const { return false; } diff --git a/src/helpers/AlertReporter.cpp b/src/helpers/AlertReporter.cpp new file mode 100644 index 0000000000..4bce1fd082 --- /dev/null +++ b/src/helpers/AlertReporter.cpp @@ -0,0 +1,322 @@ +#include "AlertReporter.h" + +#include +#include +#include +#include + +// Header layout for PAYLOAD_TYPE_GRP_TXT before encryption: +// [0..3] timestamp (uint32_t LE) — also helps make packet_hash unique +// [4] TXT_TYPE_PLAIN +// [5..] ": " (null-terminated by sender for legacy parsers) +#ifndef MAX_ALERT_TEXT_LEN +// Conservative ceiling: matches BaseChatMesh::MAX_TEXT_LEN (10 * 16 = 160) and +// stays under MAX_PACKET_PAYLOAD - 4(timestamp) - 1(type) - CIPHER_MAC_SIZE - 1. +#define MAX_ALERT_TEXT_LEN 160 +#endif + +#ifndef ALERT_TXT_TYPE_PLAIN +#define ALERT_TXT_TYPE_PLAIN 0 +#endif + +#ifdef MQTT_DEBUG +#include +#define ALERT_DEBUG_PRINTLN(...) Serial.printf("Alert: " __VA_ARGS__); Serial.println() +#else +#define ALERT_DEBUG_PRINTLN(...) do {} while (0) +#endif + +#ifdef WITH_MQTT_BRIDGE +AlertReporter::AlertReporter() + : _prefs(nullptr), _obs(nullptr), _mesh(nullptr), _callbacks(nullptr), +#ifdef WITH_MQTT_BRIDGE + _bridge(nullptr), +#endif + _next_check_ms(0) { +#ifdef WITH_MQTT_BRIDGE + memset(&_wifi, 0, sizeof(_wifi)); + memset(&_mqtt, 0, sizeof(_mqtt)); +#endif +} + +void AlertReporter::begin(NodePrefs* prefs, MQTTPrefs* obs, mesh::Mesh* mesh, CommonCLICallbacks* callbacks) { + _prefs = prefs; + _obs = obs; + _mesh = mesh; + _callbacks = callbacks; + onConfigChanged(); +} + +void AlertReporter::setBridge(MQTTBridge* bridge) { + _bridge = bridge; +} +#endif // WITH_MQTT_BRIDGE (AlertReporter methods, part 1) + +// Channels banned as fault-alert destinations. Fault alerts are noisy +// operator-infrastructure messages; routing them to community channels would +// flood every nearby companion app (and amplify via well-known auto-responder +// bots), so the firmware refuses these keys at both CLI set-time and at +// runtime in resolveChannel. +// +// Provenance for each row can be re-derived with: +// printf '#name' | openssl dgst -sha256 | cut -c1-32 +// or for the Public PSK: +// echo 'izOH6cXN6mrJ5e26oRXNcg==' | base64 -d | xxd -p -c 16 +// +// To ban an additional channel: append one new row; no other code changes +// required. Both the table entries and `alert_psk_hex` are 32 lowercase hex +// chars (16-byte secret), so the matcher is a direct strcmp. +struct BannedAlertChannel { + const char* label; + const char* secret_hex; // 32 lowercase hex chars (no 0x, no separators) +}; + +static const BannedAlertChannel BANNED_ALERT_CHANNELS[] = { + // Public group PSK ("izOH6cXN6mrJ5e26oRXNcg==") + { "PUBLIC", "8b3387e9c5cdea6ac9e5edbaa115cd72" }, + // sha256("#test")[0..15] — auto-responders in many regions + { "#test", "9cd8fcf22a47333b591d96a2b848b73f" }, + // sha256("#bot")[0..15] — generic bot channel, frequent auto-responders + { "#bot", "eb50a1bcb3e4e5d7bf69a57c9dada211" }, +}; + +const char* alertReporterBannedChannelMatch(const uint8_t* secret16) { + char hex[33]; + mesh::Utils::toHex(hex, secret16, 16); + for (size_t i = 0; i < sizeof(BANNED_ALERT_CHANNELS) / sizeof(BANNED_ALERT_CHANNELS[0]); i++) { + if (strcmp(hex, BANNED_ALERT_CHANNELS[i].secret_hex) == 0) { + return BANNED_ALERT_CHANNELS[i].label; + } + } + return nullptr; +} + +const char* alertReporterBannedChannelMatchHex(const char* psk_hex) { + if (!psk_hex || strlen(psk_hex) != 32) return nullptr; + uint8_t secret[16]; + if (!mesh::Utils::fromHex(secret, 16, psk_hex)) return nullptr; + return alertReporterBannedChannelMatch(secret); +} + +#ifdef WITH_MQTT_BRIDGE +bool AlertReporter::resolveChannel(mesh::GroupChannel& out) const { + if (!_prefs) return false; + + // alert_psk_hex is the single source of truth — `set alert.hashtag` + // pre-derives the hex-encoded PSK from sha256("#name")[0..15] at CLI time. + // Only 16-byte secrets (32 hex chars) are supported; 32-byte channel keys + // are not used anywhere in MeshCore practice and not represented in the + // banned table either. + const char* psk = _obs->alert_psk_hex; + if (strlen(psk) != 32) return false; + + memset(out.secret, 0, sizeof(out.secret)); + if (!mesh::Utils::fromHex(out.secret, 16, psk)) return false; + + // Belt-and-suspenders against an operator pasting a banned PSK directly + // into alert.psk, or a hashtag whose hash somehow collides with one of the + // banned 16-byte secrets (astronomically improbable, but free to check). + const char* banned = alertReporterBannedChannelMatch(out.secret); + if (banned) { + ALERT_DEBUG_PRINTLN("refused banned channel '%s' for alert", banned); + return false; + } + + mesh::Utils::sha256(out.hash, sizeof(out.hash), out.secret, 16); + return true; +} + +void AlertReporter::onConfigChanged() { + // Reset transient state so a config change re-arms the edge detector. +#ifdef WITH_MQTT_BRIDGE + _wifi.state = OK; + _wifi.fired_at_ms = 0; + for (size_t i = 0; i < sizeof(_mqtt) / sizeof(_mqtt[0]); i++) { + _mqtt[i].state = OK; + _mqtt[i].fired_at_ms = 0; + } +#endif +} + +bool AlertReporter::sendChannel(const char* text) { + if (!_mesh || !_prefs) return false; + + mesh::GroupChannel channel; + if (!resolveChannel(channel)) return false; + + // Build ": " plaintext payload. Sender = node name (current). + uint8_t buf[5 + MAX_ALERT_TEXT_LEN + 32]; + uint32_t timestamp = _mesh->getRTCClock()->getCurrentTime(); + memcpy(buf, ×tamp, 4); + buf[4] = ALERT_TXT_TYPE_PLAIN; + + const char* sender = _prefs->node_name[0] ? _prefs->node_name : "node"; + int n = snprintf((char*)&buf[5], MAX_ALERT_TEXT_LEN, "%s: %s", sender, text); + if (n < 0) return false; + if (n >= MAX_ALERT_TEXT_LEN) n = MAX_ALERT_TEXT_LEN - 1; + + mesh::Packet* pkt = _mesh->createGroupDatagram(PAYLOAD_TYPE_GRP_TXT, channel, + buf, 5 + (size_t)n); + if (!pkt) { + ALERT_DEBUG_PRINTLN("createGroupDatagram failed (pool empty?)"); + return false; + } + + // Ride the repeater's default scope (or `alert.region` override) when the + // host MyMesh provides one — same path MyMesh uses for adverts and + // broadcast channel messages. Falls back to plain (unscoped) flood when + // no callbacks are wired or no scope is configured, matching the + // pre-scoped behavior on builds without RegionMap. + // + // path_hash_size must honor the repeater's configured path.hash.mode (1, 2, + // or 3-byte hashes); the Mesh.h default of 1 would silently downgrade + // observers running on 2/3-byte regional meshes. + const uint8_t path_hash_size = (uint8_t)(_prefs->path_hash_mode + 1); + TransportKey scope; + bool have_scope = _callbacks && _callbacks->resolveAlertScope(scope) && !scope.isNull(); + if (have_scope) { + uint16_t codes[2]; + codes[0] = scope.calcTransportCode(pkt); + codes[1] = 0; + _mesh->sendFlood(pkt, codes, 0, path_hash_size); + } else { + _mesh->sendFlood(pkt, 0, path_hash_size); + } + ALERT_DEBUG_PRINTLN("sent: %s", text); + return true; +} + +bool AlertReporter::sendText(const char* text) { + // sendText() is the manual entry point (`alert test` CLI). Deliberately + // does NOT check alert_enabled so operators can verify the PSK / hashtag + // setup without enabling automatic fault firing. + if (!_prefs || !text || !*text) return false; + return sendChannel(text); +} + +void AlertReporter::formatAge(unsigned long age_ms, char* out, size_t out_size) const { + unsigned long secs = age_ms / 1000UL; + unsigned long h = secs / 3600UL; + unsigned long m = (secs % 3600UL) / 60UL; + if (h > 0) { + snprintf(out, out_size, "%luh%lum", h, m); + } else { + snprintf(out, out_size, "%lum", m); + } +} + +void AlertReporter::onLoop(unsigned long now_ms) { + if (!_prefs || !_obs || !_obs->alert_enabled) return; + if (!_mesh) return; + + // Throttle: ~5 s cadence. The thresholds are minutes-scale so this is fine. + if ((long)(now_ms - _next_check_ms) < 0) return; + _next_check_ms = now_ms + 5000UL; + +#ifdef WITH_MQTT_BRIDGE + // Clamp to a 60-minute floor regardless of what's in NodePrefs. The CLI + // already enforces this on set, but a stale prefs file or future field + // tweak shouldn't be able to drag the floor below 1 hour and let a + // flapping link spam the mesh. + // + // The rate limiter only applies between two real sends: fired_at_ms == 0 + // means "never fired since boot/config change", and treating it as a send + // at millis()==0 would suppress every first alert until uptime reaches + // min_interval (observed as a 30-minute alert.mqtt threshold not reporting + // until 60 minutes after a reboot). + uint16_t cfg_min = _obs->alert_min_interval_min; + if (cfg_min < 60) cfg_min = 60; + unsigned long min_interval_ms = (unsigned long)cfg_min * 60000UL; + + // -------- WiFi fault -------- + if (_obs->alert_wifi_minutes > 0) { + unsigned long wifi_disc_ms = MQTTBridge::getLastWifiDisconnectTime(); + unsigned long wifi_conn_ms = MQTTBridge::getWifiConnectedAtMillis(); + bool wifi_down = (wifi_disc_ms != 0 && wifi_conn_ms == 0); + unsigned long down_ms = wifi_down ? (now_ms - wifi_disc_ms) : 0; + unsigned long thresh_ms = (unsigned long)_obs->alert_wifi_minutes * 60000UL; + + if (_wifi.state == OK) { + if (wifi_down && down_ms >= thresh_ms && + (_wifi.fired_at_ms == 0 || (now_ms - _wifi.fired_at_ms) >= min_interval_ms)) { + char age[16]; + formatAge(down_ms, age, sizeof(age)); + uint8_t reason = MQTTBridge::getLastWifiDisconnectReason(); + char text[80]; + if (reason != 0) { + snprintf(text, sizeof(text), "WiFi down %s (reason %u)", age, (unsigned)reason); + } else { + snprintf(text, sizeof(text), "WiFi down %s", age); + } + if (sendChannel(text)) { + _wifi.state = FIRING; + _wifi.fired_at_ms = now_ms; + _wifi.last_outage_started_ms = wifi_disc_ms; + } + } + } else { // FIRING + if (!wifi_down) { + unsigned long total = (wifi_conn_ms != 0 && _wifi.last_outage_started_ms != 0) + ? (wifi_conn_ms - _wifi.last_outage_started_ms) : 0; + char age[16]; + formatAge(total, age, sizeof(age)); + char text[80]; + snprintf(text, sizeof(text), "WiFi recovered after %s", age); + sendChannel(text); + _wifi.state = OK; + } + } + } else if (_wifi.state == FIRING) { + _wifi.state = OK; // threshold disabled mid-fault: silently re-arm + } + + // -------- MQTT slot faults -------- + if (_obs->alert_mqtt_minutes > 0 && _bridge != nullptr) { + int n = MQTTBridge::getRuntimeSlotCount(); + if (n > (int)(sizeof(_mqtt) / sizeof(_mqtt[0]))) n = (int)(sizeof(_mqtt) / sizeof(_mqtt[0])); + unsigned long thresh_ms = (unsigned long)_obs->alert_mqtt_minutes * 60000UL; + + for (int i = 0; i < n; i++) { + Fault& f = _mqtt[i]; + if (!_bridge->isSlotEnabledAndAttempted(i)) { + if (f.state == FIRING) f.state = OK; // slot disabled mid-fault + continue; + } + unsigned long outage_start = _bridge->getSlotCurrentOutageStartMs(i); + bool down = (outage_start != 0); + unsigned long down_ms = down ? (now_ms - outage_start) : 0; + + if (f.state == OK) { + if (down && down_ms >= thresh_ms && + (f.fired_at_ms == 0 || (now_ms - f.fired_at_ms) >= min_interval_ms)) { + char age[16]; + formatAge(down_ms, age, sizeof(age)); + char text[100]; + snprintf(text, sizeof(text), "MQTT slot %d (%s) down %s", + i + 1, _bridge->getSlotPresetName(i), age); + if (sendChannel(text)) { + f.state = FIRING; + f.fired_at_ms = now_ms; + f.last_outage_started_ms = outage_start; + } + } + } else { // FIRING + if (!down) { + unsigned long total = (f.last_outage_started_ms != 0) + ? (now_ms - f.last_outage_started_ms) : 0; + char age[16]; + formatAge(total, age, sizeof(age)); + char text[100]; + snprintf(text, sizeof(text), "MQTT slot %d (%s) recovered after %s", + i + 1, _bridge->getSlotPresetName(i), age); + sendChannel(text); + f.state = OK; + } + } + } + } +#else + (void)now_ms; +#endif +} +#endif // WITH_MQTT_BRIDGE (AlertReporter methods, part 2) diff --git a/src/helpers/AlertReporter.h b/src/helpers/AlertReporter.h new file mode 100644 index 0000000000..390262b688 --- /dev/null +++ b/src/helpers/AlertReporter.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include "CommonCLI.h" + +#ifdef WITH_MQTT_BRIDGE +#include "bridges/MQTTBridge.h" +#endif + +/** + * Returns the label of a banned alert channel if \a secret16 matches one of + * the channels in the BANNED_ALERT_CHANNELS table (e.g. "PUBLIC", "#test", + * "#bot"), or nullptr otherwise. Centralized here so both AlertReporter and + * the CommonCLI `set alert.psk` / `set alert.hashtag` handlers can share one + * source of truth — adding a new banned channel is a one-line table edit. + */ +const char* alertReporterBannedChannelMatch(const uint8_t* secret16); + +/** + * Convenience: hex-decodes \a psk_hex (32 lowercase/uppercase hex chars) and + * forwards to alertReporterBannedChannelMatch. Returns nullptr if not banned + * (or if the input isn't a valid 32-char hex string — only 16-byte secrets + * are present in the banned table). + */ +const char* alertReporterBannedChannelMatchHex(const char* psk_hex); + +/** + * \brief Send-only group-channel "fault alert" reporter for repeater/observer + * builds. + * + * Polls WiFi and per-MQTT-slot outage timers from MQTTBridge. When any timer + * exceeds its configured threshold, floods a single PAYLOAD_TYPE_GRP_TXT + * message on the configured alert channel ("WiFi down 47m — MyObserver"), + * then arms a "recovered" message for the next state transition. + * + * The alert channel must be explicitly configured to either a private hex + * PSK (`set alert.psk`) or a hashtag name (`set alert.hashtag`); the + * well-known PUBLIC group key (and a small list of other auto-responder + * channels — see BANNED_ALERT_CHANNELS in AlertReporter.cpp) are rejected on + * purpose so fault alerts never spam community channels. + * + * Edge-triggered + rate-limited via NodePrefs::alert_min_interval_min so a + * flapping link cannot spam the channel. + * + * Designed to compile and run on any repeater build: + * - The channel-send path uses only mesh::Mesh primitives that already + * exist in the Dispatcher hierarchy (createGroupDatagram + sendFlood). + * - WiFi/MQTT polling is #ifdef WITH_MQTT_BRIDGE-gated; without it, the + * reporter still supports manual `alert test` sends. + */ +#ifdef WITH_MQTT_BRIDGE +class AlertReporter { +public: + AlertReporter(); + + /** + * Wire up the reporter. Must be called from MyMesh::begin() after prefs + * are loaded. \a callbacks is optional — when non-null the reporter uses + * it to resolve a TransportKey scope for outgoing alert floods (so the + * packet rides the repeater's default scope or an `alert.region` override). + */ + void begin(NodePrefs* prefs, MQTTPrefs* obs, mesh::Mesh* mesh, CommonCLICallbacks* callbacks = nullptr); + +#ifdef WITH_MQTT_BRIDGE + /** Bridge can be (re)created lazily; pass nullptr to detach. */ + void setBridge(MQTTBridge* bridge); +#endif + + /** + * Re-derive the cached GroupChannel from \a alert_psk_hex. Call from the + * CLI hot-reload hook after `set alert.psk` / `set alert.hashtag` / `set alert on|off`. + */ + void onConfigChanged(); + + /** + * Cooperative tick. Fast: returns immediately if disabled, throttled + * internally to ~5 s checks. Safe to call every loop(). + */ + void onLoop(unsigned long now_ms); + + /** + * Send an arbitrary text immediately (used by `alert test` CLI). Returns + * false when disabled, PSK invalid, or the underlying flood-send fails. + * Bypasses the rate limiter and edge logic. + */ + bool sendText(const char* text); + +private: + bool resolveChannel(mesh::GroupChannel& out) const; + bool sendChannel(const char* text); + void formatAge(unsigned long age_ms, char* out, size_t out_size) const; + + enum FaultState { OK, FIRING }; + struct Fault { + FaultState state; + unsigned long fired_at_ms; // millis() when we last sent a "down" alert + unsigned long last_outage_started_ms; // remembered so the recovered msg can quote duration + }; + + NodePrefs* _prefs; + MQTTPrefs* _obs; + mesh::Mesh* _mesh; + CommonCLICallbacks* _callbacks; +#ifdef WITH_MQTT_BRIDGE + MQTTBridge* _bridge; + Fault _wifi; + Fault _mqtt[RUNTIME_MQTT_SLOTS]; +#endif + unsigned long _next_check_ms; +}; +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index b318bb58e8..427d456bfd 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -2,12 +2,28 @@ #include "CommonCLI.h" #include "TxtDataHelpers.h" #include "AdvertDataHelpers.h" -#include "TxtDataHelpers.h" +#include "AlertReporter.h" // for alertReporterBannedChannelMatch() +#include "MQTTPrefsAtomicStore.h" #include +#include #ifndef BRIDGE_MAX_BAUD #define BRIDGE_MAX_BAUD 115200 #endif +#ifdef ESP_PLATFORM +#include +#include +#include +#include +#else +#include // mallinfo() for the `memory` command on nRF52/RP2040 +#endif +#ifdef WITH_MQTT_BRIDGE +#include "bridges/MQTTBridge.h" +#include "MQTTDefaults.h" +#include "MQTTPrefsCodec.h" +#include "MQTTPrefsRecovery.h" +#endif // Believe it or not, this std C function is busted on some platforms! static uint32_t _atoi(const char* sp) { @@ -19,15 +35,40 @@ static uint32_t _atoi(const char* sp) { return n; } + static bool isValidName(const char *n) { while (*n) { - if (*n == '[' || *n == ']' || *n == '\\' || *n == ':' || *n == ',' || *n == '?' || *n == '*') return false; + if (*n == '[' || *n == ']' || *n == '/' || *n == '\\' || *n == ':' || *n == ',' || *n == '?' || *n == '*') return false; n++; } return true; } +// Old fork firmware persisted the (since removed) NodePrefs MQTT fields to /com_prefs +// as a zero-filled gap between owner_info (which ends at offset 290) and a trailing +// observer block (rx_boosted_gain, flood_max_*, snmp/watchdog/alert settings). +// The gap size depended on MAX_MQTT_SLOTS at the time: 306 bytes of non-slot fields +// plus 186 bytes per slot (preset 24 + host 64 + port 2 + username 32 + password 64). +// loadPrefsInt() uses the file size to tell the eras apart and recover the tail. +static const size_t LEGACY_MQTT_GAP_6SLOT = 306 + 6 * 186; // 1422 +static const size_t LEGACY_MQTT_GAP_3SLOT = 306 + 3 * 186; // 864 +static const size_t LEGACY_OBS_TAIL_MAX = 124; // rx_boosted(1) + flood(2) + snmp(25) + watchdog(1) + alert block(95) + +// Bytes the last binary layout wrote after owner_info (offsets 290-294): +// rx_boosted_gain, flood_max_unscoped, flood_max_advert, radio_fem_rxgain, +// cad_enabled. loadPrefsInt() treats any larger remainder as a legacy MQTT-gap +// file. Prefs are now written as JSON, so this describes read-side history only. +static const size_t COM_PREFS_TAIL_BYTES = 5; + + void CommonCLI::loadPrefs(FILESYSTEM* fs) { + bool is_fresh_install = false; + bool is_upgrade = false; + // Set when prefs came from one of the legacy binary files; they are republished + // as /prefs.json below. The legacy file is never removed, so it stays available + // as a fallback if the JSON save does not commit this boot. + bool loaded_from_legacy = false; + if (fs->exists("/prefs.json")) { #if defined(RP2040_PLATFORM) File file = fs->open("/prefs.json", "r"); @@ -35,15 +76,73 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { File file = fs->open("/prefs.json"); #endif if (file) { - _prefs->loadSerial(file); // new Serial prefs + _prefs->loadSerial(file); file.close(); } } else if (fs->exists("/com_prefs")) { + // Legacy binary layout. This is a file-format migration only: settings keep + // their stored values, so it must not trigger the bridge.source upgrade below. loadPrefsInt(fs, "/com_prefs"); - if (savePrefs(fs)) { // save to new Serial prefs - // fs->remove("/com_prefs"); // remove old + loaded_from_legacy = true; + } else if (fs->exists("/node_prefs")) { + loadPrefsInt(fs, "/node_prefs"); + is_upgrade = true; // pre-/com_prefs filename + loaded_from_legacy = true; + } else { + // File doesn't exist - set default bridge settings for fresh installs + is_fresh_install = true; + _prefs->bridge_pkt_src = 1; // Default to RX (logRx) for new installs + } +#ifdef WITH_MQTT_BRIDGE + // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt_prefs. + // Readers (MQTTBridge, AlertReporter, observer CLI) use _mqtt_prefs directly — + // these fields no longer exist in NodePrefs, so there is nothing to sync. + MQTTPrefsAtomicStore::LegacyUpgradeGate legacy_upgrade( + _com_prefs_needs_upgrade || loaded_from_legacy); + loadMQTTPrefs(fs, &legacy_upgrade); + if (_mqtt_prefs_hold) legacy_upgrade.holdMqttSource(); + + // For MQTT bridge, migrate bridge.source to RX (logRx) only on fresh installs or upgrades + // so legacy "tx" is not the default. mqtt.rx / mqtt.tx are separate (fresh default: advert for TX) + if ((is_fresh_install || is_upgrade) && _prefs->bridge_pkt_src == 0) { + if (legacy_upgrade.blocksComPrefsRewrite()) { + MESH_DEBUG_PRINTLN("MQTT Bridge: deferring bridge.source migration until legacy prefs are preserved"); + } else { + MESH_DEBUG_PRINTLN("MQTT Bridge: Migrating bridge.source from tx to rx (MQTT bridge default)"); + _prefs->bridge_pkt_src = 1; // Set to RX (logRx) + if (loaded_from_legacy) { + // The /prefs.json migration below persists this in-memory change. + MESH_DEBUG_PRINTLN("MQTT Bridge: bridge.source will be saved with the prefs migration"); + } else { + savePrefs(fs); // Save the updated preference + } + } + } + // mqtt_rx_enabled: new field appended to end of MQTTPrefs. On upgrade from older firmware, + // the shorter /mqtt_prefs file won't contain it, so it keeps the default value (1 = on) + // set by setMQTTPrefsDefaults(). No explicit migration needed. +#endif + + // Republish legacy binary prefs as /prefs.json. Old-format files also carried a + // trailing observer block, which loadPrefsInt() recovered into _legacy_tail; wait + // for loadMQTTPrefs() to commit that to /mqtt_prefs first. The legacy file is left + // on flash either way, so a deferred or failed save just retries on the next boot. +#ifdef WITH_MQTT_BRIDGE + if (loaded_from_legacy || _com_prefs_needs_upgrade) { + if (legacy_upgrade.mayRewriteComPrefs()) { + savePrefs(fs, false); // loadMQTTPrefs already committed the MQTT payload + legacy_upgrade.recordComPrefsRewrite(); + _com_prefs_needs_upgrade = false; + } else { + MESH_DEBUG_PRINTLN("Prefs: deferring /prefs.json migration until /mqtt_prefs commits"); } } +#else + if (loaded_from_legacy || _com_prefs_needs_upgrade) { + savePrefs(fs); + _com_prefs_needs_upgrade = false; + } +#endif } void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { // Legacy prefs loader @@ -70,7 +169,7 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { // Legacy file.read((uint8_t *)&_prefs->tx_delay_factor, sizeof(_prefs->tx_delay_factor)); // 84 file.read((uint8_t *)&_prefs->guest_password[0], sizeof(_prefs->guest_password)); // 88 file.read((uint8_t *)&_prefs->direct_tx_delay_factor, sizeof(_prefs->direct_tx_delay_factor)); // 104 - file.read(pad, 4); // 108 : 4 bytes unused + file.read(pad, 4); // 108 file.read((uint8_t *)&_prefs->sf, sizeof(_prefs->sf)); // 112 file.read((uint8_t *)&_prefs->cr, sizeof(_prefs->cr)); // 113 file.read((uint8_t *)&_prefs->allow_read_only, sizeof(_prefs->allow_read_only)); // 114 @@ -97,12 +196,130 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { // Legacy file.read((uint8_t *)&_prefs->discovery_mod_timestamp, sizeof(_prefs->discovery_mod_timestamp)); // 162 file.read((uint8_t *)&_prefs->adc_multiplier, sizeof(_prefs->adc_multiplier)); // 166 file.read((uint8_t *)_prefs->owner_info, sizeof(_prefs->owner_info)); // 170 - file.read((uint8_t *)&_prefs->rx_boosted_gain, sizeof(_prefs->rx_boosted_gain)); // 290 - file.read((uint8_t *)&_prefs->flood_max_unscoped, sizeof(_prefs->flood_max_unscoped)); // 291 - file.read((uint8_t *)&_prefs->flood_max_advert, sizeof(_prefs->flood_max_advert)); // 292 - file.read((uint8_t *)&_prefs->radio_fem_rxgain, sizeof(_prefs->radio_fem_rxgain)); // 293 - file.read((uint8_t *)&_prefs->cad_enabled, sizeof(_prefs->cad_enabled)); // 294 - // next: 295 + // MQTT/observer settings are no longer stored in /com_prefs — they live in + // /mqtt_prefs (loaded by loadMQTTPrefs). Old fork firmware wrote a zero-filled + // MQTT gap here followed by a trailing observer block; detect that layout by the + // extra length, skip the gap, and recover the tail so those settings survive + // the upgrade (the file is rewritten in the new layout by loadPrefs afterwards). + // Defaults for the trailing fields that older/shorter files may not contain. + // (upstream defaults: FEM RX gain on, CAD off) — overwritten below if present. + _prefs->radio_fem_rxgain = 1; + _prefs->cad_enabled = 0; + // A remainder larger than the new-format tail means an old fork file with the + // legacy MQTT gap; detect and recover it below. + size_t extra = file.available(); + if (extra > COM_PREFS_TAIL_BYTES) { + _com_prefs_needs_upgrade = true; + size_t gap = 0; + if (extra > LEGACY_MQTT_GAP_6SLOT && extra <= LEGACY_MQTT_GAP_6SLOT + LEGACY_OBS_TAIL_MAX) { + gap = LEGACY_MQTT_GAP_6SLOT; + } else if (extra > LEGACY_MQTT_GAP_3SLOT && extra <= LEGACY_MQTT_GAP_3SLOT + LEGACY_OBS_TAIL_MAX) { + gap = LEGACY_MQTT_GAP_3SLOT; + } + // Unrecognized legacy sizes (e.g. pre-slot-era files) leave gap == 0: the tail + // is not read and everything past owner_info degrades to defaults. + if (gap > 0) { + uint8_t skip_buf[64]; + size_t remaining = gap; + while (remaining > 0) { + size_t n = remaining > sizeof(skip_buf) ? sizeof(skip_buf) : remaining; + file.read(skip_buf, n); + remaining -= n; + } + file.read((uint8_t *)&_prefs->rx_boosted_gain, sizeof(_prefs->rx_boosted_gain)); + // Tail layout: flood_max_unscoped, flood_max_advert, then the snmp fields — + // except legacy flex-branch files where snmp starts right after + // rx_boosted_gain (no flood_max_*). Same heuristic the old firmware used: + // snmp_enabled is 0/1 and the first community char is printable (> 64). + uint8_t b1 = 0, b2 = 0; + bool have_flood_bytes = file.available() >= 2; + if (have_flood_bytes) { + file.read(&b1, 1); + file.read(&b2, 1); + } +#ifdef WITH_MQTT_BRIDGE + // Pre-fill with the same defaults applyMQTTDefaults() uses, so fields a + // shorter (older) tail doesn't contain degrade to defaults when applied. + memset(&_legacy_tail, 0, sizeof(_legacy_tail)); + strncpy(_legacy_tail.snmp_community, "public", sizeof(_legacy_tail.snmp_community) - 1); + _legacy_tail.radio_watchdog_minutes = 5; + _legacy_tail.alert_wifi_minutes = 30; + _legacy_tail.alert_mqtt_minutes = 240; + _legacy_tail.alert_min_interval_min = 60; +#endif + if (have_flood_bytes && b1 <= 1 && b2 > 64) { + // Legacy variant: no flood_max_* — b1/b2 are snmp_enabled + community[0] +#ifdef WITH_MQTT_BRIDGE + _legacy_tail.snmp_enabled = b1; + _legacy_tail.snmp_community[0] = (char) b2; + if (file.available() >= (int)(sizeof(_legacy_tail.snmp_community) - 1)) { + file.read((uint8_t *)&_legacy_tail.snmp_community[1], sizeof(_legacy_tail.snmp_community) - 1); + } +#endif + } else if (have_flood_bytes) { + _prefs->flood_max_unscoped = b1; + _prefs->flood_max_advert = b2; +#ifdef WITH_MQTT_BRIDGE + if (file.available() >= (int)sizeof(_legacy_tail.snmp_enabled)) { + file.read((uint8_t *)&_legacy_tail.snmp_enabled, sizeof(_legacy_tail.snmp_enabled)); + } + if (file.available() >= (int)sizeof(_legacy_tail.snmp_community)) { + file.read((uint8_t *)&_legacy_tail.snmp_community, sizeof(_legacy_tail.snmp_community)); + } +#endif + } +#ifdef WITH_MQTT_BRIDGE + if (file.available() >= (int)sizeof(_legacy_tail.radio_watchdog_minutes)) { + file.read((uint8_t *)&_legacy_tail.radio_watchdog_minutes, sizeof(_legacy_tail.radio_watchdog_minutes)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_enabled)) { + file.read((uint8_t *)&_legacy_tail.alert_enabled, sizeof(_legacy_tail.alert_enabled)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_psk_hex)) { + file.read((uint8_t *)&_legacy_tail.alert_psk_hex, sizeof(_legacy_tail.alert_psk_hex)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_wifi_minutes)) { + file.read((uint8_t *)&_legacy_tail.alert_wifi_minutes, sizeof(_legacy_tail.alert_wifi_minutes)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_mqtt_minutes)) { + file.read((uint8_t *)&_legacy_tail.alert_mqtt_minutes, sizeof(_legacy_tail.alert_mqtt_minutes)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_min_interval_min)) { + file.read((uint8_t *)&_legacy_tail.alert_min_interval_min, sizeof(_legacy_tail.alert_min_interval_min)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_hashtag)) { + file.read((uint8_t *)&_legacy_tail.alert_hashtag, sizeof(_legacy_tail.alert_hashtag)); + } + if (file.available() >= (int)sizeof(_legacy_tail.alert_region)) { + file.read((uint8_t *)&_legacy_tail.alert_region, sizeof(_legacy_tail.alert_region)); + } + _legacy_tail.snmp_enabled = constrain(_legacy_tail.snmp_enabled, 0, 1); + _legacy_tail.radio_watchdog_minutes = constrain(_legacy_tail.radio_watchdog_minutes, 0, 120); + _legacy_tail.alert_enabled = constrain(_legacy_tail.alert_enabled, 0, 1); + _legacy_tail.snmp_community[sizeof(_legacy_tail.snmp_community) - 1] = '\0'; + _legacy_tail.alert_psk_hex[sizeof(_legacy_tail.alert_psk_hex) - 1] = '\0'; + _legacy_tail.alert_hashtag[sizeof(_legacy_tail.alert_hashtag) - 1] = '\0'; + _legacy_tail.alert_region[sizeof(_legacy_tail.alert_region) - 1] = '\0'; + _legacy_tail.valid = true; +#endif + } + } else { + if (file.available() >= (int)sizeof(_prefs->rx_boosted_gain)) { + file.read((uint8_t *)&_prefs->rx_boosted_gain, sizeof(_prefs->rx_boosted_gain)); + } + if (file.available() >= (int)sizeof(_prefs->flood_max_unscoped)) { + file.read((uint8_t *)&_prefs->flood_max_unscoped, sizeof(_prefs->flood_max_unscoped)); + } + if (file.available() >= (int)sizeof(_prefs->flood_max_advert)) { + file.read((uint8_t *)&_prefs->flood_max_advert, sizeof(_prefs->flood_max_advert)); + } + if (file.available() >= (int)sizeof(_prefs->radio_fem_rxgain)) { // 293 + file.read((uint8_t *)&_prefs->radio_fem_rxgain, sizeof(_prefs->radio_fem_rxgain)); + } + if (file.available() >= (int)sizeof(_prefs->cad_enabled)) { // 294 + file.read((uint8_t *)&_prefs->cad_enabled, sizeof(_prefs->cad_enabled)); + } + } // sanitise bad pref values _prefs->rx_delay_base = constrain(_prefs->rx_delay_base, 0, 20.0f); @@ -117,6 +334,9 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { // Legacy _prefs->multi_acks = constrain(_prefs->multi_acks, 0, 1); _prefs->adc_multiplier = constrain(_prefs->adc_multiplier, 0.0f, 10.0f); _prefs->path_hash_mode = constrain(_prefs->path_hash_mode, 0, 2); // NOTE: mode 3 reserved for future + _prefs->loop_detect = constrain(_prefs->loop_detect, 0, 3); // LOOP_DETECT_OFF..LOOP_DETECT_STRICT + _prefs->radio_fem_rxgain = constrain(_prefs->radio_fem_rxgain, 0, 1); // boolean + _prefs->cad_enabled = constrain(_prefs->cad_enabled, 0, 1); // boolean // sanitise bad bridge pref values _prefs->bridge_enabled = constrain(_prefs->bridge_enabled, 0, 1); @@ -130,7 +350,6 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { // Legacy _prefs->gps_enabled = constrain(_prefs->gps_enabled, 0, 1); _prefs->advert_loc_policy = constrain(_prefs->advert_loc_policy, 0, 2); - // sanitise settings _prefs->rx_boosted_gain = constrain(_prefs->rx_boosted_gain, 0, 1); // boolean _prefs->radio_fem_rxgain = constrain(_prefs->radio_fem_rxgain, 0, 1); // boolean _prefs->radio_fem_txgain = constrain(_prefs->radio_fem_txgain, 0, 1); // boolean @@ -140,7 +359,7 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { // Legacy } } -bool CommonCLI::savePrefs(FILESYSTEM* fs) { +bool CommonCLI::savePrefs(FILESYSTEM* fs, bool save_mqtt) { #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) fs->remove("/prefs.json"); File file = fs->open("/prefs.json", FILE_O_WRITE); @@ -149,20 +368,436 @@ bool CommonCLI::savePrefs(FILESYSTEM* fs) { #else File file = fs->open("/prefs.json", "w", true); #endif + bool success = false; if (file) { - bool success = _prefs->saveSerial(file); + success = _prefs->saveSerial(file); file.close(); - return success; + } +#ifdef WITH_MQTT_BRIDGE + // Observer config (MQTT/WiFi/timezone/SNMP/alert) is persisted separately. The + // observer CLI writes _mqtt_prefs directly, so no NodePrefs->MQTTPrefs sync runs. + // Runs regardless of the NodePrefs result so a failed JSON write cannot strand it. + if (save_mqtt) saveMQTTPrefs(fs); +#endif + return success; +} + +#ifdef WITH_MQTT_BRIDGE +// Set default values for MQTT preferences (used when file doesn't exist or is corrupted) +static void setMQTTPrefsDefaults(MQTTPrefs* prefs) { + applyMQTTDefaults(prefs); +} + +static File openMqttPrefsRead(FILESYSTEM* fs, const char* path = "/mqtt_prefs") { +#if defined(RP2040_PLATFORM) + return fs->open(path, "r"); +#else + return fs->open(path); +#endif +} + +static MQTTPrefsRecovery::FileState mqttPrefsFileState(FILESYSTEM* fs, const char* path) { + if (!fs->exists(path)) return MQTTPrefsRecovery::FileState::Missing; + File file = openMqttPrefsRead(fs, path); + if (!file) return MQTTPrefsRecovery::FileState::Preserve; + const size_t file_size = file.size(); + uint8_t prefix[sizeof(MQTTPrefsHeader)] = {}; + const size_t prefix_size = file_size < sizeof(prefix) ? file_size : sizeof(prefix); + const size_t prefix_read = file.read(prefix, prefix_size); + file.close(); + return MQTTPrefsCodec::classify(prefix, prefix_read, file_size).preserve_file + ? MQTTPrefsRecovery::FileState::Preserve + : MQTTPrefsRecovery::FileState::Usable; +} + +// Restore the only usable image before the normal loader inspects /mqtt_prefs. +// SPIFFS cannot rename over an existing destination, so publishing moves the +// old primary to .bak before moving the verified temp into the empty name. +// The decision helper deliberately treats unsupported/corrupt files as opaque: +// no recovery path overwrites one with an older layout. +static bool recoverMqttPrefsFiles(FILESYSTEM* fs) { + const MQTTPrefsRecovery::FileState primary = mqttPrefsFileState(fs, "/mqtt_prefs"); + const MQTTPrefsRecovery::FileState temp = mqttPrefsFileState(fs, "/mqtt_prefs.tmp"); + const MQTTPrefsRecovery::FileState backup = mqttPrefsFileState(fs, "/mqtt_prefs.bak"); + const MQTTPrefsRecovery::Action action = MQTTPrefsRecovery::select(primary, temp, backup); + + if (action == MQTTPrefsRecovery::Action::KeepPrimary) { + // A current/known legacy primary has already published. Every transaction + // artifact is therefore unpublished or stale, including a partial temp + // left by a reset during write(), and can be discarded. Preserve artifacts + // only when the primary itself is opaque (the branch above still keeps it). + if (primary == MQTTPrefsRecovery::FileState::Usable) { + if (temp != MQTTPrefsRecovery::FileState::Missing) fs->remove("/mqtt_prefs.tmp"); + if (backup != MQTTPrefsRecovery::FileState::Missing) fs->remove("/mqtt_prefs.bak"); + } + return false; + } + if (action == MQTTPrefsRecovery::Action::PromoteTemp) { + if (fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs")) { + // A usable temp is now the committed primary. Its backup is necessarily + // a stale transaction artifact, even if this firmware cannot decode it. + if (temp == MQTTPrefsRecovery::FileState::Usable && + backup != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt_prefs.bak"); + } + MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt_prefs from transaction temp"); + return false; + } + MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt_prefs temp; files preserved"); + return true; + } + if (action == MQTTPrefsRecovery::Action::PromoteBackup) { + if (fs->rename("/mqtt_prefs.bak", "/mqtt_prefs")) { + // Symmetric case: a usable backup is now primary, so any interrupted + // temp is no longer authoritative and must not block a later save. + if (backup == MQTTPrefsRecovery::FileState::Usable && + temp != MQTTPrefsRecovery::FileState::Missing) { + fs->remove("/mqtt_prefs.tmp"); + } + MESH_DEBUG_PRINTLN("MQTT: recovered /mqtt_prefs from transaction backup"); + return false; + } + MESH_DEBUG_PRINTLN("MQTT: could not recover /mqtt_prefs backup; files preserved"); + return true; } return false; } +// Filesystem adapter for MQTTPrefsAtomicStore. It writes the new image to +// /mqtt_prefs.tmp and verifies its size. Publishing is a recoverable SPIFFS +// transaction: primary -> .bak, then tmp -> primary, then best-effort backup +// cleanup. A power loss at every boundary leaves at least one recoverable file. +class MQTTPrefsFileStore { +public: + explicit MQTTPrefsFileStore(FILESYSTEM* fs) : _fs(fs) {} + + bool begin() { + _finished = false; + _open = false; + _owns_temp = false; + _bytes_written = 0; + // Recovery owns stale artifacts. Do not delete them here: a failed commit + // may have moved the old primary to .bak and left a verified temp that the + // next boot must choose between. Refusing the save is safer than erasing an + // image this firmware cannot decode. + if (_fs->exists("/mqtt_prefs.tmp") || _fs->exists("/mqtt_prefs.bak")) return false; +#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) + _file = _fs->open("/mqtt_prefs.tmp", FILE_O_WRITE); +#elif defined(RP2040_PLATFORM) + _file = _fs->open("/mqtt_prefs.tmp", "w"); +#else + _file = _fs->open("/mqtt_prefs.tmp", "w", true); +#endif + _open = _file; + _owns_temp = _open; + return _open; + } + + size_t write(const uint8_t* bytes, size_t size) { + if (!_open) return 0; + const size_t written = _file.write(bytes, size); + _bytes_written += written; + return written; + } + + bool finish() { + if (!_open) return false; + _file.close(); + _open = false; +#if defined(RP2040_PLATFORM) + File verify = _fs->open("/mqtt_prefs.tmp", "r"); +#else + File verify = _fs->open("/mqtt_prefs.tmp"); +#endif + if (!verify) return false; + const bool complete = verify.size() == _bytes_written; + verify.close(); + if (!complete) return false; + _finished = true; + return true; + } + + bool commit() { + if (!_finished) return false; + // SPIFFS refuses rename(tmp, existing_dest). Move the existing image to a + // recoverable backup first, then publish temp into the now-empty primary. + // Never remove either image after a failed boundary; boot recovery selects + // the completed temp or restores the backup. + if (_fs->exists("/mqtt_prefs.bak")) return false; + if (_fs->exists("/mqtt_prefs") && !_fs->rename("/mqtt_prefs", "/mqtt_prefs.bak")) { + return false; + } + if (!_fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs")) return false; + // Cleanup failure is non-fatal: the new primary is published and recovery + // will remove a known-good stale backup on a later boot. + if (_fs->exists("/mqtt_prefs.bak")) _fs->remove("/mqtt_prefs.bak"); + return true; + } + + void abort() { + if (_open) _file.close(); + _open = false; + // Once finish() has verified the temp, commit may already have moved the + // primary to .bak. Keep the temp on a commit failure so recovery can + // publish it (or fall back to .bak) after reset. + if (_owns_temp && !_finished && _fs->exists("/mqtt_prefs.tmp")) { + _fs->remove("/mqtt_prefs.tmp"); + } + _finished = false; + _owns_temp = false; + } + +private: + FILESYSTEM* _fs; + File _file; + bool _open = false; + bool _finished = false; + bool _owns_temp = false; + size_t _bytes_written = 0; +}; + +#endif // WITH_MQTT_BRIDGE + +#ifdef WITH_MQTT_BRIDGE + +static const char* mqttPrefsSaveResultName(MQTTPrefsAtomicStore::Result result) { + switch (result) { + case MQTTPrefsAtomicStore::Result::BeginFailed: return "begin"; + case MQTTPrefsAtomicStore::Result::HeaderWriteFailed: return "header write"; + case MQTTPrefsAtomicStore::Result::PayloadWriteFailed: return "payload write"; + case MQTTPrefsAtomicStore::Result::FinishFailed: return "close"; + case MQTTPrefsAtomicStore::Result::CommitFailed: return "rename"; + case MQTTPrefsAtomicStore::Result::Committed: return "committed"; + } + return "unknown"; +} + +void CommonCLI::loadMQTTPrefs( + FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade) { + setMQTTPrefsDefaults(&_mqtt_prefs); + // Complete or preserve an interrupted SPIFFS transaction before decoding. + // A failed recovery leaves the artifacts untouched and blocks this boot from + // replacing them with defaults through a later CLI save. + _mqtt_prefs_hold = recoverMqttPrefsFiles(fs); + bool has_observer_fields = false; + bool mqtt_rewrite_pending = false; + bool migrated_legacy_mqtt = false; + + if (fs->exists("/mqtt_prefs")) { + File file = openMqttPrefsRead(fs); + if (file) { + const size_t file_size = file.size(); + uint8_t prefix[sizeof(MQTTPrefsHeader)] = {}; + const size_t prefix_size = file_size < sizeof(prefix) ? file_size : sizeof(prefix); + const size_t prefix_read = file.read(prefix, prefix_size); + file.close(); + + const MQTTPrefsCodec::DecodePlan plan = + MQTTPrefsCodec::classify(prefix, prefix_read, file_size); + if (plan.preserve_file) { + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs is unsupported or corrupt, using defaults (file preserved)"); + } else if (plan.source == MQTTPrefsCodec::Source::Current) { + file = openMqttPrefsRead(fs); + MQTTPrefsHeader header; + if (!file || file.read((uint8_t *)&header, sizeof(header)) != sizeof(header) || + file.read((uint8_t *)&_mqtt_prefs, plan.payload_len) != plan.payload_len) { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs read failed, using defaults (file preserved)"); + } else { + has_observer_fields = plan.observer_fields_present; + // Written by a later build with appended fields. Everything this + // binary knows loaded normally; say so, because the next `set` will + // rewrite the file at this length and drop the newer settings. + if (file_size - sizeof(MQTTPrefsHeader) > plan.payload_len) { + MESH_DEBUG_PRINTLN( + "MQTT: /mqtt_prefs written by newer firmware (%u > %u bytes); " + "config loaded, newer settings ignored and dropped on next save", + (unsigned)(file_size - sizeof(MQTTPrefsHeader)), + (unsigned)plan.payload_len); + } + } + if (file) file.close(); + } else if (plan.rewrite_legacy) { + bool migrated = false; + file = openMqttPrefsRead(fs); + if (file) { + switch (plan.source) { + case MQTTPrefsCodec::Source::LegacyPreSlot: { + union { + OldMQTTPrefs post_wifi_power; + PreWifiPowerOldMQTTPrefs pre_wifi_power; + } old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, sizeof(old_prefs)) == sizeof(old_prefs)) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, sizeof(old_prefs))) { + if (MQTTPrefsCodec::looksLikePreWifiPower((uint8_t *)&old_prefs, sizeof(old_prefs))) { + MQTTPrefsCodec::migratePreWifiPower(old_prefs.pre_wifi_power, &_mqtt_prefs); + } else { + MQTTPrefsCodec::migratePreSlot(old_prefs.post_wifi_power, &_mqtt_prefs); + } + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + case MQTTPrefsCodec::Source::LegacyThreeSlotBase: { + ThreeSlotBaseMQTTPrefs old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, sizeof(old_prefs)) == sizeof(old_prefs)) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, sizeof(old_prefs))) { + MQTTPrefsCodec::migrateThreeSlot(old_prefs, &_mqtt_prefs); + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + case MQTTPrefsCodec::Source::LegacyThreeSlot: { + ThreeSlotMQTTPrefs old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, sizeof(old_prefs)) == sizeof(old_prefs)) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, sizeof(old_prefs))) { + MQTTPrefsCodec::migrateThreeSlot(old_prefs, &_mqtt_prefs); + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + case MQTTPrefsCodec::Source::LegacySixSlotBase: + case MQTTPrefsCodec::Source::LegacySixSlotAudience: + case MQTTPrefsCodec::Source::LegacySixSlotAudienceRx: + case MQTTPrefsCodec::Source::LegacySixSlot: { + Legacy6SlotMQTTPrefs old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, plan.payload_len) == plan.payload_len) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, plan.payload_len)) { + MQTTPrefsCodec::migrateLegacySixSlot(old_prefs, plan.source, &_mqtt_prefs); + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + default: + break; + } + file.close(); + } + if (migrated) { + // Do not save yet: a legacy /com_prefs observer tail may still need + // to be overlaid below. Publish the complete v1 image once, after it. + mqtt_rewrite_pending = true; + migrated_legacy_mqtt = true; + } else { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy read failed, using defaults (file preserved)"); + } + } + } else { + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs could not be opened, using defaults (file preserved)"); + } + } + + if (_legacy_tail.valid && !has_observer_fields) { + _mqtt_prefs.snmp_enabled = _legacy_tail.snmp_enabled; + memcpy(_mqtt_prefs.snmp_community, _legacy_tail.snmp_community, sizeof(_mqtt_prefs.snmp_community)); + _mqtt_prefs.radio_watchdog_minutes = _legacy_tail.radio_watchdog_minutes; + _mqtt_prefs.alert_enabled = _legacy_tail.alert_enabled; + memcpy(_mqtt_prefs.alert_psk_hex, _legacy_tail.alert_psk_hex, sizeof(_mqtt_prefs.alert_psk_hex)); + _mqtt_prefs.alert_wifi_minutes = _legacy_tail.alert_wifi_minutes; + _mqtt_prefs.alert_mqtt_minutes = _legacy_tail.alert_mqtt_minutes; + _mqtt_prefs.alert_min_interval_min = _legacy_tail.alert_min_interval_min; + memcpy(_mqtt_prefs.alert_hashtag, _legacy_tail.alert_hashtag, sizeof(_mqtt_prefs.alert_hashtag)); + memcpy(_mqtt_prefs.alert_region, _legacy_tail.alert_region, sizeof(_mqtt_prefs.alert_region)); + mqtt_rewrite_pending = true; + MESH_DEBUG_PRINTLN("MQTT: Migrated observer settings from legacy /com_prefs trailing block"); + } + + // Keep persisted values inside the signed-delta millis() scheduling window. + // This also repairs any manually-written or experimental value from firmware + // that briefly accepted intervals longer than the supported two-week cap. + if (_mqtt_prefs.mqtt_neighbors_interval < MQTT_NEIGHBORS_MIN_INTERVAL_MS || + _mqtt_prefs.mqtt_neighbors_interval > MQTT_NEIGHBORS_MAX_INTERVAL_MS) { + _mqtt_prefs.mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; + // Persist the repair so a corrupt flash value is not re-clamped every boot. + // Skip when hold is set so we never overwrite a deliberately preserved file. + if (!_mqtt_prefs_hold) { + mqtt_rewrite_pending = true; + } + MESH_DEBUG_PRINTLN("MQTT: invalid neighbors interval reset to %u hours", + (unsigned)MQTT_NEIGHBORS_DEFAULT_INTERVAL_HOURS); + } + _legacy_tail.valid = false; + + if (mqtt_rewrite_pending) { + legacy_upgrade->requireMqttRewrite(); + if (migrated_legacy_mqtt) { + MESH_DEBUG_PRINTLN("MQTT: Migrating headerless /mqtt_prefs to versioned layout"); + } else { + MESH_DEBUG_PRINTLN("MQTT: Persisting observer tail into /mqtt_prefs before /com_prefs compaction"); + } + if (saveMQTTPrefs(fs)) { + legacy_upgrade->recordMqttSave(true); + } else { + // The legacy source(s) remain intact because the failed transaction never + // published its temp file. Hold this boot so loadPrefs leaves /com_prefs + // untouched; the next boot can recover the tail and retry the transaction. + _mqtt_prefs_hold = true; + legacy_upgrade->recordMqttSave(false); + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs migration save failed; legacy files preserved and held"); + } + } +} + +bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { + if (_mqtt_prefs_hold) { + // Loading deliberately preserved the source file. Do not replace it with this + // boot's defaults after an unsupported, corrupt, or temporarily failed read. + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs held, not overwriting"); + return false; + } + + // Write header and payload sequentially so the transaction needs no second + // full-size (2.8 KiB) staging buffer on constrained targets. The length is + // the shortest that still round-trips this config, so a node with default + // packet filters keeps writing a payload older firmware can read. + const size_t payload_len = MQTTPrefsCodec::payloadLenFor(_mqtt_prefs); + const MQTTPrefsHeader header = MQTTPrefsCodec::makeHeader(payload_len); + MQTTPrefsFileStore store(fs); + const MQTTPrefsAtomicStore::Result result = MQTTPrefsAtomicStore::write( + store, (const uint8_t *)&header, sizeof(header), + (const uint8_t *)&_mqtt_prefs, payload_len); + if (!MQTTPrefsAtomicStore::committed(result)) { + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt_prefs save failed at %s; source preserved", + mqttPrefsSaveResultName(result)); + return false; + } + return true; +} + +#endif + #define MIN_LOCAL_ADVERT_INTERVAL 60 void CommonCLI::savePrefs() { + uint8_t old_advert_interval = _prefs->advert_interval; if (_prefs->advert_interval * 2 < MIN_LOCAL_ADVERT_INTERVAL) { _prefs->advert_interval = 0; // turn it off, now that device has been manually configured } + // If advert_interval was changed, update the timer to reflect the change + if (old_advert_interval != _prefs->advert_interval) { + _callbacks->updateAdvertTimer(); + } _callbacks->savePrefs(); } @@ -180,6 +815,9 @@ uint8_t CommonCLI::buildAdvertData(uint8_t node_type, uint8_t* app_data) { } void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* reply) { + // Observer-only top-level commands (ota check/update, tls.bundletest, alert test) + // live in CommonCLI_Observer.cpp. + if (handleObserverCommand(sender_timestamp, command, reply)) return; if (memcmp(command, "poweroff", 8) == 0 || memcmp(command, "shutdown", 8) == 0) { _board->powerOff(); // doesn't return } else if (memcmp(command, "reboot", 6) == 0) { @@ -206,8 +844,34 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re } else { strcpy(reply, "ERR: clock cannot go backwards"); } + } else if (memcmp(command, "memory", 6) == 0) { +#ifdef ESP_PLATFORM + sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d, IntFree: %d, IntMax: %d, PSRAM: %d/%d", + ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(), + _callbacks->getQueueSize(), + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL), + (int)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL), + (int)heap_caps_get_free_size(MALLOC_CAP_SPIRAM), + (int)heap_caps_get_total_size(MALLOC_CAP_SPIRAM)); +#else + // newlib arena stats — the portable equivalent on nRF52/RP2040. There is + // no min-ever-free or largest-free-block counterpart, so those fields are + // left out rather than filled with numbers that mean something different. + // Frags is the free-chunk count, the closest available fragmentation hint. + struct mallinfo mi = mallinfo(); + sprintf(reply, "Free: %d, Used: %d, Arena: %d, Frags: %d, Queue: %d", + (int)mi.fordblks, (int)mi.uordblks, (int)mi.arena, (int)mi.ordblks, + _callbacks->getQueueSize()); +#endif } else if (memcmp(command, "start ota", 9) == 0) { - if (!_board->startOTAUpdate(_prefs->node_name, reply)) { + // Manual OTA: bring up the ElegantOTA web UI for a hand-uploaded binary. + // Plain "start ota" serves on the station IP when joined to WiFi, else + // raises the MeshCore-OTA SoftAP. "start ota ap" forces the SoftAP even + // when connected, so the UI is reachable when the network applies client + // isolation and the station IP can't be reached. (&& short-circuits keep + // the [10]/[11] reads in-bounds when command == "start ota".) + bool force_ap = (command[9] == ' ' && command[10] == 'a' && command[11] == 'p'); + if (!_board->startOTAUpdate(_prefs->node_name, reply, force_ap)) { strcpy(reply, "Error"); } } else if (memcmp(command, "clock", 5) == 0) { @@ -257,8 +921,7 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re // change admin password StrHelper::strncpy(_prefs->password, &command[9], sizeof(_prefs->password)); savePrefs(); - sprintf(reply, "password now: "); - StrHelper::strncpy(&reply[14], _prefs->password, 160-15); // echo back just to let admin know for sure!! + sprintf(reply, "password now: %s", _prefs->password); // echo back just to let admin know for sure!! } else if (memcmp(command, "clear stats", 11) == 0) { _callbacks->clearStats(); strcpy(reply, "(OK - stats reset)"); @@ -436,6 +1099,8 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re strcpy(reply, " EOF"); } else if (sender_timestamp == 0 && memcmp(command, "stats-packets", 13) == 0 && (command[13] == 0 || command[13] == ' ')) { _callbacks->formatPacketStatsReply(reply); + } else if (sender_timestamp == 0 && memcmp(command, "stats-radio-diag", 16) == 0 && (command[16] == 0 || command[16] == ' ')) { + _callbacks->formatRadioDiagReply(reply); } else if (sender_timestamp == 0 && memcmp(command, "stats-radio", 11) == 0 && (command[11] == 0 || command[11] == ' ')) { _callbacks->formatRadioStatsReply(reply); } else if (sender_timestamp == 0 && memcmp(command, "stats-core", 10) == 0 && (command[10] == 0 || command[10] == ' ')) { @@ -447,6 +1112,8 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* reply) { const char* config = &command[4]; + // Observer/MQTT/WiFi/timezone/alert/SNMP commands live in CommonCLI_Observer.cpp. + if (handleObserverSetCmd(sender_timestamp, config, reply)) return; if (memcmp(config, "dutycycle ", 10) == 0) { float dc = atof(&config[10]); if (dc < 1 || dc > 100) { @@ -471,6 +1138,28 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep _prefs->cad_enabled = memcmp(&config[4], "on", 2) == 0; savePrefs(); strcpy(reply, "OK"); + } else if (memcmp(config, "radio.fem.rxgain ", 17) == 0) { + if (!_board->canControlLoRaFemLna()) { + strcpy(reply, "Error: unsupported"); + } else if (memcmp(&config[17], "on", 2) == 0) { + if (_board->setLoRaFemLnaEnabled(true)) { + _prefs->radio_fem_rxgain = 1; + savePrefs(); + strcpy(reply, "OK - LoRa FEM RX gain on"); + } else { + strcpy(reply, "Error: failed to apply LoRa FEM RX gain"); + } + } else if (memcmp(&config[17], "off", 3) == 0) { + if (_board->setLoRaFemLnaEnabled(false)) { + _prefs->radio_fem_rxgain = 0; + savePrefs(); + strcpy(reply, "OK - LoRa FEM RX gain off"); + } else { + strcpy(reply, "Error: failed to apply LoRa FEM RX gain"); + } + } else { + strcpy(reply, "Error: state must be on or off"); + } } else if (memcmp(config, "agc.reset.interval ", 19) == 0) { _prefs->agc_reset_interval = atoi(&config[19]) / 4; savePrefs(); @@ -731,6 +1420,15 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep } } else if (memcmp(config, "bridge.source ", 14) == 0) { _prefs->bridge_pkt_src = memcmp(&config[14], "rx", 2) == 0; +#ifdef WITH_MQTT_BRIDGE + if (_prefs->bridge_pkt_src == 1) { + _mqtt_prefs.mqtt_rx_enabled = 1; + _mqtt_prefs.mqtt_tx_enabled = 0; + } else { + _mqtt_prefs.mqtt_rx_enabled = 0; + _mqtt_prefs.mqtt_tx_enabled = 1; + } +#endif savePrefs(); strcpy(reply, "OK"); #endif @@ -799,13 +1497,14 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep } #endif } else { - strcpy(reply, "unknown config: "); - StrHelper::strncpy(&reply[16], config, 160-17); + sprintf(reply, "unknown config: %s", config); } } void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* reply) { const char* config = &command[4]; + // Observer/MQTT/WiFi/timezone/alert/SNMP commands live in CommonCLI_Observer.cpp. + if (handleObserverGetCmd(sender_timestamp, config, reply)) return; if (memcmp(config, "dutycycle", 9) == 0) { float dc = 100.0f / (_prefs->airtime_factor + 1.0f); int dc_int = (int)dc; @@ -817,6 +1516,12 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep sprintf(reply, "> %d", (uint32_t) _prefs->interference_threshold); } else if (memcmp(config, "cad", 3) == 0) { sprintf(reply, "> %s", _prefs->cad_enabled ? "on" : "off"); + } else if (memcmp(config, "radio.fem.rxgain", 16) == 0) { + if (!_board->canControlLoRaFemLna()) { + strcpy(reply, "Error: unsupported"); + } else { + sprintf(reply, "> %s", _board->isLoRaFemLnaEnabled() ? "on" : "off"); + } } else if (memcmp(config, "agc.reset.interval", 18) == 0) { sprintf(reply, "> %d", ((uint32_t) _prefs->agc_reset_interval) * 4); } else if (memcmp(config, "multi.acks", 10) == 0) { @@ -874,11 +1579,10 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep } else if (memcmp(config, "direct.txdelay", 14) == 0) { sprintf(reply, "> %s", StrHelper::ftoa(_prefs->direct_tx_delay_factor)); } else if (memcmp(config, "owner.info", 10) == 0) { - auto start = reply; *reply++ = '>'; *reply++ = ' '; const char* sp = _prefs->owner_info; - while (*sp && reply - start < 159) { + while (*sp) { *reply++ = (*sp == '\n') ? '|' : *sp; // translate newline back to orig '|' sp++; } diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 237c758e9f..e0a0a71b86 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -4,10 +4,11 @@ #include #include #include +#include // For MAX_MQTT_SLOTS (used in NodePrefs struct layout) #include #include -#if defined(WITH_RS232_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) +#if defined(WITH_RS232_BRIDGE) || defined(WITH_ESPNOW_BRIDGE) || defined(WITH_MQTT_BRIDGE) #define WITH_BRIDGE #endif @@ -36,7 +37,7 @@ class NodePrefs : public ConfigSerializer { float tx_delay_factor = 0; char guest_password[16]; float direct_tx_delay_factor = 0; - uint32_t guard; + uint32_t guard = 0; uint8_t sf = 0; uint8_t cr = 0; uint8_t allow_read_only = 0; @@ -50,7 +51,7 @@ class NodePrefs : public ConfigSerializer { // Bridge settings uint8_t bridge_enabled = 0; // boolean uint16_t bridge_delay = 0; // milliseconds (default 500 ms) - uint8_t bridge_pkt_src = 0; // 0 = logTx, 1 = logRx (default logTx) + uint8_t bridge_pkt_src = 0; // 0 = logTx, 1 = logRx (fresh installs default to logRx) uint32_t bridge_baud = 0; // 9600, 19200, 38400, 57600, 115200 (default 115200) uint8_t bridge_channel = 0; // 1-14 (ESP-NOW only) char bridge_secret[16]; // for XOR encryption of bridge packets (ESP-NOW only) @@ -64,13 +65,17 @@ class NodePrefs : public ConfigSerializer { float adc_multiplier = 0; char owner_info[120]; uint8_t rx_boosted_gain = 0; // power settings - uint8_t radio_fem_rxgain = 0; // LoRa FEM RX gain setting + uint8_t radio_fem_rxgain = 0; // LoRa FEM RX-gain (LNA); hardware driving is wired per-board uint8_t radio_fem_txgain = 0; // LoRa FEM TX gain setting uint8_t path_hash_mode = 0; // which path mode to use when sending uint8_t loop_detect = 0; uint8_t cad_enabled = 0; // hardware Channel Activity Detection before TX (boolean) uint8_t extra_sf[4]; + // NOTE: observer settings (MQTT/WiFi/timezone/SNMP/alert) are not in NodePrefs. + // They live in MQTTPrefs, persisted separately to /mqtt_prefs, so this struct + // stays aligned with upstream. See struct MQTTPrefs below. + private: class RadioPrefs : public ConfigSerializer { NodePrefs* _parent; @@ -105,7 +110,7 @@ class NodePrefs : public ConfigSerializer { void structure() override { def("en", _parent->bridge_enabled); // boolean def("delay", _parent->bridge_delay); // milliseconds (default 500 ms) - def("src", _parent->bridge_pkt_src); // 0 = logTx, 1 = logRx (default logTx) + def("src", _parent->bridge_pkt_src); // 0 = logTx, 1 = logRx def("baud", _parent->bridge_baud); // 9600, 19200, 38400, 57600, 115200 (default 115200) def("ch", _parent->bridge_channel); // 1-14 (ESP-NOW only) def("secret", _parent->bridge_secret, sizeof(_parent->bridge_secret)); // for XOR encryption of bridge packets (ESP-NOW only) @@ -176,6 +181,7 @@ class NodePrefs : public ConfigSerializer { def("f_adv_int", flood_advert_interval); def("lat", node_lat); def("lon", node_lon); + def("disc_mod", discovery_mod_timestamp); // gates 'since'-filtered DISCOVER replies def("radio", radio); def("bridge", bridge); def("gps", gps); @@ -194,6 +200,31 @@ class NodePrefs : public ConfigSerializer { } }; +#ifdef WITH_MQTT_BRIDGE +#include +static_assert(MQTT_PREFS_SLOT_COUNT == MAX_MQTT_SLOTS, + "MQTT prefs layout and slot count must change together"); + +// Observer settings captured from the trailing block of an old-format /com_prefs +// (fork firmware that predates the NodePrefs -> MQTTPrefs split). loadPrefsInt() +// fills this in when it detects the old file layout; loadMQTTPrefs() then applies +// the values one-time if the loaded /mqtt_prefs predates the appended observer +// fields, so SNMP/watchdog/alert config survives the firmware upgrade. +struct LegacyObserverTail { + bool valid = false; + uint8_t snmp_enabled; + char snmp_community[24]; + uint8_t radio_watchdog_minutes; + uint8_t alert_enabled; + char alert_psk_hex[33]; + uint16_t alert_wifi_minutes; + uint16_t alert_mqtt_minutes; + uint16_t alert_min_interval_min; + char alert_hashtag[24]; + char alert_region[31]; +}; +#endif + class CommonCLICallbacks { public: virtual void savePrefs() = 0; @@ -214,6 +245,7 @@ class CommonCLICallbacks { }; virtual void formatStatsReply(char *reply) = 0; virtual void formatRadioStatsReply(char *reply) = 0; + virtual void formatRadioDiagReply(char *reply) { strcpy(reply, "Not supported"); } virtual void formatPacketStatsReply(char *reply) = 0; virtual mesh::LocalIdentity& getSelfId() = 0; virtual void saveIdentity(const mesh::LocalIdentity& new_id) = 0; @@ -238,6 +270,49 @@ class CommonCLICallbacks { // no op by default }; + virtual void restartBridgeSlot(int slot) { + // Default: fall back to full restart + restartBridge(); + }; + + // Schedule a pull-OTA firmware update to run shortly (from the app loop), after + // the "Beginning update..." CLI reply has been transmitted. Deferred because the + // flash blocks the loop and then reboots, so it can't run inline with the reply. + // Returns true if scheduled. Default: not supported. + virtual bool beginDeferredOtaUpdate() { + return false; + }; + + virtual int getQueueSize() { + return 0; // no op by default + }; + + virtual bool syncMqttNtp() { + return false; // WITH_MQTT_BRIDGE builds override + }; + + virtual bool isMqttBridgeRunning() { + return false; + }; + + // Browser-based config portal (ESP32 WITH_MQTT_BRIDGE builds override). + // force_ap=true requests the SoftAP setup portal even when WiFi is configured. + // Returns true if handled (reply filled either way when true). + virtual bool startWebConfig(bool force_ap, char* reply) { + (void)force_ap; (void)reply; + return false; + }; + virtual bool stopWebConfig(char* reply) { + (void)reply; + return false; + }; + + // Probe all configured NTP servers for connectivity (verbose=serial console gets a + // detailed table; otherwise reply gets a compact " ok|fail" list). + virtual bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) { + return false; // WITH_MQTT_BRIDGE builds override + }; + virtual bool setRxBoostedGain(bool enable) { return false; // CommonCLI reports unsupported if not overridden by wrapper }; @@ -245,10 +320,34 @@ class CommonCLICallbacks { #if defined(USE_LR2021) virtual bool configSideDetectors(const uint8_t sideDetSFs[], uint8_t num, float bw) { return false; // Override in wrapper - } + } #endif + + // Fault-alert channel hooks (see NodePrefs::alert_*). The default no-op + // implementations keep CLI commands harmless on builds that don't wire up + // an AlertReporter. + virtual void onAlertConfigChanged() { + // no op by default + } + virtual bool sendAlertText(const char* /*text*/) { + return false; // no op by default + } + // Resolve the TransportKey scope to use for outgoing fault-alert floods. + // Implementations should consult NodePrefs::alert_region first (look up via + // RegionMap), then fall back to the repeater's default_scope, then return + // false if neither yields a usable key. AlertReporter falls back to an + // unscoped flood when this returns false. + virtual bool resolveAlertScope(TransportKey& /*dest*/) { + return false; // no op by default + } }; +#ifdef WITH_MQTT_BRIDGE +namespace MQTTPrefsAtomicStore { +class LegacyUpgradeGate; +} +#endif + class CommonCLI { mesh::RTCClock* _rtc; NodePrefs* _prefs; @@ -258,21 +357,50 @@ class CommonCLI { RegionMap* _region_map; ClientACL* _acl; char tmp[PRV_KEY_SIZE*2 + 4]; +#ifdef WITH_MQTT_BRIDGE + MQTTPrefs _mqtt_prefs; + LegacyObserverTail _legacy_tail; + // /mqtt_prefs is newer, corrupt, or temporarily unreadable. The in-memory prefs + // run on defaults and saveMQTTPrefs() must not overwrite the source file. + bool _mqtt_prefs_hold = false; +#endif + bool _com_prefs_needs_upgrade = false; // old-format legacy prefs detected; rewrite once after load mesh::RTCClock* getRTCClock() { return _rtc; } void savePrefs(); void loadPrefsInt(FILESYSTEM* _fs, const char* filename); +#ifdef WITH_MQTT_BRIDGE + void loadMQTTPrefs(FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade); + bool saveMQTTPrefs(FILESYSTEM* fs); +#endif void handleRegionCmd(char* command, char* reply); void handleGetCmd(uint32_t sender_timestamp, char* command, char* reply); void handleSetCmd(uint32_t sender_timestamp, char* command, char* reply); + // Observer/MQTT/WiFi/timezone/alert/SNMP CLI handling lives in the fork-owned + // CommonCLI_Observer.cpp to keep these branches out of the upstream-tracked + // CommonCLI.cpp. Each returns true if it recognized (handled) the command, or + // false to fall through to the base get/set parsing. + bool handleObserverSetCmd(uint32_t sender_timestamp, const char* config, char* reply); + bool handleObserverGetCmd(uint32_t sender_timestamp, const char* config, char* reply); + // Observer-only top-level commands (ota check/update, tls.bundletest, alert test) + // also live in CommonCLI_Observer.cpp; returns true if it handled the command. + bool handleObserverCommand(uint32_t sender_timestamp, char* command, char* reply); + public: CommonCLI(mesh::MainBoard& board, mesh::RTCClock& rtc, SensorManager& sensors, RegionMap& region_map, ClientACL& acl, NodePrefs* prefs, CommonCLICallbacks* callbacks) : _board(&board), _rtc(&rtc), _sensors(&sensors), _region_map(®ion_map), _acl(&acl), _prefs(prefs), _callbacks(callbacks) { } void loadPrefs(FILESYSTEM* _fs); - bool savePrefs(FILESYSTEM* _fs); + bool savePrefs(FILESYSTEM* _fs, bool save_mqtt = true); void handleCommand(uint32_t sender_timestamp, char* command, char* reply); + mesh::MainBoard* getBoard() { return _board; } uint8_t buildAdvertData(uint8_t node_type, uint8_t* app_data); +#ifdef WITH_MQTT_BRIDGE + // Observer config (MQTT/WiFi/timezone/SNMP/alert), persisted to /mqtt_prefs. + // Exposed so the app can hand it to MQTTBridge/AlertReporter, which read these + // fields directly (they no longer live in NodePrefs). + MQTTPrefs* getObserverPrefs() const { return const_cast(&_mqtt_prefs); } +#endif }; diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp new file mode 100644 index 0000000000..c1b7e74a11 --- /dev/null +++ b/src/helpers/CommonCLI_Observer.cpp @@ -0,0 +1,1180 @@ +// CommonCLI_Observer.cpp — fork-owned observer/MQTT/WiFi/timezone/alert/SNMP CLI +// command handling, split out of CommonCLI.cpp so the upstream-tracked file carries +// only two small delegation hooks. These are CommonCLI member functions, so they +// retain full access to _prefs/_callbacks/_board/savePrefs() with no re-plumbing. +// +// Behavior is intentionally identical to the previously-inlined branches. NOTE: +// the entire body of each set/get handler here is compiled under WITH_MQTT_BRIDGE +// (see the #ifdef at the top of each), so on an observer build without the bridge +// the WiFi/timezone/alert/SNMP commands compile out too — they are not guarded +// independently of the MQTT commands. Each handler returns true if it recognized +// the command, false to fall through to the base get/set parser in CommonCLI.cpp. + +#include +#include "CommonCLI.h" +#include "TxtDataHelpers.h" +#include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() +#include "MQTTObserverValidation.h" // pure input validators (host-testable) +#include +#ifdef ESP_PLATFORM +#include +#include +#include +#endif +#ifdef WITH_MQTT_BRIDGE +#include "bridges/MQTTBridge.h" +#include "MQTTConnectionPolicy.h" // classifySlotActivation() — "will this slot connect here?" +#include "MQTTDefaults.h" +#include "MQTTPacketFilter.h" +#endif + +// Local copy of the busted-libc-safe atoi (the original in CommonCLI.cpp is static). +static uint32_t _atoi(const char* sp) { + uint32_t n = 0; + while (*sp && *sp >= '0' && *sp <= '9') { + n *= 10; + n += (*sp++ - '0'); + } + return n; +} + +#ifdef ESP_PLATFORM +// Optional embedded CA bundle symbols produced by board_build.embed_files. +// Weak linkage keeps non-bundle builds linkable. +extern const uint8_t rootca_crt_bundle_start[] asm("_binary_src_certs_x509_crt_bundle_bin_start") __attribute__((weak)); +extern const uint8_t rootca_crt_bundle_end[] asm("_binary_src_certs_x509_crt_bundle_bin_end") __attribute__((weak)); + +static bool parseTlsBundleTarget(const char* input, char* host_out, size_t host_out_size, uint16_t* port_out) { + if (!input || !host_out || host_out_size == 0 || !port_out) return false; + + while (*input == ' ') input++; + if (*input == '\0') return false; + + const char* start = input; + const char* scheme = strstr(input, "://"); + if (scheme) start = scheme + 3; + + const char* end = start; + while (*end && *end != '/' && *end != '?' && *end != '#') end++; + if (end <= start) return false; + + uint16_t port = 443; + const char* host_start = start; + const char* host_end = end; + + if (*host_start == '[') { + const char* close = (const char*)memchr(host_start, ']', host_end - host_start); + if (!close) return false; + if ((close + 1) < host_end && *(close + 1) == ':') { + int p = atoi(close + 2); + if (p <= 0 || p > 65535) return false; + port = (uint16_t)p; + } + host_start++; + host_end = close; + } else { + const char* colon = (const char*)memchr(host_start, ':', host_end - host_start); + if (colon) { + int p = atoi(colon + 1); + if (p <= 0 || p > 65535) return false; + port = (uint16_t)p; + host_end = colon; + } + } + + size_t host_len = (size_t)(host_end - host_start); + if (host_len == 0 || host_len >= host_out_size) return false; + memcpy(host_out, host_start, host_len); + host_out[host_len] = '\0'; + *port_out = port; + return true; +} +#endif + +#ifdef WITH_MQTT_BRIDGE +static int getMQTTPresetNameCount() { + // Include virtual presets accepted by CLI parser. + return MQTT_PRESET_COUNT + 2; // built-ins + custom + none +} + +// Reject a value that wouldn't fit its destination MQTTPrefs buffer (which must +// hold the string plus a NUL) so an over-long CLI/web submission fails loudly +// instead of being silently truncated. Fills reply and returns true when too +// long. reply is the caller's 160-byte command buffer. +static bool valueTooLong(const char* val, size_t bufsize, char* reply, const char* label) { + if (!mqttValueFits(val, bufsize)) { + snprintf(reply, 160, "Error: %s too long (max %u chars)", label, (unsigned)(bufsize - 1)); + return true; + } + return false; +} + +static const char* getMQTTPresetNameByIndex(int index) { + if (index < MQTT_PRESET_COUNT) return MQTT_PRESETS[index].name; + if (index == MQTT_PRESET_COUNT) return MQTT_PRESET_CUSTOM; + if (index == MQTT_PRESET_COUNT + 1) return MQTT_PRESET_NONE; + return nullptr; +} + +static void formatMQTTPresetListReply(char* reply, size_t reply_size, int start) { + if (!reply || reply_size == 0) return; + reply[0] = '\0'; + + const int total = getMQTTPresetNameCount(); + if (start < 0 || start >= total) { + snprintf(reply, reply_size, "Error: preset list start must be 0-%d", total - 1); + return; + } + + // Keep room for continuation marker and null terminator. + const size_t reserve_for_next = 18; + size_t used = 0; + bool wrote_any = false; + + int index = start; + while (index < total) { + const char* name = getMQTTPresetNameByIndex(index); + if (!name) break; + size_t name_len = strlen(name); + size_t room = reply_size - used; + if (room <= reserve_for_next) break; + size_t needed = name_len + (wrote_any ? 1 : 0); // comma separator + if (needed >= room - reserve_for_next) break; + if (wrote_any) { + reply[used++] = ','; + } + memcpy(reply + used, name, name_len); + used += name_len; + reply[used] = '\0'; + wrote_any = true; + index++; + } + + if (!wrote_any) { + strcpy(reply, "Error: list page too small"); + return; + } + + if (index < total) { + snprintf(reply + used, reply_size - used, "... next:%d", index); + } +} +#endif + +bool CommonCLI::handleObserverSetCmd(uint32_t sender_timestamp, const char* config, char* reply) { +#ifdef WITH_MQTT_BRIDGE + bool handled = true; + if (memcmp(config, "snmp.community ", 15) == 0) { + if (valueTooLong(&config[15], sizeof(_mqtt_prefs.snmp_community), reply, "snmp.community")) return true; + StrHelper::strncpy(_mqtt_prefs.snmp_community, &config[15], sizeof(_mqtt_prefs.snmp_community)); + savePrefs(); + strcpy(reply, "OK - restart to apply"); + } else if (memcmp(config, "snmp ", 5) == 0) { + _mqtt_prefs.snmp_enabled = memcmp(&config[5], "on", 2) == 0; + savePrefs(); + strcpy(reply, "OK - restart to apply"); + } else if (memcmp(config, "radio.watchdog ", 15) == 0) { + const char* val = &config[15]; + bool all_digits = (*val != '\0'); + for (const char* sp = val; *sp; sp++) { + if (*sp < '0' || *sp > '9') { all_digits = false; break; } + } + if (*val == '\0') { + strcpy(reply, "Error: missing radio.watchdog minutes"); + } else if (!all_digits) { + strcpy(reply, "Error: radio.watchdog must be an integer 0-120"); + } else { + int mins = atoi(val); + if (mins > 120) { + strcpy(reply, "Error: radio.watchdog must be 0-120 minutes"); + } else { + _mqtt_prefs.radio_watchdog_minutes = (uint8_t)mins; + savePrefs(); + if (mins == 0) { + strcpy(reply, "OK - radio watchdog disabled"); + } else { + sprintf(reply, "OK - radio watchdog %d min", mins); + } + } + } +#ifdef WITH_MQTT_BRIDGE + } else if (strcmp(config, "mqtt.origin") == 0) { + _mqtt_prefs.mqtt_origin[0] = '\0'; + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.origin ", 12) == 0) { + if (valueTooLong(&config[12], sizeof(_mqtt_prefs.mqtt_origin), reply, "origin")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_origin, &config[12], sizeof(_mqtt_prefs.mqtt_origin)); + StrHelper::stripSurroundingQuotes(_mqtt_prefs.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.iata ", 10) == 0) { + const char* iata = &config[10]; + size_t iata_len = strlen(iata); + if (iata_len == 0) { + // Empty clears the region code (meshcore-topic publishing stays disabled + // until one is set). This keeps the pre-existing "clear IATA" capability. + _mqtt_prefs.mqtt_iata[0] = '\0'; + savePrefs(); + _callbacks->restartBridge(); + strcpy(reply, "OK - IATA cleared"); + } else { + // A region code goes straight into MQTT topic paths, so require exactly + // three alphanumeric characters (real IATA codes are 3 letters, e.g. DEN). + if (!mqttIataValid(iata)) { + strcpy(reply, "Error: IATA code must be exactly 3 letters/digits (e.g. DEN)"); + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_iata, iata, sizeof(_mqtt_prefs.mqtt_iata)); + for (int i = 0; _mqtt_prefs.mqtt_iata[i]; i++) { + _mqtt_prefs.mqtt_iata[i] = toupper(_mqtt_prefs.mqtt_iata[i]); + } + savePrefs(); + _callbacks->restartBridge(); + strcpy(reply, "OK"); + } + } + } else if (memcmp(config, "mqtt.status ", 12) == 0) { + _mqtt_prefs.mqtt_status_enabled = memcmp(&config[12], "on", 2) == 0; + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.packets ", 13) == 0) { + _mqtt_prefs.mqtt_packets_enabled = memcmp(&config[13], "on", 2) == 0; + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.raw ", 9) == 0) { + _mqtt_prefs.mqtt_raw_enabled = memcmp(&config[9], "on", 2) == 0; + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.tx ", 8) == 0) { + if (memcmp(&config[8], "advert", 6) == 0) { + _mqtt_prefs.mqtt_tx_enabled = 2; + } else { + _mqtt_prefs.mqtt_tx_enabled = memcmp(&config[8], "on", 2) == 0 ? 1 : 0; + } + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.rx ", 8) == 0) { + _mqtt_prefs.mqtt_rx_enabled = memcmp(&config[8], "on", 2) == 0 ? 1 : 0; + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.interval ", 14) == 0) { + uint32_t minutes = _atoi(&config[14]); + if (minutes >= 1 && minutes <= 60) { + _mqtt_prefs.mqtt_status_interval = minutes * 60000; + savePrefs(); + _callbacks->restartBridge(); + sprintf(reply, "OK - interval set to %u minutes (%lu ms), bridge restarted", minutes, (unsigned long)_mqtt_prefs.mqtt_status_interval); + } else { + strcpy(reply, "Error: interval must be between 1-60 minutes"); + } +#if defined(WITH_MQTT_NEIGHBORS) + } else if (memcmp(config, "mqtt.neighbors.interval ", 24) == 0) { + // Hours in, milliseconds stored. The 12-336h band keeps the interval under + // INT32_MAX so the mesh's wrap-safe signed-delta millis math stays valid. + uint32_t hours = _atoi(&config[24]); + if (hours >= MQTT_NEIGHBORS_MIN_INTERVAL_HOURS && hours <= MQTT_NEIGHBORS_MAX_INTERVAL_HOURS) { + _mqtt_prefs.mqtt_neighbors_interval = hours * 3600000UL; + savePrefs(); + sprintf(reply, "OK - neighbors interval set to %u hours (%lu ms)", (unsigned)hours, + (unsigned long)_mqtt_prefs.mqtt_neighbors_interval); + } else { + strcpy(reply, "Error: neighbors interval must be between 12-336 hours"); + } + } else if (memcmp(config, "mqtt.neighbors ", 15) == 0) { + // The mesh loop reads this live, so no bridge restart is needed; enabling it + // triggers a discovery on the next eligible loop pass. + _mqtt_prefs.mqtt_neighbors_enabled = memcmp(&config[15], "on", 2) == 0; + savePrefs(); + strcpy(reply, "OK"); +#elif defined(WITH_MQTT_BRIDGE) + } else if (memcmp(config, "mqtt.neighbors.interval ", 24) == 0 || + memcmp(config, "mqtt.neighbors ", 15) == 0) { + strcpy(reply, "Err - neighbors not enabled in this build"); +#endif + } else if (memcmp(config, "mqtt.ntp ", 9) == 0) { + const char* host = &config[9]; + while (*host == ' ') host++; + bool clearing = strcmp(host, "none") == 0; + if (!clearing && !mqttNtpHostnameValid(host)) { + strcpy(reply, "Error: invalid NTP hostname"); + } else { + if (clearing) { + _mqtt_prefs.mqtt_ntp_server[0] = '\0'; + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_ntp_server, host, sizeof(_mqtt_prefs.mqtt_ntp_server)); + } + savePrefs(); +#ifdef ESP_PLATFORM + // Queue a sync on the MQTT task (Core 0) but do NOT block: this handler + // runs on the Arduino loop task, shared with mesh/radio processing and the + // web config batch, so a synchronous wait of up to 30 s would stall the + // node. The sync runs in the background; verify with `get mqtt.ntp.diag`. + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "OK - saved (WiFi not connected; NTP sync pending)"); + } else if (!_callbacks->isMqttBridgeRunning()) { + strcpy(reply, "OK - saved (MQTT bridge not running)"); + } else if (_callbacks->syncMqttNtp()) { + strcpy(reply, "OK - saved (NTP sync started; check 'get mqtt.ntp.diag')"); + } else { + strcpy(reply, "OK - saved (NTP sync unavailable)"); + } +#else + strcpy(reply, "OK - saved"); +#endif + } + } else if (memcmp(config, "wifi.ssid ", 10) == 0) { + if (valueTooLong(&config[10], sizeof(_mqtt_prefs.wifi_ssid), reply, "wifi.ssid")) return true; + StrHelper::strncpy(_mqtt_prefs.wifi_ssid, &config[10], sizeof(_mqtt_prefs.wifi_ssid)); + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "wifi.pwd ", 9) == 0) { + if (valueTooLong(&config[9], sizeof(_mqtt_prefs.wifi_password), reply, "wifi.pwd")) return true; + StrHelper::strncpy(_mqtt_prefs.wifi_password, &config[9], sizeof(_mqtt_prefs.wifi_password)); + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "wifi.powersave ", 15) == 0) { + const char* value = &config[15]; + uint8_t ps_value; + bool valid = false; + if (memcmp(value, "min", 3) == 0 && (value[3] == 0 || value[3] == ' ')) { + ps_value = 0; + valid = true; + } else if (memcmp(value, "none", 4) == 0 && (value[4] == 0 || value[4] == ' ')) { + ps_value = 1; + valid = true; + } else if (memcmp(value, "max", 3) == 0 && (value[3] == 0 || value[3] == ' ')) { + ps_value = 2; + valid = true; + } + if (!valid) { + strcpy(reply, "Error: must be none, min, or max"); + } else { + _mqtt_prefs.wifi_power_save = ps_value; + savePrefs(); +#ifdef ESP_PLATFORM + if (WiFi.status() == WL_CONNECTED) { + wifi_ps_type_t ps_mode = (ps_value == 1) ? WIFI_PS_NONE : + (ps_value == 2) ? WIFI_PS_MAX_MODEM : WIFI_PS_MIN_MODEM; + esp_err_t ps_result = esp_wifi_set_ps(ps_mode); + if (ps_result == ESP_OK) { + const char* ps_name = (ps_value == 1) ? "none" : (ps_value == 2) ? "max" : "min"; + sprintf(reply, "OK - power save set to %s", ps_name); + } else { + sprintf(reply, "OK - saved, but failed to apply: %d", ps_result); + } + } else { + const char* ps_name = (ps_value == 1) ? "none" : (ps_value == 2) ? "max" : "min"; + sprintf(reply, "OK - saved as %s (will apply on next WiFi connection)", ps_name); + } +#else + const char* ps_name = (ps_value == 1) ? "none" : (ps_value == 2) ? "max" : "min"; + sprintf(reply, "OK - saved as %s", ps_name); +#endif + } + } else if (memcmp(config, "timezone ", 9) == 0) { + if (valueTooLong(&config[9], sizeof(_mqtt_prefs.timezone_string), reply, "timezone")) return true; + StrHelper::strncpy(_mqtt_prefs.timezone_string, &config[9], sizeof(_mqtt_prefs.timezone_string)); + savePrefs(); + strcpy(reply, "OK"); + } else if (memcmp(config, "timezone.offset ", 16) == 0) { + int8_t offset = _atoi(&config[16]); + if (offset >= -12 && offset <= 14) { + _mqtt_prefs.timezone_offset = offset; + savePrefs(); + strcpy(reply, "OK"); + } else { + strcpy(reply, "Error: timezone offset must be between -12 and +14"); + } + } else if (config[0] == 'm' && config[1] == 'q' && config[2] == 't' && config[3] == 't' && + config[4] >= '1' && config[4] <= ('0' + MAX_MQTT_SLOTS) && config[5] == '.') { + // Slot-based commands: set mqtt1.preset , set mqtt1.server , etc. + int slot = config[4] - '1'; // 0-5 + const char* subcmd = &config[6]; + if (memcmp(subcmd, "preset ", 7) == 0) { + const char* preset_name = &subcmd[7]; + // Validate preset name + if (findMQTTPreset(preset_name) != nullptr || + strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0 || + strcmp(preset_name, MQTT_PRESET_NONE) == 0) { + // Reject duplicate presets (except "none" and "custom") + int dup_slot = -1; + if (findMQTTPreset(preset_name) != nullptr) { + for (int s = 0; s < MAX_MQTT_SLOTS; s++) { + if (s != slot && strcmp(_mqtt_prefs.mqtt_slot_preset[s], preset_name) == 0) { + dup_slot = s; + break; + } + } + } + if (dup_slot >= 0) { + sprintf(reply, "Error: preset '%s' is already assigned to slot %d", preset_name, dup_slot + 1); + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], preset_name, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); + savePrefs(); + _callbacks->restartBridgeSlot(slot); + // Check if the slot has everything it needs to connect + const MQTTPresetDef* p = findMQTTPreset(preset_name); + if (p && p->topic_style == MQTT_TOPIC_MESHRANK && _mqtt_prefs.mqtt_slot_token[slot][0] == '\0') { + sprintf(reply, "OK - slot %d preset: %s (run 'set mqtt%d.token ' to connect)", slot + 1, preset_name, slot + 1); + } else if (p && p->topic_style == MQTT_TOPIC_MESHCORE && + (strlen(_mqtt_prefs.mqtt_iata) == 0 || strcmp(_mqtt_prefs.mqtt_iata, "XXX") == 0)) { + sprintf(reply, "OK - slot %d preset: %s (run 'set mqtt.iata ' to publish)", slot + 1, preset_name); + } else if (p && mqttPresetNeedsSlotPassword(p) && + _mqtt_prefs.mqtt_slot_password[slot][0] == '\0' && + !mqttPresetNeedsSlotUsername(p)) { + sprintf(reply, + "OK - slot %d preset: %s (run 'set mqtt%d.password ' to connect)", + slot + 1, preset_name, slot + 1); + } else if (p && mqttPresetNeedsSlotCredentials(p) && + (_mqtt_prefs.mqtt_slot_username[slot][0] == '\0' || + _mqtt_prefs.mqtt_slot_password[slot][0] == '\0')) { + sprintf(reply, + "OK - slot %d preset: %s (run 'set mqtt%d.username ' and 'set mqtt%d.password ' to connect)", + slot + 1, preset_name, slot + 1, slot + 1); + } else { + sprintf(reply, "OK - slot %d preset: %s", slot + 1, preset_name); + } + // Warn when this slot won't actually connect on this hardware. The set + // is never blocked — prefs persist so the config carries over if the + // device is moved to a board with more slots — but flag it, or the + // operator waits for a connection that never comes (A15). Two failure + // modes, keyed off the same rule the bridge's setup loop uses + // (classifySlotActivation): slots past the runtime array are never + // tried; slots within it are skipped once more than getMaxActiveSlots() + // are enabled (each WSS/TLS link costs ~40 KB heap). + if (strcmp(preset_name, MQTT_PRESET_NONE) != 0) { + bool slot_enabled[MAX_MQTT_SLOTS]; + for (int s = 0; s < MAX_MQTT_SLOTS; s++) { + slot_enabled[s] = _mqtt_prefs.mqtt_slot_preset[s][0] != '\0' && + strcmp(_mqtt_prefs.mqtt_slot_preset[s], MQTT_PRESET_NONE) != 0; + } + const int max_active = MQTTBridge::getMaxActiveSlots(); + const MQTTConnectionPolicy::SlotActivation act = + MQTTConnectionPolicy::classifySlotActivation(slot, slot_enabled, + RUNTIME_MQTT_SLOTS, max_active); + size_t used = strlen(reply); + if (used < 158) { + if (act == MQTTConnectionPolicy::SlotActivation::BeyondArray) { + snprintf(reply + used, 160 - used, " (slot inactive on this hardware)"); + } else if (act == MQTTConnectionPolicy::SlotActivation::OverActiveCap) { + snprintf(reply + used, 160 - used, + " (won't connect: %d-slot limit on this hardware)", max_active); + } + } + } + } + } else { + strcpy(reply, "Error: unknown preset. Use 'get mqtt.presets'"); + } + } else if (memcmp(subcmd, "server ", 7) == 0) { + if (valueTooLong(&subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot]), reply, "server")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_host[slot], &subcmd[7], sizeof(_mqtt_prefs.mqtt_slot_host[slot])); + savePrefs(); + // Reconfigure the slot so the new host reaches the live connection (other + // custom-slot setters do the same; without it the change only applies on + // the next reboot/bridge restart). + _callbacks->restartBridgeSlot(slot); + strcpy(reply, "OK"); + } else if (memcmp(subcmd, "port ", 5) == 0) { + int port = atoi(&subcmd[5]); + if (port > 0 && port <= 65535) { + _mqtt_prefs.mqtt_slot_port[slot] = port; + savePrefs(); + _callbacks->restartBridgeSlot(slot); + strcpy(reply, "OK"); + } else { + strcpy(reply, "Error: port must be between 1 and 65535"); + } + } else if (memcmp(subcmd, "username ", 9) == 0) { + if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_username[slot]), reply, "username")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_username[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_username[slot])); + savePrefs(); + _callbacks->restartBridgeSlot(slot); + strcpy(reply, "OK"); + } else if (memcmp(subcmd, "password ", 9) == 0) { + if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_password[slot]), reply, "password")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_password[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_password[slot])); + savePrefs(); + _callbacks->restartBridgeSlot(slot); + strcpy(reply, "OK"); + } else if (memcmp(subcmd, "token ", 6) == 0) { + if (valueTooLong(&subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_token[slot]), reply, "token")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_token[slot], &subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_token[slot])); + savePrefs(); + _callbacks->restartBridgeSlot(slot); + sprintf(reply, "OK - slot %d token set", slot + 1); + } else if (memcmp(subcmd, "topic ", 6) == 0) { + if (strcmp(_mqtt_prefs.mqtt_slot_preset[slot], "custom") != 0) { + sprintf(reply, "Error: topic template only applies to custom preset slots"); + } else if (valueTooLong(&subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_topic[slot]), reply, "topic")) { + return true; + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_topic[slot], &subcmd[6], sizeof(_mqtt_prefs.mqtt_slot_topic[slot])); + savePrefs(); + _callbacks->restartBridgeSlot(slot); + sprintf(reply, "OK - slot %d topic: %s", slot + 1, _mqtt_prefs.mqtt_slot_topic[slot]); + } + } else if (memcmp(subcmd, "audience ", 9) == 0) { + if (valueTooLong(&subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_audience[slot]), reply, "audience")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_audience[slot], &subcmd[9], sizeof(_mqtt_prefs.mqtt_slot_audience[slot])); + savePrefs(); + _callbacks->restartBridgeSlot(slot); + if (_mqtt_prefs.mqtt_slot_audience[slot][0] != '\0') { + sprintf(reply, "OK - slot %d JWT audience: %s", slot + 1, _mqtt_prefs.mqtt_slot_audience[slot]); + } else { + sprintf(reply, "OK - slot %d JWT audience cleared (using username/password auth)", slot + 1); + } + } else if (memcmp(subcmd, "audience", 8) == 0 && subcmd[8] == '\0') { + // "set mqttN.audience" with no value — clear the audience + _mqtt_prefs.mqtt_slot_audience[slot][0] = '\0'; + savePrefs(); + _callbacks->restartBridgeSlot(slot); + sprintf(reply, "OK - slot %d JWT audience cleared (using username/password auth)", slot + 1); + } else if (strcmp(subcmd, "filter") == 0 || + strncmp(subcmd, "filter ", 7) == 0) { + // Empty/bare input resets to the backwards-compatible all-types default. + const char* filter_value = subcmd[6] == '\0' ? "" : &subcmd[7]; + uint16_t filter_mask = 0; + if (!MQTTPacketFilter::parse(filter_value, &filter_mask)) { + strcpy(reply, "Error: filter must be all, none, or a CSV of types 0-15 / names (advert,txt_msg,...)"); + } else { + _mqtt_prefs.mqtt_slot_packet_filter[slot] = filter_mask; + savePrefs(); + char filter_text[MQTTPacketFilter::kFilterTextSize]; + MQTTPacketFilter::format(filter_mask, filter_text, sizeof(filter_text)); + snprintf(reply, 160, "OK - slot %d packet types: %s", slot + 1, filter_text); + // A non-default filter extends /mqtt_prefs past what pre-filter + // firmware can read (see MQTTPrefsCodec::payloadLenFor), so say when + // that cost buys nothing: slots beyond the runtime array are never + // published to on this board, the same warning `preset` gives. + if (slot >= RUNTIME_MQTT_SLOTS && + filter_mask != MQTTPacketFilter::kAllPacketTypes) { + size_t used = strlen(reply); + if (used < 158) { + snprintf(reply + used, 160 - used, + " (slot inactive on this hardware; blocks firmware rollback)"); + } + } + } + } else { + sprintf(reply, "unknown config: %s", config); + } + } else if (memcmp(config, "mqtt.analyzer.us ", 17) == 0) { + const int slot = 0; + if (memcmp(&config[17], "on", 2) == 0) { + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], "analyzer-us", sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], MQTT_PRESET_NONE, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); + } + savePrefs(); + _callbacks->restartBridgeSlot(slot); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.analyzer.eu ", 17) == 0) { + const int slot = 1; + if (memcmp(&config[17], "on", 2) == 0) { + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], "analyzer-eu", sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); + } else { + StrHelper::strncpy(_mqtt_prefs.mqtt_slot_preset[slot], MQTT_PRESET_NONE, sizeof(_mqtt_prefs.mqtt_slot_preset[slot])); + } + savePrefs(); + _callbacks->restartBridgeSlot(slot); + strcpy(reply, "OK"); + } else if (memcmp(config, "mqtt.owner ", 11) == 0) { + const char* owner_key = &config[11]; + if (owner_key[0] == '\0') { + // Owner key is optional — empty clears it (previously this errored, so a + // set key could never be removed via the portal/CLI). + _mqtt_prefs.mqtt_owner_public_key[0] = '\0'; + savePrefs(); + strcpy(reply, "OK - owner key cleared"); + } else if (mqttOwnerKeyValid(owner_key)) { + StrHelper::strncpy(_mqtt_prefs.mqtt_owner_public_key, owner_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); + savePrefs(); + strcpy(reply, "OK"); + } else { + strcpy(reply, "Error: public key must be 64 hex characters (32 bytes)"); + } + } else if (memcmp(config, "mqtt.email ", 11) == 0) { + if (valueTooLong(&config[11], sizeof(_mqtt_prefs.mqtt_email), reply, "email")) return true; + StrHelper::strncpy(_mqtt_prefs.mqtt_email, &config[11], sizeof(_mqtt_prefs.mqtt_email)); + savePrefs(); + strcpy(reply, "OK"); +#endif + } else if (memcmp(config, "alert ", 6) == 0) { + // set alert on|off + const char* val = &config[6]; + if (memcmp(val, "on", 2) == 0 && (val[2] == 0 || val[2] == ' ')) { + _mqtt_prefs.alert_enabled = 1; + savePrefs(); + _callbacks->onAlertConfigChanged(); + strcpy(reply, "OK - alerts on"); + } else if (memcmp(val, "off", 3) == 0 && (val[3] == 0 || val[3] == ' ')) { + _mqtt_prefs.alert_enabled = 0; + savePrefs(); + _callbacks->onAlertConfigChanged(); + strcpy(reply, "OK - alerts off"); + } else { + strcpy(reply, "Error: usage set alert on|off"); + } + } else if (memcmp(config, "alert.psk", 9) == 0 && (config[9] == 0 || config[9] == ' ')) { + // `set alert.psk` with no argument clears the field (alerts then disabled + // until a new psk/hashtag is configured). + const char* val = (config[9] == ' ') ? &config[10] : ""; + while (*val == ' ') val++; + size_t len = strlen(val); + if (len == 0) { + _mqtt_prefs.alert_psk_hex[0] = '\0'; + _mqtt_prefs.alert_hashtag[0] = '\0'; + savePrefs(); + _callbacks->onAlertConfigChanged(); + strcpy(reply, "OK - alert.psk cleared (alerts disabled until configured)"); + } else if (val[0] == '#') { + strcpy(reply, "Error: use 'set alert.hashtag' for hashtag channels"); + } else if (len != 32) { + // 16-byte channel secret = 32 hex chars. This is what the mobile app's + // "Share Channel" emits, what `set alert.hashtag` derives, and what the + // BANNED_ALERT_CHANNELS table holds. 32-byte channels aren't used + // anywhere in MeshCore practice. + strcpy(reply, "Error: PSK must be 32 hex chars (16-byte channel secret)"); + } else { + // Validate all-hex, then normalize via fromHex/toHex so the stored + // form is always lowercase regardless of input case. + uint8_t raw[16]; + bool all_hex = true; + for (size_t i = 0; i < len; i++) { + if (!mesh::Utils::isHexChar(val[i])) { all_hex = false; break; } + } + if (!all_hex || !mesh::Utils::fromHex(raw, 16, val)) { + strcpy(reply, "Error: PSK must be 32 hex chars (16-byte channel secret)"); + } else { + char normalized[33]; + mesh::Utils::toHex(normalized, raw, 16); + if (const char* banned = alertReporterBannedChannelMatchHex(normalized)) { + // Refuse any key on the banned channel list (Public PSK, well-known + // auto-responder hashtags like #test/#bot, etc.). Fault alerts on + // those channels would spam every node in the area. + sprintf(reply, "Error: refusing banned channel '%s'; pick a private key or hashtag", banned); + } else { + StrHelper::strncpy(_mqtt_prefs.alert_psk_hex, normalized, sizeof(_mqtt_prefs.alert_psk_hex)); + // The new PSK is operator-supplied, so any previously-derived + // hashtag name is no longer accurate provenance — drop it. + _mqtt_prefs.alert_hashtag[0] = '\0'; + savePrefs(); + _callbacks->onAlertConfigChanged(); + strcpy(reply, "OK - alert.psk updated"); + } + } + } + } else if (memcmp(config, "alert.hashtag", 13) == 0 && (config[13] == 0 || config[13] == ' ')) { + const char* val = (config[13] == ' ') ? &config[14] : ""; + while (*val == ' ') val++; + size_t in_len = strlen(val); + if (in_len == 0) { + _mqtt_prefs.alert_psk_hex[0] = '\0'; + _mqtt_prefs.alert_hashtag[0] = '\0'; + savePrefs(); + _callbacks->onAlertConfigChanged(); + strcpy(reply, "OK - alert.hashtag cleared (alerts disabled until configured)"); + } else { + // Canonical stored form is "#name" because the leading '#' is part of + // the sha256 input (matching the companion-app hashtag-channel + // derivation in docs/companion_protocol.md). Accept the user typing + // either "alerts" or "#alerts". + char hashtag[sizeof(_mqtt_prefs.alert_hashtag)]; + size_t need = (val[0] == '#') ? in_len : in_len + 1; + if (need >= sizeof(hashtag)) { + strcpy(reply, "Error: hashtag too long"); + } else { + if (val[0] == '#') { + StrHelper::strncpy(hashtag, val, sizeof(hashtag)); + } else { + hashtag[0] = '#'; + StrHelper::strncpy(&hashtag[1], val, sizeof(hashtag) - 1); + } + + // Derive the channel key once: first 16 bytes of sha256("#name"), + // store hex-encoded in alert_psk_hex. We don't re-derive on every + // send — operators can later override with `set alert.psk` without + // leaving stale hashtag text behind. + uint8_t digest[32]; + mesh::Utils::sha256(digest, sizeof(digest), + (const uint8_t*)hashtag, (int)strlen(hashtag)); + if (const char* banned = alertReporterBannedChannelMatch(digest)) { + // Hashtag derives to a banned key (e.g. `set alert.hashtag test` + // hits the #test entry). Refuse before clobbering existing config. + sprintf(reply, "Error: refusing banned channel '%s'", banned); + } else { + char hex[33]; + mesh::Utils::toHex(hex, digest, 16); + StrHelper::strncpy(_mqtt_prefs.alert_hashtag, hashtag, sizeof(_mqtt_prefs.alert_hashtag)); + StrHelper::strncpy(_mqtt_prefs.alert_psk_hex, hex, sizeof(_mqtt_prefs.alert_psk_hex)); + savePrefs(); + _callbacks->onAlertConfigChanged(); + sprintf(reply, "OK - alert.hashtag: %s", _mqtt_prefs.alert_hashtag); + } + } + } + } else if (memcmp(config, "alert.region", 12) == 0 && (config[12] == 0 || config[12] == ' ')) { + // `set alert.region ` overrides the repeater's default_scope for + // alert sends only. `set alert.region` (no arg) clears it. The name is + // looked up lazily via RegionMap at send time; we deliberately don't + // mutate the region map here, so naming an unknown region is allowed + // but will silently fall back to default_scope until the operator runs + // `region put` for it. + const char* val = (config[12] == ' ') ? &config[13] : ""; + while (*val == ' ') val++; + size_t len = strlen(val); + if (len == 0) { + _mqtt_prefs.alert_region[0] = '\0'; + savePrefs(); + _callbacks->onAlertConfigChanged(); + strcpy(reply, "OK - alert.region cleared (using default scope)"); + } else if (len >= sizeof(_mqtt_prefs.alert_region)) { + strcpy(reply, "Error: alert.region too long"); + } else { + StrHelper::strncpy(_mqtt_prefs.alert_region, val, sizeof(_mqtt_prefs.alert_region)); + StrHelper::stripSurroundingQuotes(_mqtt_prefs.alert_region, sizeof(_mqtt_prefs.alert_region)); + savePrefs(); + _callbacks->onAlertConfigChanged(); + sprintf(reply, "OK - alert.region: %s", _mqtt_prefs.alert_region); + } + } else if (memcmp(config, "alert.wifi ", 11) == 0) { + int mins = (int)_atoi(&config[11]); + if (mins < 0 || mins > 1440) { + strcpy(reply, "Error: alert.wifi must be 0-1440 minutes (0=off)"); + } else { + _mqtt_prefs.alert_wifi_minutes = (uint16_t)mins; + savePrefs(); + sprintf(reply, "OK - alert.wifi %d min%s", mins, mins == 0 ? " (disabled)" : ""); + } + } else if (memcmp(config, "alert.mqtt ", 11) == 0) { + int mins = (int)_atoi(&config[11]); + if (mins < 0 || mins > 10080) { + strcpy(reply, "Error: alert.mqtt must be 0-10080 minutes (0=off)"); + } else { + _mqtt_prefs.alert_mqtt_minutes = (uint16_t)mins; + savePrefs(); + sprintf(reply, "OK - alert.mqtt %d min%s", mins, mins == 0 ? " (disabled)" : ""); + } + } else if (memcmp(config, "alert.interval ", 15) == 0) { + int mins = (int)_atoi(&config[15]); + // Floor at 60 min: faster re-fires would let a flapping link spam the + // mesh with a fresh GRP_TXT flood every minute — terrible for airtime. + if (mins < 60 || mins > 10080) { + strcpy(reply, "Error: alert.interval must be 60-10080 minutes"); + } else { + _mqtt_prefs.alert_min_interval_min = (uint16_t)mins; + savePrefs(); + sprintf(reply, "OK - alert.interval %d min", mins); + } + } else { + handled = false; + } + return handled; +#else + (void)sender_timestamp; (void)config; (void)reply; + return false; +#endif +} + +bool CommonCLI::handleObserverGetCmd(uint32_t sender_timestamp, const char* config, char* reply) { +#ifdef WITH_MQTT_BRIDGE + bool handled = true; + if (memcmp(config, "snmp.community", 14) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.snmp_community); + } else if (memcmp(config, "snmp", 4) == 0 && (config[4] == '\0' || config[4] == '\n' || config[4] == '\r')) { + strcpy(reply, _mqtt_prefs.snmp_enabled ? "> on" : "> off"); + } else if (memcmp(config, "radio.watchdog", 14) == 0) { + sprintf(reply, "> %d", (uint32_t)_mqtt_prefs.radio_watchdog_minutes); +#ifdef WITH_MQTT_BRIDGE + } else if (memcmp(config, "mqtt.origin", 11) == 0) { + char effective_origin[32]; + MQTTBridge::getEffectiveMqttOrigin(_prefs, &_mqtt_prefs, effective_origin, sizeof(effective_origin)); + sprintf(reply, "> %s", effective_origin); + } else if (memcmp(config, "mqtt.iata", 9) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_iata); + } else if (memcmp(config, "mqtt.presets", 12) == 0 && (config[12] == '\0' || config[12] == ' ')) { + int start = 0; + if (config[12] == ' ') { + const char* start_arg = &config[13]; + if (*start_arg == '\0') { + strcpy(reply, "Error: usage get mqtt.presets [start]"); + return true; + } + for (const char* sp = start_arg; *sp; sp++) { + if (*sp < '0' || *sp > '9') { + strcpy(reply, "Error: usage get mqtt.presets [start]"); + return true; + } + } + start = (int)_atoi(start_arg); + } + formatMQTTPresetListReply(reply, 160, start); + } else if (memcmp(config, "mqtt.stats", 10) == 0) { + MQTTBridge::formatMqttStatsReply(reply, 160); + } else if (memcmp(config, "mqtt.status", 11) == 0) { + MQTTBridge::formatMqttStatusReply(reply, 160, &_mqtt_prefs); + } else if (memcmp(config, "mqtt.packets", 12) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_packets_enabled ? "on" : "off"); + } else if (memcmp(config, "mqtt.raw", 8) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_raw_enabled ? "on" : "off"); + } else if (memcmp(config, "mqtt.tx", 7) == 0) { + const char* tx_str = _mqtt_prefs.mqtt_tx_enabled == 2 ? "advert" : (_mqtt_prefs.mqtt_tx_enabled ? "on" : "off"); + sprintf(reply, "> %s", tx_str); + } else if (memcmp(config, "mqtt.rx", 7) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_rx_enabled ? "on" : "off"); + } else if (memcmp(config, "mqtt.interval", 13) == 0) { + uint32_t minutes = (_mqtt_prefs.mqtt_status_interval + 29999) / 60000; + sprintf(reply, "> %u minutes (%lu ms)", minutes, (unsigned long)_mqtt_prefs.mqtt_status_interval); +#if defined(WITH_MQTT_NEIGHBORS) + // Longer token first: a bare "mqtt.neighbors" (14) would otherwise swallow + // "mqtt.neighbors.interval" since the GET tokens carry no trailing space. + } else if (memcmp(config, "mqtt.neighbors.interval", 23) == 0) { + uint32_t hours = (_mqtt_prefs.mqtt_neighbors_interval + 3599999) / 3600000; + sprintf(reply, "> %u hours (%lu ms)", (unsigned)hours, (unsigned long)_mqtt_prefs.mqtt_neighbors_interval); + } else if (memcmp(config, "mqtt.neighbors", 14) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_neighbors_enabled ? "on" : "off"); +#elif defined(WITH_MQTT_BRIDGE) + } else if (memcmp(config, "mqtt.neighbors.interval", 23) == 0 || + memcmp(config, "mqtt.neighbors", 14) == 0) { + strcpy(reply, "Err - neighbors not enabled in this build"); +#endif + } else if (memcmp(config, "mqtt.ntp.diag", 13) == 0 && (config[13] == '\0' || config[13] == ' ')) { +#ifdef ESP_PLATFORM + // Connectivity probe across all configured NTP servers; never updates the clock. + // Serial console (sender_timestamp == 0) gets a detailed table; LoRa gets a compact list. + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "Error: WiFi not connected"); + } else if (!_callbacks->isMqttBridgeRunning()) { + strcpy(reply, "Error: MQTT bridge not running"); + } else if (!_callbacks->runMqttNtpDiag(reply, 160, sender_timestamp == 0)) { + strcpy(reply, "Error: NTP diag unavailable"); + } +#else + strcpy(reply, "Error: not supported on this platform"); +#endif + } else if (memcmp(config, "mqtt.ntp", 8) == 0 && (config[8] == '\0' || config[8] == ' ')) { + sprintf(reply, "> %s", MQTTBridge::effectiveNtpPrimary(&_mqtt_prefs)); + } else if (config[0] == 'm' && config[1] == 'q' && config[2] == 't' && config[3] == 't' && + config[4] >= '1' && config[4] <= ('0' + MAX_MQTT_SLOTS) && config[5] == '.') { + // Slot-based commands: get mqtt1.preset, get mqtt1.server, etc. + int slot = config[4] - '1'; // 0-5 + const char* subcmd = &config[6]; + if (memcmp(subcmd, "preset", 6) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_preset[slot]); + } else if (memcmp(subcmd, "server", 6) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_host[slot]); + } else if (memcmp(subcmd, "port", 4) == 0) { + sprintf(reply, "> %d", _mqtt_prefs.mqtt_slot_port[slot]); + } else if (memcmp(subcmd, "username", 8) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_username[slot]); + } else if (memcmp(subcmd, "password", 8) == 0) { + // Serial only; remote sees set/unset. + if (sender_timestamp == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_password[slot]); + } else { + strcpy(reply, _mqtt_prefs.mqtt_slot_password[slot][0] ? "> ******** (serial only)" : "> (not set)"); + } + } else if (memcmp(subcmd, "token", 5) == 0) { + // Serial only; remote sees set/unset. + if (_mqtt_prefs.mqtt_slot_token[slot][0] == '\0') { + strcpy(reply, "> (not set)"); + } else if (sender_timestamp == 0) { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_token[slot]); + } else { + strcpy(reply, "> ******** (serial only)"); + } + } else if (memcmp(subcmd, "topic", 5) == 0) { + if (_mqtt_prefs.mqtt_slot_topic[slot][0] != '\0') { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_topic[slot]); + } else { + strcpy(reply, "> (default: meshcore/{iata}/{device}/{type})"); + } + } else if (memcmp(subcmd, "audience", 8) == 0) { + if (_mqtt_prefs.mqtt_slot_audience[slot][0] != '\0') { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_slot_audience[slot]); + } else { + strcpy(reply, "> (not set - custom slots use username/password auth)"); + } + } else if (strcmp(subcmd, "filter") == 0) { + char filter_text[MQTTPacketFilter::kFilterTextSize]; + if (MQTTPacketFilter::format(_mqtt_prefs.mqtt_slot_packet_filter[slot], + filter_text, sizeof(filter_text))) { + snprintf(reply, 160, "> %s", filter_text); + } else { + strcpy(reply, "Error: invalid stored packet filter"); + } + } else if (memcmp(subcmd, "diag", 4) == 0) { + MQTTBridge::formatSlotDiagReply(reply, 160, slot); + } else { + sprintf(reply, "??: %s", config); + } + } else if (memcmp(config, "wifi.ssid", 9) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.wifi_ssid); + } else if (memcmp(config, "wifi.pwd", 8) == 0) { + // Serial only (WiFi creds grant LAN access); remote sees set/unset. + if (sender_timestamp == 0) { + sprintf(reply, "> %s", _mqtt_prefs.wifi_password); + } else { + strcpy(reply, _mqtt_prefs.wifi_password[0] ? "> ******** (serial only)" : "> (not set)"); + } + } else if (memcmp(config, "wifi.status", 11) == 0) { + wl_status_t status = WiFi.status(); + const char* status_str; + switch (status) { + case WL_CONNECTED: status_str = "connected"; break; + case WL_NO_SSID_AVAIL: status_str = "no_ssid"; break; + case WL_CONNECT_FAILED: status_str = "connect_failed"; break; + case WL_CONNECTION_LOST: status_str = "connection_lost"; break; + case WL_DISCONNECTED: status_str = "disconnected"; break; + case 255: status_str = "not_started"; break; + default: status_str = "unknown"; break; + } + if (status == WL_CONNECTED) { + sprintf(reply, "> %s, IP: %s, RSSI: %d dBm", status_str, WiFi.localIP().toString().c_str(), WiFi.RSSI()); +#ifdef WITH_MQTT_BRIDGE + unsigned long connect_at = MQTTBridge::getWifiConnectedAtMillis(); + if (connect_at != 0) { + unsigned long uptime_ms = millis() - connect_at; + unsigned long uptime_sec = uptime_ms / 1000; + unsigned long d = uptime_sec / 86400; + unsigned long h = (uptime_sec % 86400) / 3600; + unsigned long m = (uptime_sec % 3600) / 60; + unsigned long s = uptime_sec % 60; + // reply points at the caller's char[160] command buffer (see main.cpp); + // compute the actual remaining space instead of assuming 128. + const size_t kReplyBufSize = 160; + size_t len = strlen(reply); + const size_t reply_remaining = (len < kReplyBufSize) ? (kReplyBufSize - len) : 0; + if (d > 0) { + snprintf(reply + len, reply_remaining, ", uptime: %lud %luh %lum %lus", d, h, m, s); + } else if (h > 0) { + snprintf(reply + len, reply_remaining, ", uptime: %luh %lum %lus", h, m, s); + } else if (m > 0) { + snprintf(reply + len, reply_remaining, ", uptime: %lum %lus", m, s); + } else { + snprintf(reply + len, reply_remaining, ", uptime: %lus", s); + } + } +#endif + } else { +#ifdef WITH_MQTT_BRIDGE + uint8_t reason = MQTTBridge::getLastWifiDisconnectReason(); + if (reason != 0) { + const char* desc = MQTTBridge::wifiReasonStr(reason); + if (desc) { + sprintf(reply, "> %s: %s (reason: %d)", status_str, desc, reason); + } else { + sprintf(reply, "> %s: reason %d", status_str, reason); + } + } else { + sprintf(reply, "> %s (code: %d)", status_str, status); + } +#else + sprintf(reply, "> %s (code: %d)", status_str, status); +#endif + } + } else if (memcmp(config, "wifi.powersave", 14) == 0) { + uint8_t ps = _mqtt_prefs.wifi_power_save; + const char* ps_name = (ps == 1) ? "none" : (ps == 2) ? "max" : "min"; + sprintf(reply, "> %s", ps_name); + } else if (memcmp(config, "timezone.offset", 15) == 0) { + // Must precede the "timezone" (8-byte) check below — that prefix-matches + // "timezone.offset" too, so the more-specific key has to come first or + // `get timezone.offset` returns the string and never the offset (A3). + sprintf(reply, "> %d", _mqtt_prefs.timezone_offset); + } else if (memcmp(config, "timezone", 8) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.timezone_string); + } else if (memcmp(config, "mqtt.analyzer.us", 17) == 0) { + sprintf(reply, "> %s", strcmp(_mqtt_prefs.mqtt_slot_preset[0], "analyzer-us") == 0 ? "on" : "off"); + } else if (memcmp(config, "mqtt.analyzer.eu", 17) == 0) { + sprintf(reply, "> %s", strcmp(_mqtt_prefs.mqtt_slot_preset[1], "analyzer-eu") == 0 ? "on" : "off"); + } else if (sender_timestamp == 0 && memcmp(config, "mqtt.owner", 10) == 0) { + if (_mqtt_prefs.mqtt_owner_public_key[0] != '\0') { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_owner_public_key); + } else { + strcpy(reply, "> (not set)"); + } + } else if (sender_timestamp == 0 && memcmp(config, "mqtt.email", 10) == 0) { + if (_mqtt_prefs.mqtt_email[0] != '\0') { + sprintf(reply, "> %s", _mqtt_prefs.mqtt_email); + } else { + strcpy(reply, "> (not set)"); + } + } else if (memcmp(config, "mqtt.config.valid", 17) == 0) { + bool valid = MQTTBridge::isConfigValid(&_mqtt_prefs); + sprintf(reply, "> %s", valid ? "valid" : "invalid"); +#endif + } else if (memcmp(config, "alert.hashtag", 13) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.alert_hashtag[0] ? _mqtt_prefs.alert_hashtag : "(unset)"); + } else if (sender_timestamp == 0 && memcmp(config, "alert.psk", 9) == 0) { // from serial command line only + sprintf(reply, "> %s", _mqtt_prefs.alert_psk_hex[0] ? _mqtt_prefs.alert_psk_hex : "(unset)"); + } else if (memcmp(config, "alert.region", 12) == 0) { + sprintf(reply, "> %s", _mqtt_prefs.alert_region[0] ? _mqtt_prefs.alert_region : "(unset, using default scope)"); + } else if (memcmp(config, "alert.wifi", 10) == 0) { + sprintf(reply, "> %u min%s", (unsigned)_mqtt_prefs.alert_wifi_minutes, + _mqtt_prefs.alert_wifi_minutes == 0 ? " (disabled)" : ""); + } else if (memcmp(config, "alert.mqtt", 10) == 0) { + sprintf(reply, "> %u min%s", (unsigned)_mqtt_prefs.alert_mqtt_minutes, + _mqtt_prefs.alert_mqtt_minutes == 0 ? " (disabled)" : ""); + } else if (memcmp(config, "alert.interval", 14) == 0) { + sprintf(reply, "> %u min", (unsigned)_mqtt_prefs.alert_min_interval_min); + } else if (memcmp(config, "alert", 5) == 0 && (config[5] == 0 || config[5] == '\n' || config[5] == '\r')) { + sprintf(reply, "> %s", _mqtt_prefs.alert_enabled ? "on" : "off"); + } else { + handled = false; + } + return handled; +#else + (void)sender_timestamp; (void)config; (void)reply; + return false; +#endif +} + +bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, char* reply) { +#ifdef WITH_MQTT_BRIDGE + if (memcmp(command, "tls.bundletest ", 15) == 0) { +#ifdef ESP_PLATFORM + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "ERR: WiFi not connected"); + } else { + size_t bundle_len = 0; + if (rootca_crt_bundle_start != nullptr && + rootca_crt_bundle_end != nullptr && + rootca_crt_bundle_end > rootca_crt_bundle_start) { + bundle_len = static_cast(rootca_crt_bundle_end - rootca_crt_bundle_start); + } + if (bundle_len == 0) { + strcpy(reply, "ERR: no embedded cert bundle"); + } else { + char host[96]; + uint16_t port = 443; + if (!parseTlsBundleTarget(command + 15, host, sizeof(host), &port)) { + strcpy(reply, "ERR: usage tls.bundletest "); + } else { + WiFiClientSecure client; +#if ESP_ARDUINO_VERSION_MAJOR >= 3 + client.setCACertBundle(rootca_crt_bundle_start, bundle_len); +#else + client.setCACertBundle(rootca_crt_bundle_start); +#endif + client.setTimeout(8000); + bool ok = client.connect(host, port); + if (ok) { + client.stop(); + snprintf(reply, 160, "OK: TLS bundle verified %s:%u", host, (unsigned)port); + } else { + snprintf(reply, 160, "ERR: TLS bundle failed %s:%u", host, (unsigned)port); + } + } + } + } +#else + strcpy(reply, "ERR: unsupported on this platform"); +#endif + return true; + } else if (memcmp(command, "ota check", 9) == 0 || memcmp(command, "ota update", 10) == 0) { + // Observer pull-OTA: fetch this variant's build from the baked-in manifest + // and flash it. Intentionally a separate command from "start ota" (the + // manual ElegantOTA web-upload SoftAP) so a remote/online update is never + // triggered by someone expecting to hand-upload a binary. + // ota check -> report available build, do not flash + // ota update -> download and flash, then reboot +#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_BASE) + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "ERR: WiFi not connected"); + } else if (memcmp(command, "ota check", 9) == 0) { + // Check is synchronous so its result lands in this reply, and runs with the + // MQTT bridge UP: the slim per-variant manifest is tiny, so the fetch only + // costs a single TLS handshake (no large JSON doc) — which fits alongside + // the live MQTT sessions even on no-PSRAM boards. No bridge bounce needed. + _board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply); + } else { + // `ota update`: cheap pre-check first (plain HTTP, bridge stays up). Only + // schedule the real update — which tears the bridge down, flashes, and + // reboots — when an applicable build actually exists. otaFromManifest(dry) + // returns true iff so; otherwise it leaves the explanation (up to date / + // cable flash / error) in reply, which we send without disturbing the + // bridge or misleading the user with a "Beginning update..." that no-ops. + if (_board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply)) { + // reply now holds "update available: -> (N behind|new base)", + // where is "vX.Y.Z.B (hash)". Pull out for a friendlier + // start message. The "-> " ... trailing " (" framing is produced by + // ESP32Board::otaFromManifestImpl; ends at the LAST " (" (the + // "(N behind)"/"(new base)" suffix), since the version's own hash-paren + // comes before it. + char target[48] = {0}; + const char* arrow = strstr(reply, "-> "); + if (arrow) { + arrow += 3; + const char* suffix = nullptr; + for (const char* p = arrow; (p = strstr(p, " (")) != nullptr; p++) suffix = p; + size_t len = suffix ? (size_t)(suffix - arrow) : strlen(arrow); + if (len >= sizeof(target)) len = sizeof(target) - 1; + memcpy(target, arrow, len); + target[len] = 0; + } + // Update is DEFERRED so this ack goes out over the mesh before the flash + // blocks the loop and reboots (the app loop runs it shortly). + if (_callbacks->beginDeferredOtaUpdate()) { + if (target[0]) { + snprintf(reply, 160, "Updating to %s; reboots when done (~30s offline). Check 'ver' after.", target); + } else { + strcpy(reply, "Beginning update... (node will reboot if successful)"); + } + } else { + strcpy(reply, "ERR: online OTA not available"); + } + } + } +#else + strcpy(reply, "ERR: online OTA not supported on this build"); +#endif + return true; + } else if (memcmp(command, "start webconfig", 15) == 0 && (command[15] == 0 || command[15] == ' ')) { + // Web config portal: `start webconfig` binds to the LAN IP (or raises the + // setup AP when WiFi is unconfigured); `start webconfig ap` forces the AP. + bool force_ap = (command[15] == ' ' && strcmp(&command[16], "ap") == 0); + if (command[15] == ' ' && !force_ap) { + strcpy(reply, "ERR: usage start webconfig [ap]"); + } else if (!_callbacks->startWebConfig(force_ap, reply)) { + strcpy(reply, "ERR: webconfig not supported on this build"); + } + return true; + } else if (strcmp(command, "stop webconfig") == 0) { + if (!_callbacks->stopWebConfig(reply)) { + strcpy(reply, "ERR: webconfig not supported on this build"); + } + return true; + } else if (memcmp(command, "alert test", 10) == 0 && (command[10] == 0 || command[10] == ' ')) { + // Send a one-off test alert on the configured alert channel. + const char* extra = command[10] == ' ' ? &command[11] : ""; + char text[120]; + if (*extra) { + snprintf(text, sizeof(text), "[test] %s", extra); + } else { + strcpy(text, "[test] alert channel ok"); + } + if (!_mqtt_prefs.alert_psk_hex[0]) { + strcpy(reply, "Error: alert channel not configured (set alert.psk or set alert.hashtag)"); + } else { + bool ok = _callbacks->sendAlertText(text); + if (!ok) { + strcpy(reply, "Error: alert send failed (bad PSK or PUBLIC key refused?)"); + } else if (!_mqtt_prefs.alert_enabled) { + // `alert test` deliberately bypasses the master switch, so a successful + // send here does NOT mean automatic WiFi/MQTT/OTA alerts will fire — those + // gate on `alert on`. Flag it so a working test can't give false confidence. + strcpy(reply, "OK - test sent, but automatic alerts are OFF (run 'set alert on')"); + } else { + strcpy(reply, "OK - alert sent"); + } + } + return true; + } + return false; +#else + (void)sender_timestamp; (void)command; (void)reply; + return false; +#endif +} diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index a55abb264e..120203ab05 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -11,11 +11,24 @@ #include -bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { +bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { inhibit_sleep = true; // prevent sleep during OTA - WiFi.softAP("MeshCore-OTA", NULL); - sprintf(reply, "Started: http://%s/update", WiFi.softAPIP().toString().c_str()); + // If the device is already on a WiFi network (e.g. an observer joined in STA + // mode), serve ElegantOTA on the station IP so it's reachable from the LAN + // without joining a separate AP. Otherwise raise the MeshCore-OTA SoftAP. + // force_ap ("start ota ap") always raises the SoftAP, so the OTA UI stays + // reachable even when the joined network applies client isolation and the + // station IP can't be reached. + IPAddress ip; + if (!force_ap && WiFi.status() == WL_CONNECTED) { + ip = WiFi.localIP(); + } else { + WiFi.softAP("MeshCore-OTA", NULL); + ip = WiFi.softAPIP(); + } + + sprintf(reply, "Started: http://%s/update", ip.toString().c_str()); MESH_DEBUG_PRINTLN("startOTAUpdate: %s", reply); static char id_buf[60]; @@ -40,11 +53,376 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { } #else -bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { +bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { return false; // not supported } #endif +// --------------------------------------------------------------------------- +// Manifest-driven pull OTA (observer / MQTT-bridge builds only) +// +// The observer already holds a live WiFi station connection (for the MQTT +// bridge) and embeds a root-CA bundle, so it can fetch its own firmware. The +// caller (CommonCLI) stops the MQTT bridge first to free heap/TLS, then calls +// this. We read the web-flasher manifest (config.json), find the `flash-update` +// (app-only) build for our own variant, refuse partition-change releases (OTA +// can't rewrite the partition table), skip if already up to date, then stream +// the .bin straight into the inactive OTA slot via HTTPUpdate. +// --------------------------------------------------------------------------- +#if defined(WITH_MQTT_BRIDGE) +#include +#include +#include +#include +#include +#include +#include +#include + +// Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle +// builds still link; we check for presence at runtime. +extern const uint8_t rootca_crt_bundle_start[] asm("_binary_src_certs_x509_crt_bundle_bin_start") __attribute__((weak)); +extern const uint8_t rootca_crt_bundle_end[] asm("_binary_src_certs_x509_crt_bundle_bin_end") __attribute__((weak)); + +// Extract the trailing build hash. For a filename we first drop a ".bin" +// suffix, then take the token after the last '-'. Works for both the manifest +// asset name ("...-v1.16.0-8b084d5.bin" -> "8b084d5") and the embedded +// FIRMWARE_VERSION ("v1.16.0-observer-8b084d5" -> "8b084d5"). +static void ota_extractHash(const char* s, char* out, size_t out_sz) { + if (!s) { if (out_sz) out[0] = 0; return; } + size_t len = strlen(s); + if (len > 4 && strcmp(s + len - 4, ".bin") == 0) len -= 4; + size_t i = len; + while (i > 0 && s[i - 1] != '-') i--; + size_t n = len - i; + if (n >= out_sz) n = out_sz - 1; + memcpy(out, s + i, n); + out[n] = 0; +} + +// Split a version token "vMAJOR.MINOR.PATCH[.BUILD]" into its base +// ("vMAJOR.MINOR.PATCH") and build number (BUILD, or -1 if there's no 4th +// component). The base has exactly two dots; a third dot introduces the build. +static void ota_parseVersion(const char* ver, char* base_out, size_t base_sz, int* build_out) { + *build_out = -1; + if (base_sz) base_out[0] = 0; + if (!ver) return; + int dots = 0, third_dot = -1; + for (int j = 0; ver[j]; j++) { + if (ver[j] == '.' && ++dots == 3) { third_dot = j; break; } + } + if (third_dot >= 0) { + size_t n = (size_t)third_dot; + if (n >= base_sz) n = base_sz - 1; + memcpy(base_out, ver, n); + base_out[n] = 0; + *build_out = atoi(ver + third_dot + 1); + } else { + strncpy(base_out, ver, base_sz - 1); + base_out[base_sz - 1] = 0; + } +} + +// Canonical signature of the FLASHED partition table — MUST match +// scripts/partition_signature.py: each entry "type:subtype:offset:size" in +// lowercase hex, sorted by offset, joined by ','. Lets `ota update` compare the +// target build's partition layout (carried in the manifest as partSig) against +// what's actually on this device, instead of a blanket per-variant flag. +static void ota_partitionSignature(char* out, size_t out_sz) { + struct PE { uint8_t type, subtype; uint32_t off, size; } e[24]; + int n = 0; + esp_partition_iterator_t it = + esp_partition_find(ESP_PARTITION_TYPE_ANY, ESP_PARTITION_SUBTYPE_ANY, nullptr); + while (it != nullptr && n < (int)(sizeof(e) / sizeof(e[0]))) { + const esp_partition_t* p = esp_partition_get(it); + e[n].type = (uint8_t)p->type; + e[n].subtype = (uint8_t)p->subtype; + e[n].off = p->address; + e[n].size = p->size; + n++; + it = esp_partition_next(it); + } + esp_partition_iterator_release(it); // safe on NULL (loop exhausted) + // insertion sort by offset (matches the script's sort key) + for (int i = 1; i < n; i++) { + PE k = e[i]; + int j = i - 1; + while (j >= 0 && e[j].off > k.off) { e[j + 1] = e[j]; j--; } + e[j + 1] = k; + } + size_t pos = 0; + if (out_sz) out[0] = 0; + for (int i = 0; i < n && pos + 1 < out_sz; i++) { + pos += snprintf(out + pos, out_sz - pos, "%s%x:%x:%x:%x", + i ? "," : "", e[i].type, e[i].subtype, (unsigned)e[i].off, (unsigned)e[i].size); + } +} + +// Parameters handed to the worker task; lives on otaFromManifest()'s stack, +// which stays valid because that function blocks until the worker signals done. +struct OtaTaskArgs { + ESP32Board* self; + const char* current_ver; + bool dry_run; + char* reply; + volatile bool result; + volatile bool done; +}; + +static void ota_task_entry(void* param) { + OtaTaskArgs* a = static_cast(param); + a->result = a->self->otaFromManifestImpl(a->current_ver, a->dry_run, a->reply); + a->done = true; // on a successful `ota update` we reboot before reaching here + vTaskDelete(nullptr); +} + +bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { + // The TLS handshake (cert-bundle verify) + JSON parse / HTTPUpdate use far more + // stack than the ~8 KB loop task offers — especially when reached via the deep + // mesh-receive call chain (it overflows the loopTask canary). Run the work in a + // dedicated 24 KB-stack task and block here until it finishes. The big stack is + // freed when the task exits; on a successful update the chip reboots inside it. + OtaTaskArgs args = { this, current_ver, dry_run, reply, false, false }; + TaskHandle_t handle = nullptr; + BaseType_t ok = xTaskCreatePinnedToCore(ota_task_entry, "ota", 24576, &args, 5, &handle, 1); + if (ok != pdPASS) { + strcpy(reply, "ERR: OTA task spawn failed"); + return false; + } + while (!args.done) { + delay(50); // Arduino delay() yields to other tasks + } + return args.result; +} + +bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]) { +#if !defined(OTA_MANIFEST_BASE) || !defined(OTA_VARIANT) + strcpy(reply, "ERR: OTA not configured (build via build.sh)"); + return false; +#else + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "ERR: WiFi not connected"); + return false; + } + + size_t bundle_len = 0; + if (rootca_crt_bundle_start != nullptr && rootca_crt_bundle_end != nullptr && + rootca_crt_bundle_end > rootca_crt_bundle_start) { + bundle_len = (size_t)(rootca_crt_bundle_end - rootca_crt_bundle_start); + } + if (!dry_run && bundle_len == 0) { + strcpy(reply, "ERR: no embedded cert bundle"); + return false; + } + + // --- Fetch this variant's slim manifest ---------------------------------- + // /.json — a ~180 byte per-variant file, not the + // full config.json. + char murl[200]; + HTTPClient http; + WiFiClientSecure mclient; // only used for the HTTPS (update) path + + if (dry_run) { + // `ota check`: fetch over PLAIN HTTP. With no TLS handshake the fetch costs + // negligible heap, so the check runs with the MQTT bridge UP even on no-PSRAM + // — where the cert-bundle TLS verify would otherwise exhaust internal heap + // alongside the two live MQTT TLS sessions (free heap collapses to a few KB + // and the handshake + the bridge both fail). This only reads version info; the + // firmware download below (ota update) is always TLS-verified. Requires the + // manifest host to serve /v over HTTP (no forced HTTPS redirect). + if (strncmp(OTA_MANIFEST_BASE, "https://", 8) == 0) { + snprintf(murl, sizeof(murl), "http://%s/%s.json", OTA_MANIFEST_BASE + 8, OTA_VARIANT); + } else { + snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + } + if (!http.begin(murl)) { + strcpy(reply, "ERR: manifest connect failed"); + return false; + } + } else { + // `ota update`: HTTPS. The bridge is torn down for an update so heap is free, + // and integrity matters because we're about to flash. +#if ESP_ARDUINO_VERSION_MAJOR >= 3 + mclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); +#else + mclient.setCACertBundle(rootca_crt_bundle_start); +#endif + mclient.setTimeout(15000); + snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + if (!http.begin(mclient, murl)) { + strcpy(reply, "ERR: manifest connect failed"); + return false; + } + } + + if (!dry_run) { Serial.print("OTA: checking manifest "); Serial.println(murl); } + + // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with chunked encoding + // and no Content-Length; the raw chunked stream corrupts the parse. HTTP/1.0 + // yields a Connection: close, unframed body. + http.useHTTP10(true); + http.setTimeout(20000); + int code = http.GET(); + if (code != HTTP_CODE_OK) { + snprintf(reply, 160, "ERR: manifest HTTP %d", code); + http.end(); + return false; + } + + WiFiClient* stream = http.getStreamPtr(); + stream->setTimeout(20000); // readBytes honours this, so a slow TLS link != EOF + + JsonDocument doc; + DeserializationError err = deserializeJson(doc, *stream); + http.end(); + if (err) { + snprintf(reply, 160, "ERR: manifest parse (%s)", err.c_str()); + return false; + } + + // Copy fields out before the document is reused/cleared. + char file_url[200] = {0}, avail_version[40] = {0}, avail_base[40] = {0}, avail_hash[24] = {0}; + strncpy(file_url, doc["file"] | "", sizeof(file_url) - 1); + strncpy(avail_version, doc["version"] | "", sizeof(avail_version) - 1); + strncpy(avail_base, doc["baseVersion"] | "", sizeof(avail_base) - 1); + strncpy(avail_hash, doc["hash"] | "", sizeof(avail_hash) - 1); + int avail_build = doc["build"] | -1; + bool legacy_partition_change = doc["partitionChange"] | false; + char manifest_partsig[256] = {0}; + strncpy(manifest_partsig, doc["partSig"] | "", sizeof(manifest_partsig) - 1); + doc.clear(); + + if (!file_url[0]) { + strcpy(reply, "ERR: manifest missing file"); + return false; + } + + // Partition compatibility: prefer the precise per-build signature — compare the + // target build's partition layout (manifest partSig) to what's actually flashed + // on THIS device. Refuse only on a real mismatch (OTA can't rewrite the table). + // Fall back to the legacy bool for manifests that predate partSig. + bool partition_change; + if (manifest_partsig[0]) { + char dev_partsig[256]; + ota_partitionSignature(dev_partsig, sizeof(dev_partsig)); + partition_change = (strcmp(dev_partsig, manifest_partsig) != 0); + } else { + partition_change = legacy_partition_change; + } + + // --- Determine current-vs-available -------------------------------------- + // Our running version token (e.g. "v1.16.0.5"), i.e. current_ver up to the + // first '-' (which precedes "-observer-"). + char own_version[40] = {0}; + for (size_t i = 0; current_ver && current_ver[i] && current_ver[i] != '-' && i < sizeof(own_version) - 1; i++) { + own_version[i] = current_ver[i]; + } + char own_base[40]; + int own_build; + ota_parseVersion(own_version, own_base, sizeof(own_base), &own_build); + + // Fallback identity by commit hash (handles pre-build-number / local images that + // carry no 4th version component). Shared-prefix compare absorbs the 7- vs 8-char + // git abbreviation difference. + char cur_hash[24]; + ota_extractHash(current_ver, cur_hash, sizeof(cur_hash)); + size_t lh = strlen(avail_hash), lc = strlen(cur_hash); + size_t m = (lh < lc) ? lh : lc; + bool hash_equal = (m >= 7 && strncmp(avail_hash, cur_hash, m) == 0); + + bool same_base = (own_base[0] && avail_base[0] && strcmp(own_base, avail_base) == 0); + bool have_builds = (own_build >= 0 && avail_build >= 0); + bool diff_base = (own_base[0] && avail_base[0] && !same_base); + + int behind = 0; + bool up_to_date; + if (same_base && have_builds) { + behind = avail_build - own_build; + up_to_date = (behind <= 0); + } else if (diff_base) { + up_to_date = false; // different base version is always an update + } else { + up_to_date = hash_equal; // unknown build numbers -> fall back to hash + } + + // Display strings carry the short commit hash in the same form as the asset + // filename, e.g. "v1.16.0.2 (5acfdd7)" (or just the hash if there's no version). + char avail_disp[72], own_disp[72]; + if (avail_version[0]) snprintf(avail_disp, sizeof(avail_disp), "%s (%s)", avail_version, avail_hash); + else snprintf(avail_disp, sizeof(avail_disp), "%s", avail_hash); + if (own_version[0]) snprintf(own_disp, sizeof(own_disp), "%s (%s)", own_version, cur_hash); + else snprintf(own_disp, sizeof(own_disp), "%s", cur_hash); + const char* pc_note = partition_change ? " [partition change: cable flash]" : ""; + + // --- Report (dry run / `ota check`) -------------------------------------- + // Returns true iff an OTA-applicable update is available (not up-to-date and not + // a partition-change build). `ota check` ignores the return and just shows the + // reply; `ota update` uses it to decide whether to actually schedule the flash. + if (dry_run) { + if (up_to_date) { + snprintf(reply, 160, "up to date: %s", avail_disp); + } else if (same_base && have_builds) { + snprintf(reply, 160, "update available: %s -> %s (%d behind)%s", own_disp, avail_disp, behind, pc_note); + } else if (diff_base) { + snprintf(reply, 160, "update available: %s -> %s (new base)%s", own_disp, avail_disp, pc_note); + } else { + snprintf(reply, 160, "update available: %s -> %s%s", own_disp, avail_disp, pc_note); + } + return (!up_to_date && !partition_change); + } + + // --- Gates (real `ota update`) ------------------------------------------- + if (partition_change) { + snprintf(reply, 160, "ERR: %s needs cable flash (partition change)", avail_disp); + return false; + } + if (up_to_date) { + snprintf(reply, 160, "OK: already up to date: %s", avail_disp); + return false; + } + + // --- Stream the .bin (the manifest's full URL) into the inactive OTA slot - + Serial.printf("OTA: update %s -> %s\n", own_disp, avail_disp); + Serial.print("OTA: downloading "); Serial.println(file_url); + inhibit_sleep = true; // keep awake through the flash + + WiFiClientSecure uclient; +#if ESP_ARDUINO_VERSION_MAJOR >= 3 + uclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); +#else + uclient.setCACertBundle(rootca_crt_bundle_start); +#endif + uclient.setTimeout(20000); + + // Console progress to the USB serial (always on; MESH_DEBUG is off on the default + // observer profile). Non-capturing lambdas + a file-static decile, so the global + // httpUpdate object never holds a dangling reference after this function returns. + static int ota_progress_decile; + ota_progress_decile = -1; + httpUpdate.onProgress([](int cur, int total) { + if (total <= 0) return; + int d = (int)((int64_t)cur * 10 / total); + if (d != ota_progress_decile) { ota_progress_decile = d; Serial.printf("OTA: %d%%\n", d * 10); } + }); + httpUpdate.onEnd([]() { Serial.println("OTA: write complete, rebooting..."); }); + httpUpdate.rebootOnUpdate(true); // reboots into the new image on success + t_httpUpdate_return ret = httpUpdate.update(uclient, file_url); + + // Only reached on failure (success reboots inside update()). + inhibit_sleep = false; + snprintf(reply, 160, "ERR: OTA failed (%d): %s", (int)ret, + httpUpdate.getLastErrorString().c_str()); + Serial.print("OTA: FAILED - "); Serial.println(reply); + return false; +#endif // OTA_MANIFEST_BASE && OTA_VARIANT +} +#else +bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { + strcpy(reply, "ERR: not supported"); + return false; +} +#endif // WITH_MQTT_BRIDGE + void ESP32Board::powerOff() { enterDeepSleep(0); // Do not wakeup } diff --git a/src/helpers/ESP32Board.h b/src/helpers/ESP32Board.h index d7eb5fee26..3ecf6735ec 100644 --- a/src/helpers/ESP32Board.h +++ b/src/helpers/ESP32Board.h @@ -154,7 +154,12 @@ class ESP32Board : public mesh::MainBoard { esp_restart(); } - bool startOTAUpdate(const char* id, char reply[]) override; + bool startOTAUpdate(const char* id, char reply[], bool force_ap = false) override; + bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) override; + // Heavy body (TLS + JSON / HTTPUpdate). Runs in a dedicated large-stack task + // spawned by otaFromManifest() — public only so that task entry point can call + // it; not meant to be invoked directly. + bool otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]); void setInhibitSleep(bool inhibit) { inhibit_sleep = inhibit; diff --git a/src/helpers/ESP32WsTransportFix.cpp b/src/helpers/ESP32WsTransportFix.cpp new file mode 100644 index 0000000000..2e14bb33e9 --- /dev/null +++ b/src/helpers/ESP32WsTransportFix.cpp @@ -0,0 +1,130 @@ +#ifdef ESP_PLATFORM + +// Link-time workaround for an off-by-one heap overflow in ESP-IDF v4.4's +// WebSocket transport (components/tcp_transport/transport_ws.c), which ships +// PRECOMPILED in the Arduino-ESP32 2.x SDK (libtcp_transport.a) and cannot be +// patched at source level. +// +// The bug (transport_ws.c, ws_connect() response-read loop): +// +// header_len += len; +// ws->buffer[header_len] = '\0'; // header_len can reach WS_BUFFER_SIZE +// } while (... && header_len < WS_BUFFER_SIZE); +// +// ws->buffer is malloc(WS_BUFFER_SIZE) (1024). When a wss:// endpoint answers +// the WebSocket upgrade with >= 1024 bytes of HTTP response before the blank +// line terminator (typical for a down/misconfigured broker behind a proxy or +// CDN that serves a large HTML error page), the final iteration writes one +// '\0' one byte past the block. With heap poisoning enabled that zeroes the +// LSB of the tail canary (0xbaad5678 -> 0xbaad5600); the corruption then sits +// silent until the block is freed — which happens in ws_destroy() during +// esp_mqtt_client_destroy(), i.e. MQTTBridge::end() — and the free asserts: +// +// CORRUPT HEAP: Bad tail at 0x.... Expected 0xbaad5678 got 0xbaad5600 +// assert failed: multi_heap_free multi_heap_poisoning.c:259 +// +// On observer builds that teardown runs at the start of the deferred +// `ota update`, so a single down wss broker made every online OTA panic and +// reboot before the download began (backtrace decoded from a Heltec V3 on +// v1.16.0.11: free <- ws_destroy <- esp_transport_list_destroy <- +// esp_mqtt_client_destroy <- ~PsychicMqttClient <- destroySlotClients <- +// MQTTBridge::end <- MyMesh::setBridgeState <- MyMesh::loop). +// +// Fix: [esp32_base] adds `-Wl,--wrap=esp_transport_ws_init`, so every +// creation of a WS transport (esp-mqtt does one per wss slot) is routed +// through __wrap_esp_transport_ws_init below, which replaces the freshly +// allocated 1024-byte buffer with a (WS_BUFFER_SIZE + 1)-byte one. The +// out-of-bounds index WS_BUFFER_SIZE then lands on our extra byte and the +// handshake fails cleanly ("Upgrade" header not found) instead of corrupting +// the heap. Upstream fixed this in ESP-IDF 5.x, so this file compiles to a +// pass-through there and can be deleted (together with the --wrap flag) when +// the fork moves to Arduino core 3.x. +// +// transport_ws_t below is copied verbatim from ESP-IDF release/v4.4 +// transport_ws.c (the struct is file-private, so it is not in any shipped +// header). Source fidelity was verified against the shipped binary: addr2line +// on the crash backtrace resolves to the exact line numbers of that file +// (e.g. free(ws->buffer) at transport_ws.c:546). Only the first two members +// (path, buffer) are dereferenced here. + +#include "esp_idf_version.h" + +#if ESP_IDF_VERSION_MAJOR == 4 + +#include +#include "sdkconfig.h" +#include "esp_transport.h" +#include "esp_transport_ws.h" + +#ifndef CONFIG_WS_BUFFER_SIZE +#define CONFIG_WS_BUFFER_SIZE 1024 +#endif + +// --- copied from ESP-IDF release/v4.4 components/tcp_transport/transport_ws.c --- +typedef struct { + uint8_t opcode; + char mask_key[4]; + int payload_len; + int bytes_remaining; + bool header_received; +} ws_transport_frame_state_t; + +typedef struct { + char *path; + char *buffer; + char *sub_protocol; + char *user_agent; + char *headers; + bool propagate_control_frames; + ws_transport_frame_state_t frame_state; + esp_transport_handle_t parent; +} transport_ws_t; +// -------------------------------------------------------------------------------- + +extern "C" { + +esp_transport_handle_t __real_esp_transport_ws_init(esp_transport_handle_t parent_handle); + +esp_transport_handle_t __wrap_esp_transport_ws_init(esp_transport_handle_t parent_handle) { + esp_transport_handle_t t = __real_esp_transport_ws_init(parent_handle); + if (t != nullptr) { + transport_ws_t* ws = (transport_ws_t*)esp_transport_get_context_data(t); + if (ws != nullptr && ws->buffer != nullptr) { + // The buffer is untouched at this point (allocated moments ago inside + // __real_esp_transport_ws_init), so a swap is safe. + char* padded = (char*)malloc(CONFIG_WS_BUFFER_SIZE + 1); + if (padded != nullptr) { + free(ws->buffer); + ws->buffer = padded; + } + // On alloc failure keep the original buffer: same behavior as before + // this fix, which is still strictly better than failing init here. + } + } + return t; +} + +} // extern "C" + +#else // ESP_IDF_VERSION_MAJOR != 4 + +// IDF 5.x fixed the overflow upstream; keep a pass-through so the --wrap flag +// (set for all ESP32 envs in [esp32_base]) still links if anything references +// the symbol. + +#include "esp_transport.h" +#include "esp_transport_ws.h" + +extern "C" { + +esp_transport_handle_t __real_esp_transport_ws_init(esp_transport_handle_t parent_handle); + +esp_transport_handle_t __wrap_esp_transport_ws_init(esp_transport_handle_t parent_handle) { + return __real_esp_transport_ws_init(parent_handle); +} + +} // extern "C" + +#endif // ESP_IDF_VERSION_MAJOR + +#endif // ESP_PLATFORM diff --git a/src/helpers/JWTHelper.cpp b/src/helpers/JWTHelper.cpp new file mode 100644 index 0000000000..4e411e12bb --- /dev/null +++ b/src/helpers/JWTHelper.cpp @@ -0,0 +1,204 @@ +// MQTT-only translation unit. 22 variants re-glob helpers/*.cpp past the +// arduino_base exclusion, so the contents are guarded here rather than in +// the build filter — same idiom as helpers/esp32/WebConfigServer.cpp. +#ifdef WITH_MQTT_BRIDGE + +#include "JWTHelper.h" +#include +#include +#include +#include "ed_25519.h" +#include "mbedtls/base64.h" + +// Base64 URL encoding table (without padding) +static const char base64url_chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +bool JWTHelper::createAuthToken( + const mesh::LocalIdentity& identity, + const char* audience, + unsigned long issuedAt, + unsigned long expiresIn, + char* token, + size_t tokenSize, + const char* owner, + const char* client, + const char* email +) { + if (!audience || !token || tokenSize == 0) { + return false; + } + + // Use current time if not specified + if (issuedAt == 0) { + issuedAt = time(nullptr); + } + + // Create header + char header[256]; + size_t headerLen = createHeader(header, sizeof(header)); + if (headerLen == 0) { + return false; + } + + // Get public key as UPPERCASE HEX string + char publicKeyHex[65]; + mesh::Utils::toHex(publicKeyHex, identity.pub_key, PUB_KEY_SIZE); + for (int i = 0; publicKeyHex[i]; i++) { + publicKeyHex[i] = toupper(publicKeyHex[i]); + } + + // Create payload + char payload[512]; + size_t payloadLen = createPayload(publicKeyHex, audience, issuedAt, expiresIn, payload, sizeof(payload), owner, client, email); + if (payloadLen == 0) { + return false; + } + + // Create signing input: header.payload + char signingInput[768]; + size_t signingInputLen = headerLen + 1 + payloadLen; + if (signingInputLen >= sizeof(signingInput)) { + return false; + } + + memcpy(signingInput, header, headerLen); + signingInput[headerLen] = '.'; + memcpy(signingInput + headerLen + 1, payload, payloadLen); + + // Sign the data using direct Ed25519 signing + uint8_t signature[64]; + mesh::LocalIdentity identity_copy = identity; + + uint8_t export_buffer[96]; + size_t exported_size = identity_copy.writeTo(export_buffer, sizeof(export_buffer)); + + if (exported_size != 96) { + return false; + } + + uint8_t* private_key = export_buffer; + uint8_t* public_key = export_buffer + 64; + + ed25519_sign(signature, (const unsigned char*)signingInput, signingInputLen, public_key, private_key); + + // Verify the signature locally + int verify_result = ed25519_verify(signature, (const unsigned char*)signingInput, signingInputLen, public_key); + if (verify_result != 1) { + if (Serial.availableForWrite() > 0) Serial.println("JWTHelper: Signature verification failed!"); + return false; + } + + // Convert signature to hex + char signatureHex[129]; + for (int i = 0; i < 64; i++) { + sprintf(signatureHex + (i * 2), "%02X", signature[i]); + } + signatureHex[128] = '\0'; + + // Create final token: header.payload.signatureHex (MeshCore Decoder format) + size_t sigHexLen = strlen(signatureHex); + size_t totalLen = headerLen + 1 + payloadLen + 1 + sigHexLen; + if (totalLen >= tokenSize) { + return false; + } + + memcpy(token, header, headerLen); + token[headerLen] = '.'; + memcpy(token + headerLen + 1, payload, payloadLen); + token[headerLen + 1 + payloadLen] = '.'; + memcpy(token + headerLen + 1 + payloadLen + 1, signatureHex, sigHexLen); + token[totalLen] = '\0'; + + return true; +} + +size_t JWTHelper::base64UrlEncode(const uint8_t* input, size_t inputLen, char* output, size_t outputSize) { + if (!input || !output || outputSize == 0) { + return 0; + } + + size_t outlen = 0; + int ret = mbedtls_base64_encode((unsigned char*)output, outputSize - 1, &outlen, input, inputLen); + + if (ret != 0) { + return 0; + } + + // Convert to base64 URL format in-place (replace + with -, / with _, remove padding =) + for (size_t i = 0; i < outlen; i++) { + if (output[i] == '+') { + output[i] = '-'; + } else if (output[i] == '/') { + output[i] = '_'; + } + } + + // Remove padding '=' characters + while (outlen > 0 && output[outlen-1] == '=') { + outlen--; + } + output[outlen] = '\0'; + return outlen; +} + +size_t JWTHelper::createHeader(char* output, size_t outputSize) { + // Create JWT header: {"alg":"Ed25519","typ":"JWT"} + DynamicJsonDocument doc(256); + doc["alg"] = "Ed25519"; + doc["typ"] = "JWT"; + + char jsonBuffer[256]; + size_t len = serializeJson(doc, jsonBuffer, sizeof(jsonBuffer)); + if (len == 0 || len >= sizeof(jsonBuffer)) { + return 0; + } + + return base64UrlEncode((uint8_t*)jsonBuffer, len, output, outputSize); +} + +size_t JWTHelper::createPayload( + const char* publicKey, + const char* audience, + unsigned long issuedAt, + unsigned long expiresIn, + char* output, + size_t outputSize, + const char* owner, + const char* client, + const char* email +) { + // Create JWT payload + DynamicJsonDocument doc(512); + doc["publicKey"] = publicKey; + doc["aud"] = audience; + doc["iat"] = issuedAt; + + if (expiresIn > 0) { + doc["exp"] = issuedAt + expiresIn; + } + + // Add optional owner field if provided + if (owner && strlen(owner) > 0) { + doc["owner"] = owner; + } + + // Add optional client field if provided + if (client && strlen(client) > 0) { + doc["client"] = client; + } + + // Add optional email field if provided + if (email && strlen(email) > 0) { + doc["email"] = email; + } + + char jsonBuffer[512]; + size_t len = serializeJson(doc, jsonBuffer, sizeof(jsonBuffer)); + if (len == 0 || len >= sizeof(jsonBuffer)) { + return 0; + } + + return base64UrlEncode((uint8_t*)jsonBuffer, len, output, outputSize); +} + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/JWTHelper.h b/src/helpers/JWTHelper.h new file mode 100644 index 0000000000..a84889d891 --- /dev/null +++ b/src/helpers/JWTHelper.h @@ -0,0 +1,87 @@ +#pragma once + +#include "MeshCore.h" +#include "Identity.h" + +/** + * JWT Helper for creating authentication tokens + * + * This class provides functionality to create JWT-style authentication tokens + * signed with Ed25519 private keys for MQTT authentication. + */ +class JWTHelper { +public: + /** + * Create an authentication token for MQTT authentication + * + * @param identity LocalIdentity instance for signing + * @param audience Audience string (e.g., "mqtt-us-v1.letsmesh.net") + * @param issuedAt Unix timestamp (0 for current time) + * @param expiresIn Expiration time in seconds (0 for no expiration) + * @param token Buffer to store the resulting token + * @param tokenSize Size of the token buffer + * @param owner Optional owner public key in hex format (nullptr if not set) + * @param client Optional client string (nullptr if not set) + * @param email Optional email address (nullptr if not set) + * @return true if token was created successfully + */ + static bool createAuthToken( + const mesh::LocalIdentity& identity, + const char* audience, + unsigned long issuedAt = 0, + unsigned long expiresIn = 0, + char* token = nullptr, + size_t tokenSize = 0, + const char* owner = nullptr, + const char* client = nullptr, + const char* email = nullptr + ); + +private: + /** + * Base64 URL encode data + * + * @param input Input data + * @param inputLen Length of input data + * @param output Output buffer + * @param outputSize Size of output buffer + * @return Length of encoded data, or 0 on error + */ + static size_t base64UrlEncode(const uint8_t* input, size_t inputLen, char* output, size_t outputSize); + + /** + * Create JWT header + * + * @param output Output buffer + * @param outputSize Size of output buffer + * @return Length of header, or 0 on error + */ + static size_t createHeader(char* output, size_t outputSize); + + /** + * Create JWT payload + * + * @param publicKey Public key in hex format + * @param audience Audience string + * @param issuedAt Issued at timestamp + * @param expiresIn Expiration time in seconds (0 for no expiration) + * @param output Output buffer + * @param outputSize Size of output buffer + * @param owner Optional owner public key in hex format (nullptr if not set) + * @param client Optional client string (nullptr if not set) + * @param email Optional email address (nullptr if not set) + * @return Length of payload, or 0 on error + */ + static size_t createPayload( + const char* publicKey, + const char* audience, + unsigned long issuedAt, + unsigned long expiresIn, + char* output, + size_t outputSize, + const char* owner = nullptr, + const char* client = nullptr, + const char* email = nullptr + ); + +}; diff --git a/src/helpers/MQTTConnectionPolicy.h b/src/helpers/MQTTConnectionPolicy.h new file mode 100644 index 0000000000..0ab4be12b1 --- /dev/null +++ b/src/helpers/MQTTConnectionPolicy.h @@ -0,0 +1,199 @@ +#pragma once + +#include + +// Pure timing and state-transition policy used by MQTTBridge's connection +// maintenance loop. Keeping these decisions independent of Arduino, WiFi, and +// the MQTT client lets host tests exercise the exact production policy with a +// deterministic clock. +namespace MQTTConnectionPolicy { + +static const uint32_t kReconnectGuardMs = 15000UL; +static const uint32_t kStableResetMs = 120000UL; +static const uint32_t kCircuitBreakerProbeMs = 1800000UL; +static const uint32_t kRenewalThrottleMs = 60000UL; +static const uint32_t kSlotStaggerMs = 3000UL; +static const uint8_t kMaxFailuresAtMaxBackoff = 3; +static const uint32_t kDefaultJwtLifetimeSecs = 86400UL; +static const uint32_t kMaxJwtStaggerSecs = 300UL; +static const uint32_t kMinimumValidEpoch = 1000000000UL; +static const uint32_t kJwtClockThreshold = 1735689600UL; // 2025-01-01 UTC +// A wall clock at or past this instant was set from a real source (NTP or an +// admin); anything earlier is the firmware's unset-clock default (1715770351, +// 15 May 2024), so a delta spanning the sync is meaningless. +static const uint32_t kSyncedClockEpoch = kJwtClockThreshold; + +// Unsigned subtraction is intentionally used: it is the standard millis() +// idiom and remains correct across a single 32-bit counter rollover. +static inline uint32_t elapsedMs(uint32_t now, uint32_t then) { + return now - then; +} + +static inline bool reconnectGuardActive(uint32_t now, uint32_t last_reconnect) { + return elapsedMs(now, last_reconnect) < kReconnectGuardMs; +} + +static inline bool stableConnection(uint32_t now, uint32_t connected_at) { + return connected_at != 0 && elapsedMs(now, connected_at) >= kStableResetMs; +} + +static inline uint32_t reconnectBackoffMs(uint8_t reconnect_backoff) { + static const uint32_t kBackoffMs[] = { + 10000UL, 30000UL, 60000UL, 120000UL, 300000UL + }; + const uint8_t index = reconnect_backoff < 5 ? reconnect_backoff : 4; + return kBackoffMs[index]; +} + +static inline uint32_t reconnectDelayMs(uint8_t reconnect_backoff, uint8_t slot_index) { + return reconnectBackoffMs(reconnect_backoff) + + static_cast(slot_index) * kSlotStaggerMs; +} + +static inline bool reconnectDue(uint32_t now, uint32_t last_attempt, + uint8_t reconnect_backoff, uint8_t slot_index) { + return elapsedMs(now, last_attempt) >= reconnectDelayMs(reconnect_backoff, slot_index); +} + +struct BackoffAdvance { + uint8_t reconnect_backoff; + uint8_t max_backoff_failures; + bool circuit_breaker_tripped; + bool should_reconnect; +}; + +// Advance the ladder immediately before a due reconnect. The first visit to +// the 300-second rung changes level 4 to the saturated marker 5. Three later +// failures at that rung trip the breaker; the third does not launch another +// connection attempt. +static inline BackoffAdvance advanceBackoff(uint8_t reconnect_backoff, + uint8_t max_backoff_failures) { + BackoffAdvance result = { + reconnect_backoff, max_backoff_failures, false, true + }; + if (result.reconnect_backoff < 5) { + result.reconnect_backoff++; + return result; + } + + if (result.max_backoff_failures < UINT8_MAX) { + result.max_backoff_failures++; + } + if (result.max_backoff_failures >= kMaxFailuresAtMaxBackoff) { + result.circuit_breaker_tripped = true; + result.should_reconnect = false; + } + return result; +} + +static inline bool circuitBreakerProbeDue(uint32_t now, uint32_t last_attempt) { + return elapsedMs(now, last_attempt) >= kCircuitBreakerProbeMs; +} + +// WiFi station reconnect backoff. The bridge drives its own STA reconnect loop +// separate from the per-slot MQTT reconnects, with a slightly longer first rung +// (15 s vs the slot ladder's 10 s). Extracted from handleWiFiConnection() so the +// ladder and its wrap-safe timing are exercised by host tests instead of a +// second inline copy of the backoff math. +static inline uint32_t wifiReconnectBackoffMs(uint8_t attempt) { + static const uint32_t kBackoffMs[] = { + 15000UL, 30000UL, 60000UL, 120000UL, 300000UL + }; + const uint8_t index = attempt < 5 ? attempt : 4; + return kBackoffMs[index]; +} + +// A reconnect is due only once the link has been down for the current rung AND +// no attempt has been made within that rung (both measured wrap-safely). This +// mirrors the two-part guard the bridge applied inline. +static inline bool wifiReconnectDue(uint32_t now, uint32_t disconnected_since, + uint32_t last_attempt, uint8_t attempt) { + const uint32_t delay = wifiReconnectBackoffMs(attempt); + return elapsedMs(now, disconnected_since) >= delay && + elapsedMs(now, last_attempt) >= delay; +} + +// The attempt counter climbs to 5 and then saturates; the index clamp in +// wifiReconnectBackoffMs() holds it at the 300 s rung. +static inline uint8_t nextWifiBackoffAttempt(uint8_t attempt) { + return attempt < 5 ? static_cast(attempt + 1) : attempt; +} + +// Each later slot expires up to five percent of the base lifetime earlier, +// capped at five minutes per slot. Runtime slot indexes are bounded by the +// persisted MQTT slot count; the final clamp also prevents underflow if this +// helper is used with unexpected input. +static inline uint32_t jwtLifetimeSecs(uint32_t base_lifetime, uint8_t slot_index) { + uint32_t per_slot_stagger = base_lifetime / 20UL; + if (per_slot_stagger > kMaxJwtStaggerSecs) { + per_slot_stagger = kMaxJwtStaggerSecs; + } + uint64_t stagger = static_cast(slot_index) * per_slot_stagger; + if (stagger > base_lifetime) { + stagger = base_lifetime; + } + return base_lifetime - static_cast(stagger); +} + +static inline uint32_t renewalBufferSecs(uint32_t lifetime_secs) { + uint32_t buffer = lifetime_secs / 10UL; + if (buffer < 60UL) buffer = 60UL; + if (buffer > 300UL) buffer = 300UL; + return buffer; +} + +static inline bool tokenNeedsRenewal(bool time_synced, uint32_t current_time, + uint32_t token_expires_at, + uint32_t renewal_buffer_secs) { + if (!time_synced) { + return token_expires_at == 0; + } + if (token_expires_at < kMinimumValidEpoch) { + return true; + } + if (current_time >= token_expires_at) { + return true; + } + return current_time >= token_expires_at - renewal_buffer_secs; +} + +static inline bool renewalAttemptAllowed(uint32_t now, uint32_t last_attempt) { + return elapsedMs(now, last_attempt) >= kRenewalThrottleMs; +} + +static inline bool jwtClockAvailable(bool ntp_synced, uint32_t current_time) { + return ntp_synced || current_time >= kJwtClockThreshold; +} + +// How a given MQTT slot fares at bridge setup on this hardware. MQTTBridge's +// setup loop iterates runtime slots in index order and connects the first +// `max_active` *enabled* slots, skipping the rest (each WSS/TLS link needs +// ~40 KB internal heap, so non-PSRAM caps at 2 concurrent, PSRAM at 5). Slots at +// or beyond the runtime array size (`slot_count`, e.g. 3 on non-PSRAM) are never +// iterated at all. Extracted so the CLI can tell the operator, at +// `set mqttN.preset` time, whether a slot will actually come up — and so the +// exact rule is host-tested rather than hand-reasoned (it is easy to conflate +// slot_count with max_active). +enum class SlotActivation : uint8_t { + Connects, // enabled and within the concurrent-connection budget + Disabled, // slot has no preset ("none") — not attempted + BeyondArray, // index >= slot_count: outside the runtime slot array here + OverActiveCap, // enabled, but lower-numbered slots already fill the budget +}; + +// `enabled` must have at least `slot_count` entries; `slot` is 0-based. Mirrors +// the bridge's first-come-by-index activation order exactly. +static inline SlotActivation classifySlotActivation(int slot, const bool* enabled, + int slot_count, int max_active) { + if (slot < 0) return SlotActivation::Disabled; + if (slot >= slot_count) return SlotActivation::BeyondArray; + if (enabled == nullptr || !enabled[slot]) return SlotActivation::Disabled; + int rank = 0; // this slot's position among enabled slots, counting by index + for (int i = 0; i <= slot; i++) { + if (enabled[i]) rank++; + } + return (rank <= max_active) ? SlotActivation::Connects + : SlotActivation::OverActiveCap; +} + +} // namespace MQTTConnectionPolicy diff --git a/src/helpers/MQTTDefaults.h b/src/helpers/MQTTDefaults.h new file mode 100644 index 0000000000..5072e4229f --- /dev/null +++ b/src/helpers/MQTTDefaults.h @@ -0,0 +1,114 @@ +#pragma once + +#ifdef WITH_MQTT_BRIDGE + +#include +#include +#include "CommonCLI.h" +#include "MQTTPacketFilter.h" +#include "MQTTPresets.h" + +// Compile-time defaults for fresh /mqtt_prefs (override via platformio build_flags). +// Example: +// -D MQTT_DEFAULT_SLOT1_PRESET='"meshcore-ca-1"' +// -D MQTT_DEFAULT_IATA='"YYZ"' +// -D MQTT_DEFAULT_TIMEZONE='"America/Toronto"' +// -D MQTT_DEFAULT_TIMEZONE_OFFSET=-5 + +#ifndef MQTT_DEFAULT_SLOT1_PRESET +#define MQTT_DEFAULT_SLOT1_PRESET "analyzer-us" +#endif +#ifndef MQTT_DEFAULT_SLOT2_PRESET +#define MQTT_DEFAULT_SLOT2_PRESET "analyzer-eu" +#endif +#ifndef MQTT_DEFAULT_SLOT3_PRESET +#define MQTT_DEFAULT_SLOT3_PRESET "none" +#endif +#ifndef MQTT_DEFAULT_SLOT4_PRESET +#define MQTT_DEFAULT_SLOT4_PRESET "none" +#endif +#ifndef MQTT_DEFAULT_SLOT5_PRESET +#define MQTT_DEFAULT_SLOT5_PRESET "none" +#endif +#ifndef MQTT_DEFAULT_SLOT6_PRESET +#define MQTT_DEFAULT_SLOT6_PRESET "none" +#endif + +#ifndef MQTT_DEFAULT_IATA +#define MQTT_DEFAULT_IATA "" +#endif + +#ifndef MQTT_DEFAULT_TIMEZONE +#define MQTT_DEFAULT_TIMEZONE "" +#endif + +#ifndef MQTT_DEFAULT_TIMEZONE_OFFSET +#define MQTT_DEFAULT_TIMEZONE_OFFSET 0 +#endif + +static inline void mqttDefaultSlotPreset(char* dest, size_t dest_size, const char* preset) { + const char* resolved = MQTT_PRESET_NONE; + if (preset && preset[0] != '\0') { + if (strcmp(preset, MQTT_PRESET_NONE) == 0 || + strcmp(preset, MQTT_PRESET_CUSTOM) == 0 || + findMQTTPreset(preset) != nullptr) { + resolved = preset; + } else { + MESH_DEBUG_PRINTLN("MQTT: invalid default preset '%s', using none", preset); + } + } + strncpy(dest, resolved, dest_size - 1); + dest[dest_size - 1] = '\0'; +} + +static inline void applyMQTTDefaults(MQTTPrefs* prefs) { + memset(prefs, 0, sizeof(MQTTPrefs)); + prefs->mqtt_status_enabled = 1; + prefs->mqtt_packets_enabled = 1; + prefs->mqtt_raw_enabled = 0; + prefs->mqtt_tx_enabled = 2; + prefs->mqtt_rx_enabled = 1; + prefs->mqtt_status_interval = 300000; + prefs->wifi_power_save = 1; + + mqttDefaultSlotPreset(prefs->mqtt_slot_preset[0], sizeof(prefs->mqtt_slot_preset[0]), + MQTT_DEFAULT_SLOT1_PRESET); + mqttDefaultSlotPreset(prefs->mqtt_slot_preset[1], sizeof(prefs->mqtt_slot_preset[1]), + MQTT_DEFAULT_SLOT2_PRESET); + mqttDefaultSlotPreset(prefs->mqtt_slot_preset[2], sizeof(prefs->mqtt_slot_preset[2]), + MQTT_DEFAULT_SLOT3_PRESET); + mqttDefaultSlotPreset(prefs->mqtt_slot_preset[3], sizeof(prefs->mqtt_slot_preset[3]), + MQTT_DEFAULT_SLOT4_PRESET); + mqttDefaultSlotPreset(prefs->mqtt_slot_preset[4], sizeof(prefs->mqtt_slot_preset[4]), + MQTT_DEFAULT_SLOT5_PRESET); + mqttDefaultSlotPreset(prefs->mqtt_slot_preset[5], sizeof(prefs->mqtt_slot_preset[5]), + MQTT_DEFAULT_SLOT6_PRESET); + for (int i = 0; i < MQTT_PREFS_SLOT_COUNT; ++i) { + prefs->mqtt_slot_packet_filter[i] = MQTTPacketFilter::kAllPacketTypes; + } + + if (MQTT_DEFAULT_IATA[0] != '\0') { + strncpy(prefs->mqtt_iata, MQTT_DEFAULT_IATA, sizeof(prefs->mqtt_iata) - 1); + prefs->mqtt_iata[sizeof(prefs->mqtt_iata) - 1] = '\0'; + } + + if (MQTT_DEFAULT_TIMEZONE[0] != '\0') { + strncpy(prefs->timezone_string, MQTT_DEFAULT_TIMEZONE, sizeof(prefs->timezone_string) - 1); + prefs->timezone_string[sizeof(prefs->timezone_string) - 1] = '\0'; + } + prefs->timezone_offset = MQTT_DEFAULT_TIMEZONE_OFFSET; + + // Observer non-MQTT defaults (moved out of NodePrefs/MyMesh ctor in Phase 2). + strncpy(prefs->snmp_community, "public", sizeof(prefs->snmp_community) - 1); + prefs->radio_watchdog_minutes = 5; + prefs->alert_wifi_minutes = 30; + prefs->alert_mqtt_minutes = 240; + prefs->alert_min_interval_min = 60; + + // Neighbors publishing defaults off; a defaulted tail is a valid 24h interval + // (not 0) so an in-lineage upgrade from a pre-neighbors payload is sane. + prefs->mqtt_neighbors_enabled = 0; + prefs->mqtt_neighbors_interval = MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS; +} + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTLifecycle.h b/src/helpers/MQTTLifecycle.h new file mode 100644 index 0000000000..d85aafa9d1 --- /dev/null +++ b/src/helpers/MQTTLifecycle.h @@ -0,0 +1,328 @@ +#pragma once + +#include + +// Fork-owned, dependency-free MQTT bridge lifecycle state machine and the +// narrow dependency seam used to drive it deterministically in host tests. +// +// This is the Phase 4 "ownership and teardown test seam" from +// STABILITY_TESTABILITY_HANDOFF.md. It is intentionally pure (no Arduino, +// FreeRTOS, WiFi, or PsychicMqttClient dependencies) so the exact +// start/stop/restart contract can be exercised with a fake clock and a +// recording Ops double, the same way MQTTConnectionPolicy.h and +// MQTTRuntimeBufferLifecycle.h are tested. +// +// Scope boundary (Phase 4 vs Phase 5): this header is the SPEC and the test +// seam. It is not yet wired into MQTTBridge. Phase 5 ("Implement cooperative +// MQTT shutdown") supplies FreeRTOS/PsychicMqttClient-backed Ops and replaces +// the abrupt vTaskDelete in MQTTBridge::end() with this cooperative lifecycle. +// See MQTT_OWNERSHIP.md for the ownership model and the migration plan. +// +// Behavior source (Phase 0 discipline): every transition and invariant encoded +// here is derived from the current MQTTBridge.cpp control flow (begin()/end(), +// mqttTaskLoop(), the volatile handshakes) — not from a hardware soak. Values +// that require on-hardware characterization (the concrete stop timeout, exact +// mbedTLS teardown timing) are called out with "Phase 0 TODO" and left as +// injectable parameters rather than guessed constants. +namespace MQTTLifecycle { + +// The lifecycle proposed by the handoff: +// Stopped -> Starting -> Running -> StopRequested -> Stopping -> Stopped +// +// StopRequested: a stop has been requested and delivered through the ownership +// channel, but the MQTT task has not yet begun its ordered shutdown. +// Stopping: the MQTT task is performing its ordered client/service +// shutdown. A StopBegan signal is optional; a task may ack directly from +// StopRequested if it does not report the intermediate step. +enum class State : uint8_t { + Stopped = 0, + Starting, + Running, + StopRequested, + Stopping, +}; + +// Events driven either by the owner (loop task) or by the MQTT task reporting +// its own progress. StopTimedOut is synthesized by the Coordinator when a stop +// is not acknowledged within the bounded timeout (the reviewed fallback). +enum class Event : uint8_t { + StartRequested = 0, // owner asked the bridge to start + StartCompleted, // MQTT task signalled init complete (StartAck) + StartFailed, // init failed on the task (partial-init rollback) + StopRequested, // owner (or OTA barrier) asked the bridge to stop + StopBegan, // MQTT task began its ordered shutdown (optional) + StopAcknowledged, // MQTT task signalled ordered shutdown complete (StopAck) + StopTimedOut, // bounded timeout expired with no StopAck (fallback) +}; + +// Side effects a transition asks the caller to perform. Naming WHO does WHAT +// keeps the Phase 5 production wiring and the host fakes on one contract: +// - create_task: owner creates/pins the MQTT task. +// - deliver_stop: owner delivers the stop request through the channel. +// - release_resources: owner may now free the queue, runtime buffers, and the +// task handle. Fires only after a completed/forced stop +// or an init-failure rollback — never mid-run. +// - ota_release: the OTA barrier's completion acknowledgment is now +// available (a stop reached a terminal state). +struct Effects { + bool create_task = false; + bool deliver_stop = false; + bool release_resources = false; + bool ota_release = false; +}; + +struct Result { + State next; + Effects effects; + bool accepted; // false => the event was a no-op in this state (idempotency) +}; + +// Unsigned subtraction is the standard millis() idiom and stays correct across +// a single 32-bit rollover (mirrors MQTTConnectionPolicy::elapsedMs). +inline uint32_t elapsedMs(uint32_t now, uint32_t then) { return now - then; } + +// Pure transition function. For a rejected/no-op event the result reports the +// unchanged state, no effects, and accepted == false. +inline Result apply(State s, Event e) { + Result r{s, Effects{}, false}; + switch (s) { + case State::Stopped: + // Only a start is meaningful. A stop while already stopped is a no-op + // (idempotent stop). StartFailed/StopAck cannot occur here. + if (e == Event::StartRequested) { + r.next = State::Starting; + r.effects.create_task = true; + r.accepted = true; + } + break; + + case State::Starting: + switch (e) { + case Event::StartCompleted: + r.next = State::Running; + r.accepted = true; + break; + case Event::StartFailed: + // Partial-init rollback: release only what the attempt acquired. + r.next = State::Stopped; + r.effects.release_resources = true; + r.accepted = true; + break; + case Event::StopRequested: + // Stop before full initialization: accept it and let the task ack. + r.next = State::StopRequested; + r.effects.deliver_stop = true; + r.accepted = true; + break; + default: + break; + } + break; + + case State::Running: + if (e == Event::StopRequested) { + r.next = State::StopRequested; + r.effects.deliver_stop = true; + r.accepted = true; + } + // A duplicate StartRequested/StartCompleted while Running is a no-op. + break; + + case State::StopRequested: + switch (e) { + case Event::StopBegan: + r.next = State::Stopping; + r.accepted = true; + break; + case Event::StopAcknowledged: + case Event::StopTimedOut: + r.next = State::Stopped; + r.effects.release_resources = true; + r.effects.ota_release = true; + r.accepted = true; + break; + default: + // Duplicate StopRequested is a no-op (idempotent stop). + break; + } + break; + + case State::Stopping: + switch (e) { + case Event::StopAcknowledged: + case Event::StopTimedOut: + r.next = State::Stopped; + r.effects.release_resources = true; + r.effects.ota_release = true; + r.accepted = true; + break; + default: + break; + } + break; + } + return r; +} + +// New connects/publishes/retries/reconfigurations are permitted only while the +// bridge is actively running (or still coming up). Once a stop is requested, +// all new work must cease (handoff: "Cessation of new connects, publishes, +// retries, and reconfigurations"). +inline bool acceptsNewWork(State s) { + return s == State::Starting || s == State::Running; +} + +// A late/stale client callback may touch owner-released resources ONLY before +// the owner has freed them. After the terminal Stopped state the queue, +// buffers, and clients may be gone, so a callback arriving then must be a +// no-op. (Callbacks that fire during StopRequested/Stopping run before +// release_resources and are still safe.) +inline bool mayTouchOwnedState(State s) { return s != State::Stopped; } + +// A restart (begin()) is safe only from a completed stop. +inline bool mayRestart(State s) { return s == State::Stopped; } + +inline bool isStopInProgress(State s) { + return s == State::StopRequested || s == State::Stopping; +} + +inline const char* stateName(State s) { + switch (s) { + case State::Stopped: return "Stopped"; + case State::Starting: return "Starting"; + case State::Running: return "Running"; + case State::StopRequested: return "StopRequested"; + case State::Stopping: return "Stopping"; + } + return "?"; +} + +inline const char* eventName(Event e) { + switch (e) { + case Event::StartRequested: return "StartRequested"; + case Event::StartCompleted: return "StartCompleted"; + case Event::StartFailed: return "StartFailed"; + case Event::StopRequested: return "StopRequested"; + case Event::StopBegan: return "StopBegan"; + case Event::StopAcknowledged: return "StopAcknowledged"; + case Event::StopTimedOut: return "StopTimedOut"; + } + return "?"; +} + +// Narrow dependency seam. These are the only dependencies the lifecycle needs +// to be driven deterministically (handoff: "for only the dependencies needed"). +// Phase 5 implements this over FreeRTOS + PsychicMqttClient; host tests +// implement it as a recording double with a settable clock. +// +// Dependencies enumerated by the handoff and where they land: +// - Clock/timer -> nowMs() +// - Task start/stop/ack/timeout -> startTask()/deliverStop() + the +// onTaskStarted()/onTaskStopped() callbacks +// into Coordinator, and tick() for timeout +// - Runtime allocator + queue -> folded into releaseResources() here; the +// allocate/free symmetry itself is already +// covered by MQTTRuntimeBufferLifecycle.h, +// and queue behavior by Phase 6 +// - MQTT client connect/disconnect + delayed callbacks -> modeled by the +// mayTouchOwnedState() guard (a callback +// decides whether it may touch owned state) +// - OTA coordinator/barrier -> onStopComplete(clean) + mayBeginFlash() +struct Ops { + virtual ~Ops() = default; + virtual uint32_t nowMs() = 0; // monotonic ms (millis()) + virtual void startTask() = 0; // create/pin the MQTT task + virtual void deliverStop() = 0; // signal stop through the channel + virtual void releaseResources() = 0; // free queue/buffers/task (post-stop) + // Unblock the OTA barrier's waiter. clean == true after a StopAcknowledged; + // clean == false after a StopTimedOut, so OTA aborts rather than flashing + // under uncertain ownership (handoff OTA barrier: "MQTT stop times out: OTA + // aborts safely rather than writing under uncertain ownership"). + virtual void onStopComplete(bool clean) = 0; +}; + +// Drives the state machine against injected Ops and hosts the bounded stop +// timeout. Idempotent start/stop; safe restart after a completed stop. +class Coordinator { + public: + // stop_timeout_ms is the bound on how long a requested stop may run before the + // reviewed force-kill fallback fires. It is injected (not a constant here) and + // may be updated per stop via setStopTimeoutMs(): Phase 0 hardware + // characterization (2026-07-19) showed real mbedTLS/wss teardown scales with + // the number of connected slots (~5-6 s each, sequential), so the owner sizes + // it to the current slot count before each stop. See MQTTBridge::end(). + Coordinator(Ops& ops, uint32_t stop_timeout_ms) + : _ops(ops), _stop_timeout_ms(stop_timeout_ms) {} + + State state() const { return _state; } + bool stopTimedOut() const { return _stop_timed_out; } + + bool acceptsNewWork() const { return MQTTLifecycle::acceptsNewWork(_state); } + bool mayTouchOwnedState() const { + return MQTTLifecycle::mayTouchOwnedState(_state); + } + bool isStopInProgress() const { + return MQTTLifecycle::isStopInProgress(_state); + } + // A restart is safe from a completed stop. A stop that reached Stopped via + // the timeout fallback still allows restart (the bridge is down); only OTA + // flashing is withheld after a dirty stop. + bool mayRestart() const { return MQTTLifecycle::mayRestart(_state); } + // OTA erase/write is permitted only after a CLEAN stop. A timed-out stop + // leaves ownership uncertain, so flashing stays blocked until a clean + // start/stop cycle clears the latch. + bool mayBeginFlash() const { + return _state == State::Stopped && !_stop_timed_out; + } + + // Update the stop-timeout bound. Call before requestStop() to size the window + // to the current slot count (see MQTTBridge::end()); the value is read by + // tick() against _stop_request_ms, which requestStop() arms afterwards. + void setStopTimeoutMs(uint32_t ms) { _stop_timeout_ms = ms; } + uint32_t stopTimeoutMs() const { return _stop_timeout_ms; } + + bool requestStart() { return dispatch(Event::StartRequested); } + bool requestStop() { return dispatch(Event::StopRequested); } + bool onTaskStarted() { return dispatch(Event::StartCompleted); } + bool onTaskStartFailed() { return dispatch(Event::StartFailed); } + bool onStopBegan() { return dispatch(Event::StopBegan); } + bool onTaskStopped() { return dispatch(Event::StopAcknowledged); } + + // Call periodically from the owner. Fires the reviewed timeout fallback if a + // requested stop has not been acknowledged within stop_timeout_ms. + void tick() { + if (MQTTLifecycle::isStopInProgress(_state) && + elapsedMs(_ops.nowMs(), _stop_request_ms) >= _stop_timeout_ms) { + dispatch(Event::StopTimedOut); + } + } + + private: + bool dispatch(Event e) { + const Result r = apply(_state, e); + if (!r.accepted) return false; + + _state = r.next; + if (e == Event::StartRequested) { + _stop_timed_out = false; // a fresh start clears the dirty-stop latch + } else if (e == Event::StopRequested) { + _stop_request_ms = _ops.nowMs(); // arm the timeout window + } else if (e == Event::StopTimedOut) { + _stop_timed_out = true; + } + + if (r.effects.create_task) _ops.startTask(); + if (r.effects.deliver_stop) _ops.deliverStop(); + if (r.effects.release_resources) _ops.releaseResources(); + if (r.effects.ota_release) _ops.onStopComplete(e == Event::StopAcknowledged); + return true; + } + + Ops& _ops; + uint32_t _stop_timeout_ms; + State _state = State::Stopped; + uint32_t _stop_request_ms = 0; + bool _stop_timed_out = false; +}; + +} // namespace MQTTLifecycle diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp new file mode 100644 index 0000000000..062f52edd9 --- /dev/null +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -0,0 +1,371 @@ +// MQTT-only translation unit. 22 variants re-glob helpers/*.cpp past the +// arduino_base exclusion, so the contents are guarded here rather than in +// the build filter — same idiom as helpers/esp32/WebConfigServer.cpp. +#ifdef WITH_MQTT_BRIDGE + +#include "MQTTMessageBuilder.h" +#include "MQTTPayloadBuilder.h" +#include +#include +#include +#include +#include +#include +#include "MeshCore.h" + +void MQTTMessageBuilder::formatIsoTimestampForMqtt(time_t now, long usec, Timezone* timezone, char* buffer, size_t buffer_size) { + if (!buffer || buffer_size == 0) return; + // Always emit UTC with an explicit "+00:00" offset, matching Python's + // datetime.now(timezone.utc).isoformat(). The system clock is UTC (SNTP offset 0), + // so gmtime() is correct regardless of the prefs Timezone (now unused here). + (void)timezone; + // Clamp the sub-second to a valid microsecond range so the "%06ld" field can never + // overflow to 7 digits or go negative on a bad clock read. + if (usec < 0) usec = 0; + else if (usec > 999999) usec = 999999; + struct tm* tm_info = gmtime(&now); + if (tm_info) { + size_t n = strftime(buffer, buffer_size, "%Y-%m-%dT%H:%M:%S", tm_info); + if (n > 0 && snprintf(buffer + n, buffer_size - n, ".%06ld+00:00", usec) > 0) { + return; + } + } + strncpy(buffer, "2024-01-01T12:00:00.000000+00:00", buffer_size - 1); + buffer[buffer_size - 1] = '\0'; +} + +int MQTTMessageBuilder::buildStatusMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* model, + const char* firmware_version, + const char* radio, + const char* client_version, + const char* status, + const char* timestamp, + char* buffer, + size_t buffer_size, + int battery_mv, + int uptime_secs, + int errors, + int queue_len, + int noise_floor, + int tx_air_secs, + int rx_air_secs, + int recv_errors, + int internal_heap, + int packets_sent, + int packets_received, + const char* repeat +) { + return MQTTPayloadBuilder::buildStatusMessage( + doc, origin, origin_id, model, firmware_version, radio, client_version, + status, timestamp, buffer, buffer_size, battery_mv, uptime_secs, errors, + queue_len, noise_floor, tx_air_secs, rx_air_secs, recv_errors, internal_heap, + packets_sent, packets_received, repeat); +} + +int MQTTMessageBuilder::buildPacketMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* direction, + const char* time, + const char* date, + int len, + int packet_type, + const char* route, + int payload_len, + const char* raw, + float snr, + int rssi, + float score, + const char* hash, + const uint8_t* path_bytes, + int path_hop_count, + int path_hash_size, + char* buffer, + size_t buffer_size +) { + return MQTTPayloadBuilder::buildPacketMessage( + doc, origin, origin_id, timestamp, direction, time, date, len, packet_type, + route, payload_len, raw, snr, rssi, score, hash, path_bytes, path_hop_count, + path_hash_size, MAX_PATH_SIZE, buffer, buffer_size); +} + +int MQTTMessageBuilder::buildRawMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* raw, + char* buffer, + size_t buffer_size +) { + return MQTTPayloadBuilder::buildRawMessage( + doc, origin, origin_id, timestamp, raw, buffer, buffer_size); +} + +int MQTTMessageBuilder::buildNeighborsMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + const NeighborsMessageEntry* neighbors, + int neighbor_count, + char* buffer, + size_t buffer_size, + int total_neighbors, + int queried_neighbors, + bool truncated +) { + return MQTTPayloadBuilder::buildNeighborsMessage( + doc, origin, origin_id, timestamp, self_scopes, self_default_scope, + neighbors, neighbor_count, buffer, buffer_size, total_neighbors, + queried_neighbors, truncated); +} + +size_t MQTTMessageBuilder::measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + int total_neighbors +) { + return MQTTPayloadBuilder::measureNeighborsMessageBase( + origin, origin_id, timestamp, self_scopes, self_default_scope, + total_neighbors); +} + +size_t MQTTMessageBuilder::measureNeighborsMessageEntry( + const NeighborsMessageEntry& neighbor +) { + return MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbor); +} + +int MQTTMessageBuilder::buildPacketJSON( + JsonDocument& doc, + mesh::Packet* packet, + bool is_tx, + const char* origin, + const char* origin_id, + Timezone* timezone, + char* buffer, + size_t buffer_size +) { + if (!packet) return 0; + + // One wall-clock read: tv_sec feeds both the timestamp and the UTC time/date + // fields below (kept consistent), tv_usec is the real sub-second. + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now = now_tv.tv_sec; + char timestamp[40]; + formatIsoTimestampForMqtt(now, now_tv.tv_usec, timezone, timestamp, sizeof(timestamp)); + + // Packet time/date: UTC (gmtime), same family as meshcoretomqtt serial fields + struct tm* utc_timeinfo = gmtime(&now); + + // Format time and date (ALWAYS UTC) + char time_str[16]; + char date_str[16]; + if (utc_timeinfo) { + strftime(time_str, sizeof(time_str), "%H:%M:%S", utc_timeinfo); + strftime(date_str, sizeof(date_str), "%d/%m/%Y", utc_timeinfo); + } else { + strcpy(time_str, "12:00:00"); + strcpy(date_str, "01/01/2024"); + } + + // Convert packet to hex + char raw_hex[WIRE_HEX_SCRATCH_SIZE]; + packetToHex(packet, raw_hex, sizeof(raw_hex)); + + // Get packet characteristics + int packet_type = packet->getPayloadType(); + const char* route_str = getRouteTypeString(packet->isRouteDirect() ? 1 : 0); + + // Create proper packet hash using MeshCore's calculatePacketHash method + char hash_str[17]; + uint8_t packet_hash[MAX_HASH_SIZE]; + packet->calculatePacketHash(packet_hash); + bytesToHex(packet_hash, MAX_HASH_SIZE, hash_str, sizeof(hash_str)); + + // Routing path (direct packets only): pass raw hop bytes to buildPacketMessage, + // which emits them as an array of lowercase hex hop tokens. + bool has_path = packet->isRouteDirect() && packet->getPathHashCount() > 0; + + return buildPacketMessage( + doc, + origin, origin_id, timestamp, + is_tx ? "tx" : "rx", + time_str, date_str, + packet->getRawLength(), + packet_type, route_str, + packet->payload_len, + raw_hex, + 12.5f, // SNR - using reasonable default + -65, // RSSI - using reasonable default + NAN, // score - unknown on this reconstruction-less fallback path + hash_str, + has_path ? packet->path : nullptr, + has_path ? packet->getPathHashCount() : 0, + has_path ? packet->getPathHashSize() : 0, + buffer, buffer_size + ); +} + +int MQTTMessageBuilder::buildPacketJSONFromRaw( + JsonDocument& doc, + const uint8_t* raw_data, + int raw_len, + mesh::Packet* packet, + bool is_tx, + const char* origin, + const char* origin_id, + float snr, + float rssi, + float score, + Timezone* timezone, + char* buffer, + size_t buffer_size +) { + if (!packet || !raw_data || raw_len <= 0) return 0; + + // One wall-clock read: tv_sec feeds both the timestamp and the UTC time/date + // fields below (kept consistent), tv_usec is the real sub-second. + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + time_t now = now_tv.tv_sec; + char timestamp[40]; + formatIsoTimestampForMqtt(now, now_tv.tv_usec, timezone, timestamp, sizeof(timestamp)); + + struct tm* utc_timeinfo = gmtime(&now); + + // Format time and date (ALWAYS UTC) + char time_str[16]; + char date_str[16]; + if (utc_timeinfo) { + strftime(time_str, sizeof(time_str), "%H:%M:%S", utc_timeinfo); + strftime(date_str, sizeof(date_str), "%d/%m/%Y", utc_timeinfo); + } else { + strcpy(time_str, "12:00:00"); + strcpy(date_str, "01/01/2024"); + } + + // Convert raw radio data to hex (this includes radio headers). bytesToHex() emits + // an empty string rather than truncating if raw_len exceeds the protocol maximum. + char raw_hex[WIRE_HEX_SCRATCH_SIZE]; + bytesToHex(raw_data, raw_len, raw_hex, sizeof(raw_hex)); + + // Get packet characteristics from the parsed packet + int packet_type = packet->getPayloadType(); + const char* route_str = getRouteTypeString(packet->isRouteDirect() ? 1 : 0); + + // Create proper packet hash using MeshCore's calculatePacketHash method + char hash_str[17]; + uint8_t packet_hash[MAX_HASH_SIZE]; + packet->calculatePacketHash(packet_hash); + bytesToHex(packet_hash, MAX_HASH_SIZE, hash_str, sizeof(hash_str)); + + // Routing path (direct packets only): pass raw hop bytes to buildPacketMessage, + // which emits them as an array of lowercase hex hop tokens. + bool has_path = packet->isRouteDirect() && packet->getPathHashCount() > 0; + + return buildPacketMessage( + doc, + origin, origin_id, timestamp, + is_tx ? "tx" : "rx", + time_str, date_str, + raw_len, // Use actual raw radio data length + packet_type, route_str, + packet->payload_len, + raw_hex, + snr, // Use actual SNR from radio + rssi, // Use actual RSSI from radio + score, // Firmware rebroadcast score (NaN for tx / when unavailable) + hash_str, + has_path ? packet->path : nullptr, + has_path ? packet->getPathHashCount() : 0, + has_path ? packet->getPathHashSize() : 0, + buffer, buffer_size + ); +} + +int MQTTMessageBuilder::buildRawJSON( + JsonDocument& doc, + mesh::Packet* packet, + const char* origin, + const char* origin_id, + Timezone* timezone, + char* buffer, + size_t buffer_size +) { + if (!packet) return 0; + + // One wall-clock read: tv_sec for the timestamp, tv_usec for the real sub-second. + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + char timestamp[40]; + formatIsoTimestampForMqtt(now_tv.tv_sec, now_tv.tv_usec, timezone, timestamp, sizeof(timestamp)); + + // Convert packet to hex + char raw_hex[WIRE_HEX_SCRATCH_SIZE]; + packetToHex(packet, raw_hex, sizeof(raw_hex)); + + return buildRawMessage(doc, origin, origin_id, timestamp, raw_hex, buffer, buffer_size); +} + +const char* MQTTMessageBuilder::getRouteTypeString(int route_type) { + switch (route_type) { + case 0: return "F"; // FLOOD + case 1: return "D"; // DIRECT + case 2: return "T"; // TRANSPORT_DIRECT + default: return "U"; // UNKNOWN + } +} + +void MQTTMessageBuilder::bytesToHex(const uint8_t* data, size_t len, char* hex, size_t hex_size) { + if (hex == nullptr || hex_size == 0) return; + // Guarantee a valid (empty) string even if we bail out below, so a caller's + // uninitialized stack buffer is never serialized into the JSON raw/hash fields + // when the buffer is too small (A6). + hex[0] = '\0'; + if (hex_size < len * 2 + 1) return; + + // Nibble lookup instead of a per-byte snprintf("%02X"): same uppercase hex + // output, but avoids re-parsing the format string up to ~512 times per publish. + static const char HEX_DIGITS[] = "0123456789ABCDEF"; + for (size_t i = 0; i < len; i++) { + hex[i * 2] = HEX_DIGITS[data[i] >> 4]; + hex[i * 2 + 1] = HEX_DIGITS[data[i] & 0x0F]; + } + hex[len * 2] = '\0'; +} + +void MQTTMessageBuilder::packetToHex(mesh::Packet* packet, char* hex, size_t hex_size) { + if (hex == nullptr || hex_size == 0) return; + // Empty string on any early-out below (serialization returned nothing, or the + // hex buffer is too small) so an uninitialized raw_hex[] never reaches the + // published JSON (A6). + hex[0] = '\0'; + // Serialize full on-air/wire format using Packet::writeTo() + // This includes header, transport codes (if present), path_len, path, and payload + uint8_t raw_buf[WIRE_SCRATCH_SIZE]; + if (!canSerializePacket(packet, sizeof(raw_buf))) return; + uint8_t raw_len = packet->writeTo(raw_buf); + if (raw_len == 0) return; + + // Check if hex buffer is large enough (2 hex chars per byte + null terminator) + if (hex_size < (size_t)raw_len * 2 + 1) return; + + // Convert serialized packet to hex + bytesToHex(raw_buf, raw_len, hex, hex_size); +} + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTMessageBuilder.h b/src/helpers/MQTTMessageBuilder.h new file mode 100644 index 0000000000..91a47b6eb0 --- /dev/null +++ b/src/helpers/MQTTMessageBuilder.h @@ -0,0 +1,270 @@ +#pragma once + +#include "MeshCore.h" +#include +#include "MQTTPayloadBuilder.h" +#include "MQTTWireScratch.h" +#include +#include + +/** + * @brief Utility class for building MQTT JSON messages + * + * This class handles the formatting of mesh packets and device status + * into JSON messages for MQTT publishing according to the MeshCore + * packet capture specification. + * + * Timestamps in the JSON `timestamp` field are emitted in UTC with an explicit + * "+00:00" offset (equivalent to Python datetime.now(timezone.utc).isoformat()) for + * status, packet, and raw topics. Packet payloads also include separate `time` and + * `date` strings in UTC (gmtime) so they stay aligned with meshcoretomqtt serial + * regex fields. + */ +class MQTTMessageBuilder { +public: + // Wire-format scratch sizing and validation live in the pure, host-tested + // MQTTWireScratch; these are the firmware-facing aliases. + static const size_t WIRE_SCRATCH_SIZE = MQTTWireScratch::kWireBytes; + static const size_t WIRE_HEX_SCRATCH_SIZE = MQTTWireScratch::kWireHexChars; + + static bool canSerializePacket(const mesh::Packet* packet, size_t dest_size) { + return packet != nullptr && MQTTWireScratch::canSerialize(*packet, dest_size); + } + + /** + * Format the MQTT JSON `timestamp` field (same rule for status, packet, raw). + * Always UTC with an explicit "+00:00" offset, ISO-8601 + * "%Y-%m-%dT%H:%M:%S.uuuuuu+00:00" (matches Python + * datetime.now(timezone.utc).isoformat()). The `timezone` parameter is retained + * for API compatibility but ignored — the system clock is UTC. + * + * `usec` is the sub-second component in microseconds (0..999999), normally taken + * from the same gettimeofday() read as `now` so the two don't tear at a second + * boundary. It is a real sub-second (SNTP-maintained wall clock), not a literal; + * pass 0 if no sub-second source is available. + */ + static void formatIsoTimestampForMqtt(time_t now, long usec, Timezone* timezone, char* buffer, size_t buffer_size); + + /** + * Build status message JSON + * + * @param origin Device name + * @param origin_id Device public key (hex string) + * @param model Device model + * @param firmware_version Firmware version + * @param radio Radio information + * @param client_version Client version + * @param status Connection status ("online" or "offline") + * @param timestamp ISO-like timestamp (see formatIsoTimestampForMqtt) + * @param buffer Output buffer for JSON string + * @param buffer_size Size of output buffer + * @param battery_mv Battery voltage in millivolts (optional, -1 to omit) + * @param uptime_secs Uptime in seconds (optional, -1 to omit) + * @param errors Error flags (optional, -1 to omit) + * @param queue_len Queue length (optional, -1 to omit) + * @param noise_floor Noise floor in dBm (optional, -999 to omit) + * @param tx_air_secs TX air time in seconds (optional, -1 to omit) + * @param rx_air_secs RX air time in seconds (optional, -1 to omit) + * @param recv_errors Radio receive/CRC errors (optional, -1 to omit) + * @param internal_heap Internal heap free bytes (optional, -1 to omit) + * @param repeat Repeat/forwarding status ("on" or "off"); nullptr omits the field + * @return Length of JSON string, or 0 on error + */ + static int buildStatusMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* model, + const char* firmware_version, + const char* radio, + const char* client_version, + const char* status, + const char* timestamp, + char* buffer, + size_t buffer_size, + int battery_mv = -1, + int uptime_secs = -1, + int errors = -1, + int queue_len = -1, + int noise_floor = -999, + int tx_air_secs = -1, + int rx_air_secs = -1, + int recv_errors = -1, + int internal_heap = -1, + int packets_sent = -1, + int packets_received = -1, + const char* repeat = nullptr + ); + + /** + * Build packet message JSON + * + * @param origin Device name + * @param origin_id Device public key (hex string) + * @param timestamp ISO-like timestamp (see formatIsoTimestampForMqtt) + * @param direction Packet direction ("rx" or "tx") + * @param time Time in HH:MM:SS (UTC, gmtime; meshcoretomqtt serial parity) + * @param date Date in DD/MM/YYYY (UTC, gmtime) + * @param len Total packet length + * @param packet_type Packet type code + * @param route Routing type + * @param payload_len Payload length + * @param raw Raw packet data (hex string) + * @param snr Signal-to-noise ratio + * @param rssi Received signal strength + * @param hash Packet hash + * @param path_bytes Raw routing-path bytes (direct packets only; nullptr to omit) + * @param path_hop_count Number of path hops (0 to omit the path field) + * @param path_hash_size Bytes per hop hash (1-4) + * @param buffer Output buffer for JSON string + * @param buffer_size Size of output buffer + * @return Length of JSON string, or 0 on error + */ + static int buildPacketMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* direction, + const char* time, + const char* date, + int len, + int packet_type, + const char* route, + int payload_len, + const char* raw, + float snr, + int rssi, + float score, + const char* hash, + const uint8_t* path_bytes, + int path_hop_count, + int path_hash_size, + char* buffer, + size_t buffer_size + ); + + /** + * Build raw message JSON + * + * @param origin Device name + * @param origin_id Device public key (hex string) + * @param timestamp ISO-like timestamp (see formatIsoTimestampForMqtt) + * @param raw Raw packet data (hex string) + * @param buffer Output buffer for JSON string + * @param buffer_size Size of output buffer + * @return Length of JSON string, or 0 on error + */ + static int buildRawMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* raw, + char* buffer, + size_t buffer_size + ); + + // Neighbors table entry + JSON builder. The layout logic lives in the pure, + // host-tested MQTTPayloadBuilder; this is the firmware-facing alias/delegate, + // matching the status/packet/raw builders. + using NeighborsMessageEntry = MQTTPayloadBuilder::NeighborsMessageEntry; + static int buildNeighborsMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + const NeighborsMessageEntry* neighbors, + int neighbor_count, + char* buffer, + size_t buffer_size, + int total_neighbors = -1, + int queried_neighbors = -1, + bool truncated = false + ); + static size_t measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + int total_neighbors + ); + static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& neighbor); + + /** + * Convert packet to JSON message + * + * @param packet Mesh packet + * @param is_tx Whether packet was transmitted (true) or received (false) + * @param origin Device name + * @param origin_id Device public key (hex string) + * @param buffer Output buffer for JSON string + * @param buffer_size Size of output buffer + * @return Length of JSON string, or 0 on error + */ + static int buildPacketJSON( + JsonDocument& doc, + mesh::Packet* packet, + bool is_tx, + const char* origin, + const char* origin_id, + Timezone* timezone, + char* buffer, + size_t buffer_size + ); + + static int buildPacketJSONFromRaw( + JsonDocument& doc, + const uint8_t* raw_data, + int raw_len, + mesh::Packet* packet, + bool is_tx, + const char* origin, + const char* origin_id, + float snr, + float rssi, + float score, + Timezone* timezone, + char* buffer, + size_t buffer_size + ); + + /** + * Convert packet to raw JSON message + * + * @param packet Mesh packet + * @param origin Device name + * @param origin_id Device public key (hex string) + * @param buffer Output buffer for JSON string + * @param buffer_size Size of output buffer + * @return Length of JSON string, or 0 on error + */ + static int buildRawJSON( + JsonDocument& doc, + mesh::Packet* packet, + const char* origin, + const char* origin_id, + Timezone* timezone, + char* buffer, + size_t buffer_size + ); + +private: + /** + * Convert route type to string + */ + static const char* getRouteTypeString(int route_type); + + /** + * Convert bytes to hex string (uppercase) + */ + static void bytesToHex(const uint8_t* data, size_t len, char* hex, size_t hex_size); + + /** + * Convert packet to hex string + */ + static void packetToHex(mesh::Packet* packet, char* hex, size_t hex_size); +}; diff --git a/src/helpers/MQTTObserverValidation.h b/src/helpers/MQTTObserverValidation.h new file mode 100644 index 0000000000..2042f2267c --- /dev/null +++ b/src/helpers/MQTTObserverValidation.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +// Pure, dependency-free validators for the observer's CLI/web configuration +// inputs. Factored out of CommonCLI_Observer.cpp so the exact logic the setters +// enforce can be unit-tested on the host (see test/test_observer_validation) +// rather than only through the full CLI object. + +// IATA region code: exactly three ASCII alphanumerics. The value is placed +// directly into MQTT topic paths (meshcore/{iata}/...), so anything else (wrong +// length, spaces, topic separators) is rejected. Case is preserved here; the +// setter uppercases after validation. +static inline bool mqttIataValid(const char* s) { + if (!s || strlen(s) != 3) return false; + for (int i = 0; i < 3; i++) { + char c = s[i]; + if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'))) { + return false; + } + } + return true; +} + +// Owner public key: exactly 64 hex characters (a 32-byte Ed25519 key), any case. +static inline bool mqttOwnerKeyValid(const char* s) { + if (!s || strlen(s) != 64) return false; + for (int i = 0; i < 64; i++) { + char c = s[i]; + if (!((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'))) { + return false; + } + } + return true; +} + +// NTP hostname: non-empty, <= 63 chars, made of letters/digits/'.'/'-', with no +// leading or trailing dot. ("none" is handled as a clear by the caller.) +static inline bool mqttNtpHostnameValid(const char* host) { + if (!host || host[0] == '\0') return false; + size_t len = strlen(host); + if (len > 63) return false; + if (host[0] == '.' || host[len - 1] == '.') return false; + for (size_t i = 0; i < len; i++) { + char c = host[i]; + if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '.' || c == '-')) { + return false; + } + } + return true; +} + +// A value fits its fixed destination buffer, which must hold the string plus a +// NUL terminator (so the usable length is bufsize - 1). Used to reject an +// over-long submission up front instead of silently truncating it. +static inline bool mqttValueFits(const char* s, size_t bufsize) { + return s != NULL && bufsize > 0 && strlen(s) < bufsize; +} diff --git a/src/helpers/MQTTPacketFilter.h b/src/helpers/MQTTPacketFilter.h new file mode 100644 index 0000000000..df81182827 --- /dev/null +++ b/src/helpers/MQTTPacketFilter.h @@ -0,0 +1,226 @@ +#pragma once + +#include +#include +#include + +// Pure per-broker packet-type allowlist helpers. MeshCore payload types occupy +// the low four bits of the packet header, so a uint16_t stores the complete +// 0..15 allowlist without dynamic allocation. +namespace MQTTPacketFilter { + +static const uint8_t kMinPacketType = 0; +static const uint8_t kMaxPacketType = 15; +static const uint16_t kAllPacketTypes = 0xFFFFu; +// "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" plus the terminator. +static const size_t kFilterTextSize = 38; + +inline bool isAsciiSpace(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; +} + +inline bool tokenEquals(const char* begin, size_t len, const char* token) { + return token != nullptr && strlen(token) == len && memcmp(begin, token, len) == 0; +} + +// Payload-type spellings accepted alongside the decimal form, mirroring the +// PAYLOAD_TYPE_* names in src/Packet.h. Types 12-14 are reserved upstream and +// have no name, so they stay reachable only by number. +struct NamedPacketType { + const char* name; + uint8_t type; +}; + +inline const NamedPacketType* namedPacketTypes(size_t* count_out) { + static const NamedPacketType kNames[] = { + {"req", 0}, {"response", 1}, {"txt_msg", 2}, {"ack", 3}, + {"advert", 4}, {"grp_txt", 5}, {"grp_data", 6}, {"anon_req", 7}, + {"path", 8}, {"trace", 9}, {"multipart", 10}, {"control", 11}, + {"raw_custom", 15}, + }; + if (count_out != nullptr) *count_out = sizeof(kNames) / sizeof(kNames[0]); + return kNames; +} + +// Resolve one already-trimmed list entry to a payload type. Decimal and named +// spellings are interchangeable within a list; "all"/"none" are whole-value +// keywords and deliberately do not resolve here, so "all,2" stays invalid. +inline bool resolveToken(const char* begin, size_t len, uint8_t* type_out) { + if (begin == nullptr || type_out == nullptr || len == 0) return false; + + if (*begin >= '0' && *begin <= '9') { + unsigned value = 0; + for (size_t i = 0; i < len; ++i) { + if (begin[i] < '0' || begin[i] > '9') return false; + value = value * 10u + static_cast(begin[i] - '0'); + if (value > kMaxPacketType) return false; + } + *type_out = static_cast(value); + return true; + } + + size_t name_count = 0; + const NamedPacketType* names = namedPacketTypes(&name_count); + for (size_t i = 0; i < name_count; ++i) { + if (tokenEquals(begin, len, names[i].name)) { + *type_out = names[i].type; + return true; + } + } + return false; +} + +// Parse an allowlist value. Empty input means "all" so WebConfig can clear a +// field and retain the same backwards-compatible default as an older +// /mqtt_prefs file. Keywords and type names are deliberately lowercase; "all" +// and "none" cannot be mixed into a list. Entries may be decimal (0..15) or +// named, may carry surrounding ASCII whitespace, and may be repeated. +inline bool parse(const char* input, uint16_t* mask_out) { + if (input == nullptr || mask_out == nullptr) return false; + + const char* begin = input; + while (*begin && isAsciiSpace(*begin)) begin++; + const char* end = begin + strlen(begin); + while (end > begin && isAsciiSpace(end[-1])) end--; + + const size_t len = static_cast(end - begin); + if (len == 0 || tokenEquals(begin, len, "all")) { + *mask_out = kAllPacketTypes; + return true; + } + if (tokenEquals(begin, len, "none")) { + *mask_out = 0; + return true; + } + + uint16_t parsed = 0; + const char* cursor = begin; + for (;;) { + const char* comma = cursor; + while (comma < end && *comma != ',') comma++; + + const char* token_begin = cursor; + const char* token_end = comma; + while (token_begin < token_end && isAsciiSpace(*token_begin)) token_begin++; + while (token_end > token_begin && isAsciiSpace(token_end[-1])) token_end--; + + uint8_t type = 0; + if (!resolveToken(token_begin, static_cast(token_end - token_begin), &type)) { + return false; + } + parsed |= static_cast(1u << type); + + if (comma >= end) break; + cursor = comma + 1; + } + + *mask_out = parsed; + return true; +} + +// Format masks deterministically for CLI/API output. Subsets are emitted in +// ascending order; the two useful extremes use concise keywords. +inline bool format(uint16_t mask, char* output, size_t output_size) { + if (output == nullptr || output_size == 0) return false; + output[0] = '\0'; + + const char* keyword = nullptr; + if (mask == kAllPacketTypes) keyword = "all"; + else if (mask == 0) keyword = "none"; + if (keyword != nullptr) { + const size_t len = strlen(keyword); + if (output_size <= len) return false; + memcpy(output, keyword, len + 1); + return true; + } + + char formatted[kFilterTextSize]; + size_t pos = 0; + bool first = true; + for (uint8_t type = kMinPacketType; type <= kMaxPacketType; ++type) { + if ((mask & static_cast(1u << type)) == 0) continue; + if (!first) formatted[pos++] = ','; + if (type >= 10) formatted[pos++] = '1'; + formatted[pos++] = static_cast('0' + (type % 10)); + first = false; + } + formatted[pos] = '\0'; + + if (output_size <= pos) return false; + memcpy(output, formatted, pos + 1); + return true; +} + +// The keyword shortcut above means format() never actually emits the whole +// 0..15 list, but the emit loop is written for every type, so the internal +// buffer must still hold it: 22 digits + 15 separators + NUL. +static_assert(kFilterTextSize >= 38, "filter text buffer must hold the full 0..15 list"); + +inline bool allows(uint16_t mask, uint8_t packet_type) { + return packet_type <= kMaxPacketType && + (mask & static_cast(1u << packet_type)) != 0; +} + +// How many types a mask allows. Lets a length-constrained reply fall back to +// "N/16" instead of a clipped list, which would read as a different allowlist. +inline uint8_t countTypes(uint16_t mask) { + uint8_t total = 0; + for (uint8_t type = kMinPacketType; type <= kMaxPacketType; ++type) { + if ((mask & static_cast(1u << type)) != 0) total++; + } + return total; +} + +// Conservative "could any configured broker want this type?" mask, used to +// reject a packet before it is copied into the publish queue. Disabled slots +// contribute nothing; a slot whose topic style later turns out not to support +// the publication is still counted here, so the gate never drops a packet the +// per-slot pass would have published. +inline uint16_t enabledUnion(const uint16_t* masks, const bool* enabled, size_t count) { + if (masks == nullptr) return 0; + uint16_t combined = 0; + for (size_t i = 0; i < count; ++i) { + if (enabled != nullptr && !enabled[i]) continue; + combined = static_cast(combined | masks[i]); + } + return combined; +} + +// True when every slot still carries the default all-types mask, i.e. nothing +// depends on the packet-filter tail of /mqtt_prefs being written. +inline bool allMasksDefault(const uint16_t* masks, size_t count) { + if (masks == nullptr) return true; + for (size_t i = 0; i < count; ++i) { + if (masks[i] != kAllPacketTypes) return false; + } + return true; +} + +// Stage one of the publish gate: everything decidable before a packet is +// serialised or a topic is built. Cheap enough to run per packet per slot. +inline bool slotCandidate(bool slot_enabled, uint16_t mask, uint8_t packet_type) { + return slot_enabled && allows(mask, packet_type); +} + +// The complete gate, once the slot's topic support is known. Eligibility +// deliberately excludes connection state: a temporarily disconnected broker +// that is configured for this type still requires the shared queue's existing +// bounded retry policy. +// +// eligiblePacketSlots() calls this directly, using slotCandidate() first only +// to skip the topic build for slots the mask already rejects. A slot that +// passes both therefore has its topic built again in the publish loop; that +// second build is the accepted cost of not carrying six 128-byte topics on the +// MQTT task stack. +inline bool slotEligible(bool slot_enabled, bool topic_supported, + uint16_t mask, uint8_t packet_type) { + return slotCandidate(slot_enabled, mask, packet_type) && topic_supported; +} + +// A fully filtered/topic-incompatible packet is intentionally complete. Once +// any eligible target exists, at least one actual publish must succeed. +inline bool publishComplete(bool has_eligible_target, bool any_publish_succeeded) { + return any_publish_succeeded || !has_eligible_target; +} + +} // namespace MQTTPacketFilter diff --git a/src/helpers/MQTTPacketQueuePolicy.h b/src/helpers/MQTTPacketQueuePolicy.h new file mode 100644 index 0000000000..4272d4c863 --- /dev/null +++ b/src/helpers/MQTTPacketQueuePolicy.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include + +// Pure queue/backpressure policy shared by the FreeRTOS and circular-buffer +// MQTT packet queues. Keeping the timing and retry decisions here makes the +// production behavior deterministic under host tests without mocking either +// queue implementation or the MQTT client. +namespace MQTTPacketQueuePolicy { + +static const uint32_t kDisconnectedStaleMs = 300000UL; +static const size_t kBacklogThreshold = 5; +static const uint8_t kGentleDrainCount = 1; +static const uint8_t kBurstDrainCount = 5; +static const uint32_t kGentleDrainBudgetMs = 30UL; +static const uint32_t kBurstDrainBudgetMs = 100UL; +static const uint8_t kMaxQos0RetryAttempts = 3; +static const uint32_t kRetryDelayBaseMs = 300UL; +static const uint32_t kRetryDelayJitterMs = 200UL; + +// Unsigned subtraction is the standard millis() idiom and remains correct +// across one 32-bit counter rollover. +static inline uint32_t elapsedMs(uint32_t now, uint32_t then) { + return now - then; +} + +enum class EnqueueAction : uint8_t { + Enqueue, + EvictOldestThenEnqueue, + Reject +}; + +static inline EnqueueAction enqueueAction(size_t queue_count, size_t capacity) { + if (capacity == 0) return EnqueueAction::Reject; + return queue_count >= capacity + ? EnqueueAction::EvictOldestThenEnqueue + : EnqueueAction::Enqueue; +} + +// disconnected_since == 0 means tracking has not started. The bridge records +// the first disconnected observation and asks this helper on later cycles. +static inline bool shouldFlushDisconnected(uint32_t now, + uint32_t disconnected_since, + uint32_t stale_ms = kDisconnectedStaleMs) { + return disconnected_since != 0 && elapsedMs(now, disconnected_since) >= stale_ms; +} + +struct DrainBudget { + uint8_t max_packets; + uint32_t max_time_ms; +}; + +static inline DrainBudget drainBudget(size_t queue_count) { + if (queue_count > kBacklogThreshold) { + return {kBurstDrainCount, kBurstDrainBudgetMs}; + } + return {kGentleDrainCount, kGentleDrainBudgetMs}; +} + +static inline bool drainTimeAvailable(uint32_t now, uint32_t started_at, + uint32_t budget_ms) { + // Preserve the bridge's inclusive boundary: work may begin at exactly the + // configured limit, but not one millisecond later. + return elapsedMs(now, started_at) <= budget_ms; +} + +// retry_attempts distinguishes an unscheduled packet from a scheduled retry +// whose deadline wrapped to exactly zero. Deadlines are always less than +// 500 ms away, so the half-range comparison is unambiguous. +static inline bool retryReady(uint32_t now, uint32_t next_retry_ms, + uint8_t retry_attempts) { + if (retry_attempts == 0) return true; + return elapsedMs(now, next_retry_ms) < 0x80000000UL; +} + +enum class RetryAction : uint8_t { + Complete, + Schedule, + Drop +}; + +struct RetryDecision { + RetryAction action; + uint8_t retry_attempts; + uint32_t delay_ms; + uint32_t next_retry_ms; +}; + +// A queued packet counts as delivered if EITHER its structured-packet publish +// or its raw-frame publish reached at least one slot. Partial success (one +// succeeds while the other fails or was not attempted) is still success — the +// packet completes and is not retried. This is the (packet, raw) outcome pairing +// fed to retryDecision(); naming it keeps the "partial publish = done" contract +// explicit and host-tested rather than inline in the bridge's queue drain. +static inline bool queuedPacketPublished(bool packet_published, + bool raw_published) { + return packet_published || raw_published; +} + +static inline RetryDecision retryDecision(bool any_published, + uint8_t retry_attempts, + uint32_t now) { + if (any_published) { + return {RetryAction::Complete, retry_attempts, 0, 0}; + } + if (retry_attempts >= kMaxQos0RetryAttempts) { + return {RetryAction::Drop, retry_attempts, 0, 0}; + } + + const uint32_t delay = kRetryDelayBaseMs + (now % kRetryDelayJitterMs); + return {RetryAction::Schedule, static_cast(retry_attempts + 1), + delay, now + delay}; +} + +} // namespace MQTTPacketQueuePolicy diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp new file mode 100644 index 0000000000..265969ae16 --- /dev/null +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -0,0 +1,295 @@ +#include "MQTTPayloadBuilder.h" + +#include +#include +#include + +namespace { + +static int serializeComplete(JsonObject root, char* buffer, size_t buffer_size) { + if (!buffer || buffer_size == 0) return 0; + + size_t written = serializeJson(root, buffer, buffer_size); + // Preserve MQTTMessageBuilder's existing success criterion while clearing + // ArduinoJson's truncated prefix on failure. Callers publish only a positive + // return value, and now a failed buffer cannot be mistaken for complete JSON. + if (written == 0 || written >= buffer_size) { + buffer[0] = '\0'; + return 0; + } + return static_cast(written); +} + +} // namespace + +int MQTTPayloadBuilder::buildStatusMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* model, + const char* firmware_version, + const char* radio, + const char* client_version, + const char* status, + const char* timestamp, + char* buffer, + size_t buffer_size, + int battery_mv, + int uptime_secs, + int errors, + int queue_len, + int noise_floor, + int tx_air_secs, + int rx_air_secs, + int recv_errors, + int internal_heap, + int packets_sent, + int packets_received, + const char* repeat +) { + doc.clear(); + JsonObject root = doc.to(); + + root["status"] = status; + root["timestamp"] = timestamp; + root["origin"] = origin; + root["origin_id"] = origin_id; + root["model"] = model; + root["firmware_version"] = firmware_version; + root["radio"] = radio; + root["client_version"] = client_version; + if (repeat != nullptr) { + root["repeat"] = repeat; + } + + if (battery_mv >= 0 || uptime_secs >= 0 || errors >= 0 || queue_len >= 0 || + noise_floor > -999 || tx_air_secs >= 0 || rx_air_secs >= 0 || recv_errors >= 0 || + internal_heap >= 0 || packets_sent >= 0 || packets_received >= 0) { + JsonObject stats = root["stats"].to(); + + if (battery_mv >= 0) stats["battery_mv"] = battery_mv; + if (uptime_secs >= 0) stats["uptime_secs"] = uptime_secs; + if (packets_sent >= 0) stats["packets_sent"] = packets_sent; + if (packets_received >= 0) stats["packets_received"] = packets_received; + if (errors >= 0) stats["errors"] = errors; + if (queue_len >= 0) stats["queue_len"] = queue_len; + if (noise_floor > -999) stats["noise_floor"] = noise_floor; + if (tx_air_secs >= 0) stats["tx_air_secs"] = tx_air_secs; + if (rx_air_secs >= 0) stats["rx_air_secs"] = rx_air_secs; + if (recv_errors >= 0) stats["recv_errors"] = recv_errors; + if (internal_heap >= 0) stats["internal_heap"] = internal_heap; + } + + return serializeComplete(root, buffer, buffer_size); +} + +int MQTTPayloadBuilder::buildPacketMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* direction, + const char* time, + const char* date, + int len, + int packet_type, + const char* route, + int payload_len, + const char* raw, + float snr, + int rssi, + float score, + const char* hash, + const uint8_t* path_bytes, + int path_hop_count, + int path_hash_size, + size_t max_path_bytes, + char* buffer, + size_t buffer_size +) { + doc.clear(); + JsonObject root = doc.to(); + + char len_str[16]; + char packet_type_str[16]; + char payload_len_str[16]; + char snr_str[16]; + char rssi_str[16]; + char score_str[16]; + + snprintf(len_str, sizeof(len_str), "%d", len); + snprintf(packet_type_str, sizeof(packet_type_str), "%d", packet_type); + snprintf(payload_len_str, sizeof(payload_len_str), "%d", payload_len); + snprintf(snr_str, sizeof(snr_str), "%.1f", snr); + snprintf(rssi_str, sizeof(rssi_str), "%d", rssi); + + root["timestamp"] = timestamp; + root["hash"] = hash; + root["origin"] = origin; + root["type"] = "PACKET"; + root["direction"] = direction; + root["time"] = time; + root["date"] = date; + root["len"] = len_str; + root["packet_type"] = packet_type_str; + root["route"] = route; + root["payload_len"] = payload_len_str; + root["raw"] = raw; + root["origin_id"] = origin_id; + + if (direction && strcmp(direction, "rx") == 0) { + root["SNR"] = snr_str; + root["RSSI"] = rssi_str; + if (!isnan(score)) { + snprintf(score_str, sizeof(score_str), "%d", static_cast(score * 1000)); + root["score"] = score_str; + } + } + + if (path_bytes && path_hop_count > 0 && path_hash_size > 0) { + JsonArray path_arr = root["path"].to(); + char hop_hex[2 * 4 + 1]; + for (int i = 0; i < path_hop_count; i++) { + size_t pos = 0; + for (int b = 0; b < path_hash_size && b < 4; b++) { + size_t idx = static_cast(i) * path_hash_size + b; + if (idx >= max_path_bytes) break; + snprintf(hop_hex + pos, 3, "%02x", path_bytes[idx]); + pos += 2; + } + hop_hex[pos] = '\0'; + path_arr.add(hop_hex); + } + } + + return serializeComplete(root, buffer, buffer_size); +} + +int MQTTPayloadBuilder::buildRawMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* raw, + char* buffer, + size_t buffer_size +) { + doc.clear(); + JsonObject root = doc.to(); + + root["origin"] = origin; + root["origin_id"] = origin_id; + root["timestamp"] = timestamp; + root["type"] = "RAW"; + root["data"] = raw; + + return serializeComplete(root, buffer, buffer_size); +} + +static JsonArray buildNeighborsMessageBase( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + int total_neighbors, + int queried_neighbors, + bool truncated +) { + doc.clear(); + JsonObject root = doc.to(); + root["timestamp"] = timestamp; + root["origin"] = origin; + root["origin_id"] = origin_id; + if (total_neighbors >= 0) { + root["total_neighbors"] = total_neighbors; + root["queried_neighbors"] = queried_neighbors >= 0 ? queried_neighbors : total_neighbors; + root["truncated"] = truncated; + } + + JsonObject self = root["self"].to(); + self["scopes"] = self_scopes ? self_scopes : ""; + self["default_scope"] = self_default_scope ? self_default_scope : ""; + return root["neighbors"].to(); +} + +static void addNeighborsMessageEntry( + JsonArray& arr, + const MQTTPayloadBuilder::NeighborsMessageEntry& neighbor +) { + JsonObject nb = arr.add(); + nb["pubkey"] = neighbor.pubkey_hex; + nb["snr"] = neighbor.snr; + if (neighbor.heard_unknown) { + nb["heard_secs_ago"] = nullptr; // age unknown, not zero + } else { + nb["heard_secs_ago"] = neighbor.heard_secs_ago; + } + nb["scopes"] = neighbor.scopes ? neighbor.scopes : ""; + nb["status"] = neighbor.status; +} + +size_t MQTTPayloadBuilder::measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + int total_neighbors +) { + JsonDocument doc; + buildNeighborsMessageBase( + doc, origin, origin_id, timestamp, self_scopes, self_default_scope, + total_neighbors, total_neighbors, false); + return measureJson(doc); +} + +size_t MQTTPayloadBuilder::measureNeighborsMessageEntry( + const NeighborsMessageEntry& neighbor +) { + JsonDocument doc; + JsonArray arr = doc.to(); + addNeighborsMessageEntry(arr, neighbor); + JsonObject entry = arr[0]; + return measureJson(entry); +} + +int MQTTPayloadBuilder::buildNeighborsMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + const NeighborsMessageEntry* neighbors, + int neighbor_count, + char* buffer, + size_t buffer_size, + int total_neighbors, + int queried_neighbors, + bool truncated +) { + if (!buffer || buffer_size == 0) return 0; + + JsonArray arr = buildNeighborsMessageBase( + doc, origin, origin_id, timestamp, self_scopes, self_default_scope, + total_neighbors, queried_neighbors, truncated); + if (doc.overflowed() || arr.isNull() || measureJson(doc) >= buffer_size) return 0; + + for (int i = 0; i < neighbor_count; i++) { + addNeighborsMessageEntry(arr, neighbors[i]); + if (doc.overflowed()) return 0; + + // Entries arrive ordered most- to least-useful. Stop as soon as the next + // one would fill the fixed publish buffer, dropping the remaining tail so + // document growth stays bounded. + if (measureJson(doc) >= buffer_size) { + arr.remove(arr.size() - 1); + if (total_neighbors >= 0) doc["truncated"] = true; + break; + } + } + + return serializeComplete(doc.as(), buffer, buffer_size); +} diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h new file mode 100644 index 0000000000..7bca170db1 --- /dev/null +++ b/src/helpers/MQTTPayloadBuilder.h @@ -0,0 +1,117 @@ +#pragma once + +#include +#include +#include + +// Mesh-independent JSON serialization core for MQTT publication payloads. +// MQTTMessageBuilder keeps the firmware-facing API and delegates these three +// deterministic contracts here so they can be exercised by native tests. +class MQTTPayloadBuilder { +public: + static int buildStatusMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* model, + const char* firmware_version, + const char* radio, + const char* client_version, + const char* status, + const char* timestamp, + char* buffer, + size_t buffer_size, + int battery_mv = -1, + int uptime_secs = -1, + int errors = -1, + int queue_len = -1, + int noise_floor = -999, + int tx_air_secs = -1, + int rx_air_secs = -1, + int recv_errors = -1, + int internal_heap = -1, + int packets_sent = -1, + int packets_received = -1, + const char* repeat = nullptr + ); + + static int buildPacketMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* direction, + const char* time, + const char* date, + int len, + int packet_type, + const char* route, + int payload_len, + const char* raw, + float snr, + int rssi, + float score, + const char* hash, + const uint8_t* path_bytes, + int path_hop_count, + int path_hash_size, + size_t max_path_bytes, + char* buffer, + size_t buffer_size + ); + + static int buildRawMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* raw, + char* buffer, + size_t buffer_size + ); + + struct NeighborsMessageEntry { + const char* pubkey_hex; + float snr; + uint32_t heard_secs_ago; + const char* scopes; + const char* status; + // True renders heard_secs_ago as JSON null, for a neighbour whose stored + // stamp cannot yield an age. No default initializer: the struct stays an + // aggregate for the device toolchain, and a zeroed tail means "age known", + // so size-measurement callers keep reserving the widest numeric value. + bool heard_unknown; + }; + + // Build neighbors-table JSON for the meshcore/{iata}/{device}/neighbors topic. + // Callers order entries most- to least-useful; document growth is bounded to + // buffer_size and the remaining tail is dropped once the next entry won't fit. + static int buildNeighborsMessage( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + const NeighborsMessageEntry* neighbors, + int neighbor_count, + char* buffer, + size_t buffer_size, + int total_neighbors = -1, + int queried_neighbors = -1, + bool truncated = false + ); + + // Exact serialized-size components used by paced neighbor discovery. The + // base reserves the largest progress metadata values for this snapshot; + // callers add each entry size plus one byte for commas after the first. + static size_t measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + const char* self_default_scope, + int total_neighbors + ); + static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& neighbor); +}; diff --git a/src/helpers/MQTTPrefsAtomicStore.h b/src/helpers/MQTTPrefsAtomicStore.h new file mode 100644 index 0000000000..be77bf99e1 --- /dev/null +++ b/src/helpers/MQTTPrefsAtomicStore.h @@ -0,0 +1,127 @@ +#pragma once + +#include +#include + +// Transactional writer for /mqtt_prefs. The Store interface is intentionally +// narrow so host tests can exercise every failure boundary without an Arduino +// filesystem: begin(), write(), finish(), commit(), and abort(). The caller +// supplies header and payload separately, avoiding a second full-size buffer. +namespace MQTTPrefsAtomicStore { + +enum class Result : uint8_t { + Committed, + BeginFailed, + HeaderWriteFailed, + PayloadWriteFailed, + FinishFailed, + CommitFailed, +}; + +inline bool committed(Result result) { + return result == Result::Committed; +} + +// Generic streaming transaction for structured images such as /com_prefs. +// ImageWriter writes its fields directly to Store and returns false on any +// short write, so no contiguous staging allocation is required. +enum class ImageResult : uint8_t { + Committed, + BeginFailed, + WriteFailed, + FinishFailed, + CommitFailed, +}; + +inline bool imageCommitted(ImageResult result) { + return result == ImageResult::Committed; +} + +template +inline ImageResult writeImage(Store& store, ImageWriter write_image) { + if (!store.begin()) { + store.abort(); + return ImageResult::BeginFailed; + } + if (!write_image(store)) { + store.abort(); + return ImageResult::WriteFailed; + } + if (!store.finish()) { + store.abort(); + return ImageResult::FinishFailed; + } + if (!store.commit()) { + store.abort(); + return ImageResult::CommitFailed; + } + return ImageResult::Committed; +} + +// Coordinates a two-file legacy upgrade. /com_prefs must not be compacted +// until the observer tail it carries has been published into /mqtt_prefs. +// Keeping this state in a tiny pure helper lets host tests cover power-cut +// boundaries without an Arduino filesystem. +class LegacyUpgradeGate { +public: + explicit LegacyUpgradeGate(bool com_prefs_rewrite_pending) + : _com_prefs_rewrite_pending(com_prefs_rewrite_pending) {} + + void requireMqttRewrite() { _mqtt_rewrite_pending = true; } + + void recordMqttSave(bool did_commit) { + if (did_commit) { + _mqtt_rewrite_pending = false; + _mqtt_source_held = false; + } else { + _mqtt_source_held = true; + } + } + + void holdMqttSource() { _mqtt_source_held = true; } + + bool mqttRewritePending() const { return _mqtt_rewrite_pending; } + bool blocksComPrefsRewrite() const { + return _com_prefs_rewrite_pending && (_mqtt_rewrite_pending || _mqtt_source_held); + } + bool mayRewriteComPrefs() const { + return _com_prefs_rewrite_pending && !blocksComPrefsRewrite(); + } + + void recordComPrefsRewrite() { + if (mayRewriteComPrefs()) _com_prefs_rewrite_pending = false; + } + +private: + bool _com_prefs_rewrite_pending; + bool _mqtt_rewrite_pending = false; + bool _mqtt_source_held = false; +}; + +template +inline Result write(Store& store, const uint8_t* header, size_t header_size, + const uint8_t* payload, size_t payload_size) { + if (!store.begin()) { + store.abort(); + return Result::BeginFailed; + } + if (store.write(header, header_size) != header_size) { + store.abort(); + return Result::HeaderWriteFailed; + } + if (store.write(payload, payload_size) != payload_size) { + store.abort(); + return Result::PayloadWriteFailed; + } + if (!store.finish()) { + store.abort(); + return Result::FinishFailed; + } + if (!store.commit()) { + store.abort(); + return Result::CommitFailed; + } + return Result::Committed; +} + +} // namespace MQTTPrefsAtomicStore diff --git a/src/helpers/MQTTPrefsCodec.h b/src/helpers/MQTTPrefsCodec.h new file mode 100644 index 0000000000..7bbfe30aeb --- /dev/null +++ b/src/helpers/MQTTPrefsCodec.h @@ -0,0 +1,489 @@ +#pragma once + +#include + +#include "MQTTPacketFilter.h" +#include "MQTTPrefsStorage.h" + +#ifdef WITH_MQTT_BRIDGE + +// Pure /mqtt_prefs format classification and field-copy migration. Production +// reads directly into the selected layout; this header never requires a second +// large staging buffer. +namespace MQTTPrefsCodec { + +enum class Source : uint8_t { + Defaults, + Current, + LegacyPreSlot, + LegacyThreeSlotBase, + LegacyThreeSlot, + LegacySixSlotBase, + LegacySixSlotAudience, + LegacySixSlotAudienceRx, + LegacySixSlot, + UnsupportedVersion, + Corrupt, +}; + +struct DecodePlan { + Source source; + bool rewrite_legacy; + bool preserve_file; + // False means the decoded payload stops before snmp_enabled, so production + // may apply a captured observer tail from legacy /com_prefs. + bool observer_fields_present; + size_t payload_len; +}; + +static const size_t kV1PreObserverPayloadSize = MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE; +static const size_t kV1PreNeighborsPayloadSize = MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE; +static const size_t kV1PreFilterPayloadSize = MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE; +static const size_t kV1BaselinePayloadSize = MQTT_PREFS_V1_FULL_PAYLOAD_SIZE; +static const size_t kEncodedSize = sizeof(MQTTPrefsHeader) + kV1BaselinePayloadSize; + +// Shortest payload length that still round-trips this configuration. +// +// The packet-filter tail is the only optional part of the current layout, and +// its default (all types) is exactly what a pre-filter decoder supplies for a +// missing tail. So a device whose filters are all default keeps writing the +// 2864-byte payload that pre-filter firmware can still read. That matters +// because /mqtt_prefs also carries the WiFi credentials: an unrecognised +// longer payload sends older firmware to defaults with no network, and it +// refuses to overwrite the file, so the node cannot be recovered over the air. +// Touching any filter opts that node into the longer payload — a deliberate, +// operator-initiated trade rather than a side effect of upgrading. +inline size_t payloadLenFor(const MQTTPrefs& prefs) { + return MQTTPacketFilter::allMasksDefault(prefs.mqtt_slot_packet_filter, + MQTT_PREFS_SLOT_COUNT) + ? kV1PreFilterPayloadSize + : kV1BaselinePayloadSize; +} + +inline MQTTPrefsHeader makeHeader(size_t payload_len) { + MQTTPrefsHeader header; + memcpy(header.magic, MQTT_PREFS_MAGIC, sizeof(header.magic)); + header.version = MQTT_PREFS_VERSION; + header.payload_len = static_cast(payload_len); + return header; +} + +inline size_t encode(const MQTTPrefs& prefs, uint8_t* output, size_t output_size) { + const size_t payload_len = payloadLenFor(prefs); + const size_t encoded_size = sizeof(MQTTPrefsHeader) + payload_len; + if (output == nullptr || output_size < encoded_size) return 0; + const MQTTPrefsHeader header = makeHeader(payload_len); + memcpy(output, &header, sizeof(header)); + memcpy(output + sizeof(header), &prefs, payload_len); + return encoded_size; +} + +inline bool isMagicPrefix(const uint8_t* input, size_t available) { + if (input == nullptr || available == 0) return false; + const size_t compare_len = available < sizeof(MQTT_PREFS_MAGIC) + ? available : sizeof(MQTT_PREFS_MAGIC); + return memcmp(input, MQTT_PREFS_MAGIC, compare_len) == 0; +} + +inline DecodePlan corruptPlan() { + return {Source::Corrupt, false, true, false, 0}; +} + +// Classify from the first eight bytes and the filesystem-reported file size. +// Headerless layouts are an explicit, audited whitelist. Call +// isPlausibleLegacy() after reading the selected layout and before rewriting: +// size alone cannot distinguish a valid legacy payload from arbitrary bytes. +inline DecodePlan classify(const uint8_t* prefix, size_t prefix_read, size_t file_size) { + if (file_size == 0) return corruptPlan(); + if (prefix == nullptr || prefix_read == 0) return corruptPlan(); + const size_t expected_prefix = file_size < sizeof(MQTTPrefsHeader) + ? file_size : sizeof(MQTTPrefsHeader); + if (prefix_read < expected_prefix) return corruptPlan(); + + if (file_size < sizeof(MQTTPrefsHeader) && isMagicPrefix(prefix, prefix_read)) { + return corruptPlan(); + } + if (file_size >= sizeof(MQTTPrefsHeader)) { + MQTTPrefsHeader header; + memcpy(&header, prefix, sizeof(header)); + if (memcmp(header.magic, MQTT_PREFS_MAGIC, sizeof(header.magic)) == 0) { + if (header.version != MQTT_PREFS_VERSION) { + return {Source::UnsupportedVersion, false, true, false, 0}; + } + const size_t payload_available = file_size - sizeof(header); + if (header.payload_len != payload_available) { + return corruptPlan(); + } + // A longer same-version payload was written by a later build that + // appended fields. Within a version tag the layout is append-only, so + // every byte this binary knows is present and correctly positioned — read + // the baseline prefix and ignore the tail. + // + // This is the downgrade contract, and it is deliberately asymmetric: + // refusing the file would cost the operator WiFi credentials and every + // broker slot (a node with no network and no portal, recoverable only + // over serial), whereas reading it costs only the settings the newer + // build added. Losing a later feature's settings is the acceptable half. + // + // The tail survives until something actually writes: saveMQTTPrefs() + // rewrites at this binary's own length, so a rollback that changes no + // observer setting and is later rolled forward keeps the newer fields + // intact. Only an explicit `set` while downgraded drops them. + // + // A layout change that is NOT a pure append must bump MQTT_PREFS_VERSION; + // the version check above is what makes this rule safe. + if (header.payload_len > kV1BaselinePayloadSize) { + return {Source::Current, false, false, true, kV1BaselinePayloadSize}; + } + if (header.payload_len == kV1BaselinePayloadSize) { + return {Source::Current, false, false, true, kV1BaselinePayloadSize}; + } + if (header.payload_len == kV1PreFilterPayloadSize) { + // Written before the per-slot packet-filter tail. Defaults supply an + // all-types mask for every slot. + return {Source::Current, false, false, true, kV1PreFilterPayloadSize}; + } + if (header.payload_len == kV1PreNeighborsPayloadSize) { + // Written by observer/webconfig firmware before the neighbors tail + // existed. The observer fields ARE present; only the neighbors tail is + // missing, so it loads and keeps its defaults (off / 24h). + return {Source::Current, false, false, true, kV1PreNeighborsPayloadSize}; + } + if (header.payload_len == kV1PreObserverPayloadSize) { + return {Source::Current, false, false, false, kV1PreObserverPayloadSize}; + } + return corruptPlan(); + } + } + + switch (file_size) { + case sizeof(OldMQTTPrefs): + return {Source::LegacyPreSlot, true, false, false, file_size}; + case sizeof(ThreeSlotBaseMQTTPrefs): + return {Source::LegacyThreeSlotBase, true, false, false, file_size}; + case sizeof(ThreeSlotMQTTPrefs): + return {Source::LegacyThreeSlot, true, false, false, file_size}; + case LEGACY6_BASE_SIZE: + return {Source::LegacySixSlotBase, true, false, false, file_size}; + case LEGACY6_AUDIENCE_SIZE: + return {Source::LegacySixSlotAudience, true, false, false, file_size}; + case LEGACY6_AUDIENCE_RX_SIZE: + return {Source::LegacySixSlotAudienceRx, true, false, false, file_size}; + case sizeof(Legacy6SlotMQTTPrefs): + return {Source::LegacySixSlot, true, false, false, file_size}; + default: + return corruptPlan(); + } +} + +inline bool looksLikePreWifiPower(const uint8_t* input, size_t size) { + if (input == nullptr || size != sizeof(OldMQTTPrefs)) return false; + // At byte 144 the newer layout has wifi_power_save (0..2); the older + // layout has timezone_string[0]. A non-empty timezone is unambiguous. + if (input[144] > 2) return true; + // With an empty timezone, byte 177 is the older mqtt_server[0] but the + // newer timezone_offset. If it cannot be an offset, it is also unambiguous. + const int8_t newer_offset = static_cast(input[177]); + // Byte 176 is the older timezone_offset but the final byte of the newer + // timezone string (always NUL for values saved through the CLI). + return input[144] == 0 && + (input[176] != 0 || newer_offset < -12 || newer_offset > 14); +} + +// Headerless files have no checksum or magic, so their integrity cannot be +// proven. These checks intentionally reject obvious random data (unterminated +// strings and impossible flag/range values) without demanding application-level +// values that a real but sparsely configured device may not have set. +inline bool hasTerminatedText(const uint8_t* input, size_t size, size_t offset, size_t field_size) { + if (input == nullptr || offset > size || field_size > size - offset) return false; + for (size_t i = 0; i < field_size; ++i) { + if (input[offset + i] == '\0') return true; + } + return false; +} + +inline bool hasPlausibleCommonFields(const uint8_t* input, size_t size, bool pre_wifi_power) { + if (input == nullptr || size < sizeof(OldMQTTPrefs)) return false; + const size_t timezone_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, timezone_string) + : offsetof(OldMQTTPrefs, timezone_string); + const size_t utc_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, timezone_offset) + : offsetof(OldMQTTPrefs, timezone_offset); + const size_t timezone_size = 32; + const int8_t timezone_hours = static_cast(input[utc_offset]); + + return input[offsetof(OldMQTTPrefs, mqtt_status_enabled)] <= 1 && + input[offsetof(OldMQTTPrefs, mqtt_packets_enabled)] <= 1 && + input[offsetof(OldMQTTPrefs, mqtt_raw_enabled)] <= 1 && + input[offsetof(OldMQTTPrefs, mqtt_tx_enabled)] <= 2 && + (pre_wifi_power || input[offsetof(OldMQTTPrefs, wifi_power_save)] <= 2) && + timezone_hours >= -12 && timezone_hours <= 14 && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, mqtt_origin), + 32) && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, mqtt_iata), + 8) && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, wifi_ssid), + 32) && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, wifi_password), + 64) && + hasTerminatedText(input, size, timezone_offset, timezone_size); +} + +inline bool hasPlausibleSlotText(const uint8_t* input, size_t size, size_t slot_count, + size_t preset_offset, size_t host_offset, size_t username_offset, + size_t password_offset, size_t token_offset, size_t topic_offset, + size_t audience_offset) { + const size_t no_field = static_cast(-1); + for (size_t i = 0; i < slot_count; ++i) { + if (!hasTerminatedText(input, size, preset_offset + i * 24, 24) || + !hasTerminatedText(input, size, host_offset + i * 64, 64) || + !hasTerminatedText(input, size, username_offset + i * 32, 32) || + !hasTerminatedText(input, size, password_offset + i * 64, 64) || + (token_offset != no_field && !hasTerminatedText(input, size, token_offset + i * 48, 48)) || + (topic_offset != no_field && !hasTerminatedText(input, size, topic_offset + i * 96, 96)) || + (audience_offset != no_field && !hasTerminatedText(input, size, audience_offset + i * 64, 64))) { + return false; + } + } + return true; +} + +inline bool hasPlausibleSharedAuth(const uint8_t* input, size_t size, size_t owner_offset, + size_t email_offset) { + return hasTerminatedText(input, size, owner_offset, 65) && + hasTerminatedText(input, size, email_offset, 64); +} + +inline bool isPlausibleLegacy(Source source, const uint8_t* input, size_t size) { + const size_t no_field = static_cast(-1); + switch (source) { + case Source::LegacyPreSlot: { + if (size != sizeof(OldMQTTPrefs)) return false; + const bool pre_wifi_power = looksLikePreWifiPower(input, size); + const size_t server_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_server) + : offsetof(OldMQTTPrefs, mqtt_server); + const size_t username_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_username) + : offsetof(OldMQTTPrefs, mqtt_username); + const size_t password_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_password) + : offsetof(OldMQTTPrefs, mqtt_password); + const size_t us_enabled = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_analyzer_us_enabled) + : offsetof(OldMQTTPrefs, mqtt_analyzer_us_enabled); + const size_t eu_enabled = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_analyzer_eu_enabled) + : offsetof(OldMQTTPrefs, mqtt_analyzer_eu_enabled); + return hasPlausibleCommonFields(input, size, pre_wifi_power) && + input[us_enabled] <= 1 && input[eu_enabled] <= 1 && + hasTerminatedText(input, size, server_offset, 64) && + hasTerminatedText(input, size, username_offset, 32) && + hasTerminatedText(input, size, password_offset, 64) && + hasTerminatedText(input, size, pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_owner_public_key) + : offsetof(OldMQTTPrefs, mqtt_owner_public_key), 65) && + hasTerminatedText(input, size, pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_email) + : offsetof(OldMQTTPrefs, mqtt_email), 64); + } + case Source::LegacyThreeSlotBase: + return size == sizeof(ThreeSlotBaseMQTTPrefs) && + hasPlausibleCommonFields(input, size, false) && + hasPlausibleSharedAuth(input, size, + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_owner_public_key), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_email)) && + hasPlausibleSlotText(input, size, 3, + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_preset), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_host), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_username), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_password), + no_field, no_field, no_field); + case Source::LegacyThreeSlot: + return size == sizeof(ThreeSlotMQTTPrefs) && + hasPlausibleCommonFields(input, size, false) && + hasPlausibleSharedAuth(input, size, + offsetof(ThreeSlotMQTTPrefs, mqtt_owner_public_key), + offsetof(ThreeSlotMQTTPrefs, mqtt_email)) && + hasPlausibleSlotText(input, size, 3, + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_preset), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_host), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_username), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_password), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_token), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_topic), no_field); + case Source::LegacySixSlotBase: + case Source::LegacySixSlotAudience: + case Source::LegacySixSlotAudienceRx: + case Source::LegacySixSlot: { + const size_t expected_size = source == Source::LegacySixSlotBase ? LEGACY6_BASE_SIZE + : source == Source::LegacySixSlotAudience ? LEGACY6_AUDIENCE_SIZE + : source == Source::LegacySixSlotAudienceRx ? LEGACY6_AUDIENCE_RX_SIZE + : sizeof(Legacy6SlotMQTTPrefs); + const size_t audience_offset = source == Source::LegacySixSlotBase + ? no_field : offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_audience); + const bool has_rx = source == Source::LegacySixSlotAudienceRx || + source == Source::LegacySixSlot; + const bool has_ntp = source == Source::LegacySixSlot; + return size == expected_size && hasPlausibleCommonFields(input, size, false) && + hasPlausibleSharedAuth(input, size, + offsetof(Legacy6SlotMQTTPrefs, mqtt_owner_public_key), + offsetof(Legacy6SlotMQTTPrefs, mqtt_email)) && + (!has_rx || input[offsetof(Legacy6SlotMQTTPrefs, mqtt_rx_enabled)] <= 1) && + (!has_ntp || hasTerminatedText(input, size, + offsetof(Legacy6SlotMQTTPrefs, mqtt_ntp_server), 64)) && + hasPlausibleSlotText(input, size, MQTT_PREFS_SLOT_COUNT, + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_preset), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_host), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_username), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_password), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_token), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_topic), audience_offset); + } + default: + return false; + } +} + +inline void migratePreSlot(const OldMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, + sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); + strncpy(prefs->mqtt_slot_preset[0], old_prefs.mqtt_analyzer_us_enabled == 1 + ? "analyzer-us" : "none", sizeof(prefs->mqtt_slot_preset[0]) - 1); + strncpy(prefs->mqtt_slot_preset[1], old_prefs.mqtt_analyzer_eu_enabled == 1 + ? "analyzer-eu" : "none", sizeof(prefs->mqtt_slot_preset[1]) - 1); + if (old_prefs.mqtt_server[0] != '\0' && old_prefs.mqtt_port > 0) { + strncpy(prefs->mqtt_slot_preset[2], "custom", sizeof(prefs->mqtt_slot_preset[2]) - 1); + strncpy(prefs->mqtt_slot_host[2], old_prefs.mqtt_server, + sizeof(prefs->mqtt_slot_host[2]) - 1); + prefs->mqtt_slot_port[2] = old_prefs.mqtt_port; + strncpy(prefs->mqtt_slot_username[2], old_prefs.mqtt_username, + sizeof(prefs->mqtt_slot_username[2]) - 1); + strncpy(prefs->mqtt_slot_password[2], old_prefs.mqtt_password, + sizeof(prefs->mqtt_slot_password[2]) - 1); + } else { + strncpy(prefs->mqtt_slot_preset[2], "none", sizeof(prefs->mqtt_slot_preset[2]) - 1); + } +} + +inline void migratePreWifiPower(const PreWifiPowerOldMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + OldMQTTPrefs normalized = {}; + memcpy(normalized.mqtt_origin, old_prefs.mqtt_origin, sizeof(normalized.mqtt_origin)); + memcpy(normalized.mqtt_iata, old_prefs.mqtt_iata, sizeof(normalized.mqtt_iata)); + normalized.mqtt_status_enabled = old_prefs.mqtt_status_enabled; + normalized.mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + normalized.mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + normalized.mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + normalized.mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(normalized.wifi_ssid, old_prefs.wifi_ssid, sizeof(normalized.wifi_ssid)); + memcpy(normalized.wifi_password, old_prefs.wifi_password, sizeof(normalized.wifi_password)); + normalized.wifi_power_save = prefs->wifi_power_save; // field did not exist yet + memcpy(normalized.timezone_string, old_prefs.timezone_string, sizeof(normalized.timezone_string)); + normalized.timezone_offset = old_prefs.timezone_offset; + memcpy(normalized.mqtt_server, old_prefs.mqtt_server, sizeof(normalized.mqtt_server)); + normalized.mqtt_port = old_prefs.mqtt_port; + memcpy(normalized.mqtt_username, old_prefs.mqtt_username, sizeof(normalized.mqtt_username)); + memcpy(normalized.mqtt_password, old_prefs.mqtt_password, sizeof(normalized.mqtt_password)); + normalized.mqtt_analyzer_us_enabled = old_prefs.mqtt_analyzer_us_enabled; + normalized.mqtt_analyzer_eu_enabled = old_prefs.mqtt_analyzer_eu_enabled; + memcpy(normalized.mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, + sizeof(normalized.mqtt_owner_public_key)); + memcpy(normalized.mqtt_email, old_prefs.mqtt_email, sizeof(normalized.mqtt_email)); + migratePreSlot(normalized, prefs); +} + +template +inline void migrateThreeSlotCommon(const T& old_prefs, MQTTPrefs* prefs) { + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + for (int i = 0; i < 3; i++) { + memcpy(prefs->mqtt_slot_preset[i], old_prefs.mqtt_slot_preset[i], sizeof(prefs->mqtt_slot_preset[i])); + memcpy(prefs->mqtt_slot_host[i], old_prefs.mqtt_slot_host[i], sizeof(prefs->mqtt_slot_host[i])); + prefs->mqtt_slot_port[i] = old_prefs.mqtt_slot_port[i]; + memcpy(prefs->mqtt_slot_username[i], old_prefs.mqtt_slot_username[i], sizeof(prefs->mqtt_slot_username[i])); + memcpy(prefs->mqtt_slot_password[i], old_prefs.mqtt_slot_password[i], sizeof(prefs->mqtt_slot_password[i])); + } + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); +} + +inline void migrateThreeSlot(const ThreeSlotBaseMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + migrateThreeSlotCommon(old_prefs, prefs); +} + +inline void migrateThreeSlot(const ThreeSlotMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + migrateThreeSlotCommon(old_prefs, prefs); + for (int i = 0; i < 3; i++) { + memcpy(prefs->mqtt_slot_token[i], old_prefs.mqtt_slot_token[i], sizeof(prefs->mqtt_slot_token[i])); + memcpy(prefs->mqtt_slot_topic[i], old_prefs.mqtt_slot_topic[i], sizeof(prefs->mqtt_slot_topic[i])); + } +} + +inline void migrateLegacySixSlotCommon(const Legacy6SlotMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + memcpy(prefs->mqtt_slot_preset, old_prefs.mqtt_slot_preset, sizeof(prefs->mqtt_slot_preset)); + memcpy(prefs->mqtt_slot_host, old_prefs.mqtt_slot_host, sizeof(prefs->mqtt_slot_host)); + memcpy(prefs->mqtt_slot_port, old_prefs.mqtt_slot_port, sizeof(prefs->mqtt_slot_port)); + memcpy(prefs->mqtt_slot_username, old_prefs.mqtt_slot_username, sizeof(prefs->mqtt_slot_username)); + memcpy(prefs->mqtt_slot_password, old_prefs.mqtt_slot_password, sizeof(prefs->mqtt_slot_password)); + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); + memcpy(prefs->mqtt_slot_token, old_prefs.mqtt_slot_token, sizeof(prefs->mqtt_slot_token)); + memcpy(prefs->mqtt_slot_topic, old_prefs.mqtt_slot_topic, sizeof(prefs->mqtt_slot_topic)); +} + +inline void migrateLegacySixSlot(const Legacy6SlotMQTTPrefs& old_prefs, Source source, + MQTTPrefs* prefs) { + migrateLegacySixSlotCommon(old_prefs, prefs); + if (source == Source::LegacySixSlotAudience || source == Source::LegacySixSlotAudienceRx || + source == Source::LegacySixSlot) { + memcpy(prefs->mqtt_slot_audience, old_prefs.mqtt_slot_audience, + sizeof(prefs->mqtt_slot_audience)); + } + if (source == Source::LegacySixSlotAudienceRx || source == Source::LegacySixSlot) { + prefs->mqtt_rx_enabled = old_prefs.mqtt_rx_enabled; + } + if (source == Source::LegacySixSlot) { + memcpy(prefs->mqtt_ntp_server, old_prefs.mqtt_ntp_server, + sizeof(prefs->mqtt_ntp_server)); + } +} + +} // namespace MQTTPrefsCodec + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsRecovery.h b/src/helpers/MQTTPrefsRecovery.h new file mode 100644 index 0000000000..7534678eb3 --- /dev/null +++ b/src/helpers/MQTTPrefsRecovery.h @@ -0,0 +1,49 @@ +#pragma once + +#include + +// Pure recovery policy for the three MQTT preference transaction files. The +// writer first moves the old primary to .bak, then moves the verified .tmp to +// the primary name. On a reset, the loader uses this policy before decoding +// /mqtt_prefs. "Preserve" is deliberately distinct from "Usable": it covers +// an unsupported newer layout, corruption, or an unreadable file and must +// never be replaced by an older image. +namespace MQTTPrefsRecovery { + +enum class FileState : uint8_t { + Missing, + Usable, + Preserve, +}; + +enum class Action : uint8_t { + None, + KeepPrimary, + PromoteTemp, + PromoteBackup, +}; + +inline Action select(FileState primary, FileState temp, FileState backup) { + // A primary of any kind owns the name. In particular, do not roll a newer + // or corrupt primary back to an older backup just because it cannot be read + // by this firmware. + if (primary != FileState::Missing) return Action::KeepPrimary; + + // A completed temp is the new image and wins over the old backup. + if (temp == FileState::Usable) return Action::PromoteTemp; + + // If temp is opaque but a known-good backup exists, boot from the backup. + // The caller may discard the opaque temp once that usable backup has become + // primary. Otherwise, rename the opaque temp into the empty primary name so + // the normal loader can hold it. + if (temp == FileState::Preserve) { + return backup == FileState::Usable ? Action::PromoteBackup : Action::PromoteTemp; + } + + // No temp survived. The backup is the only recoverable image, even when it + // is a newer layout that this firmware must preserve rather than decode. + if (backup != FileState::Missing) return Action::PromoteBackup; + return Action::None; +} + +} // namespace MQTTPrefsRecovery diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h new file mode 100644 index 0000000000..e3c6a098be --- /dev/null +++ b/src/helpers/MQTTPrefsStorage.h @@ -0,0 +1,320 @@ +#pragma once + +#include +#include + +// /mqtt_prefs is a raw binary persistence format. Keep the layout-only types +// independent from CommonCLI so the migration decoder can be tested on the host +// without pulling in Arduino, filesystem, or radio dependencies. +#ifdef WITH_MQTT_BRIDGE + +// Must match MAX_MQTT_SLOTS in MQTTPresets.h. CommonCLI.h enforces that link on +// firmware builds; keeping this header standalone avoids importing preset data +// into host migration tests. +static const int MQTT_PREFS_SLOT_COUNT = 6; + +// Old MQTT preferences layout (pre-slot firmware) — used only for migration detection. +struct OldMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_server[64]; + uint16_t mqtt_port; + char mqtt_username[32]; + char mqtt_password[64]; + uint8_t mqtt_analyzer_us_enabled; + uint8_t mqtt_analyzer_eu_enabled; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; +}; + +// The pre-WiFi-power pre-slot layout has the same frozen size as +// OldMQTTPrefs, but timezone/server start one byte earlier. A conservative +// classifier distinguishes meaningful configurations before migration. +struct PreWifiPowerOldMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_server[64]; + uint16_t mqtt_port; + char mqtt_username[32]; + char mqtt_password[64]; + uint8_t mqtt_analyzer_us_enabled; + uint8_t mqtt_analyzer_eu_enabled; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; +}; + +// MQTT preferences stored separately from NodePrefs to avoid upstream layout +// conflicts. The full layout is the frozen v1 payload baseline. The prefix +// before observer settings is also an explicitly supported v1 payload: it was +// used before the observer fields were appended. +struct MQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + + char timezone_string[32]; + int8_t timezone_offset; + + char mqtt_slot_preset[MQTT_PREFS_SLOT_COUNT][24]; + char mqtt_slot_host[MQTT_PREFS_SLOT_COUNT][64]; + uint16_t mqtt_slot_port[MQTT_PREFS_SLOT_COUNT]; + char mqtt_slot_username[MQTT_PREFS_SLOT_COUNT][32]; + char mqtt_slot_password[MQTT_PREFS_SLOT_COUNT][64]; + + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + + char mqtt_slot_token[MQTT_PREFS_SLOT_COUNT][48]; + char mqtt_slot_topic[MQTT_PREFS_SLOT_COUNT][96]; + char mqtt_slot_audience[MQTT_PREFS_SLOT_COUNT][64]; + + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; + + uint8_t snmp_enabled; + char snmp_community[24]; + uint8_t radio_watchdog_minutes; + uint8_t alert_enabled; + char alert_psk_hex[33]; + uint16_t alert_wifi_minutes; + uint16_t alert_mqtt_minutes; + uint16_t alert_min_interval_min; + char alert_hashtag[24]; + char alert_region[31]; + + // Neighbors publishing (PSRAM boards only). Appended at the end of the + // observer tail so a shorter (pre-neighbors) /mqtt_prefs payload from earlier + // firmware still loads with these defaulting off/24h; keeps the format at + // VERSION 1. Field order and sizes are kept byte-identical to the flex + // neighbors build so a /mqtt_prefs written by either firmware is + // interchangeable (see the offsetof static_asserts below). + uint8_t mqtt_neighbors_enabled; + uint32_t mqtt_neighbors_interval; + + // Per-slot payload-type allow masks. Bit N controls MeshCore packet type N + // for both packets and raw MQTT topics. Appended so older v1 payloads load + // with the default all-types masks intact. + uint16_t mqtt_slot_packet_filter[MQTT_PREFS_SLOT_COUNT]; +}; + +// Neighbor discovery is scheduled with the wrap-safe millis() helpers, whose +// signed-delta comparison requires intervals below INT32_MAX ms. The 336h +// (two-week) cap stays comfortably inside that range. +static const uint32_t MQTT_NEIGHBORS_MIN_INTERVAL_HOURS = 12; +static const uint32_t MQTT_NEIGHBORS_MAX_INTERVAL_HOURS = 336; +static const uint32_t MQTT_NEIGHBORS_DEFAULT_INTERVAL_HOURS = 24; +static const uint32_t MQTT_NEIGHBORS_MIN_INTERVAL_MS = MQTT_NEIGHBORS_MIN_INTERVAL_HOURS * 3600000UL; +static const uint32_t MQTT_NEIGHBORS_MAX_INTERVAL_MS = MQTT_NEIGHBORS_MAX_INTERVAL_HOURS * 3600000UL; +static const uint32_t MQTT_NEIGHBORS_DEFAULT_INTERVAL_MS = MQTT_NEIGHBORS_DEFAULT_INTERVAL_HOURS * 3600000UL; + +// Version-1 has four payload layouts this firmware can decode. Never infer a +// compatible payload from an arbitrary SHORTER size: raw prefs have no +// checksum, so a short length has to match a boundary that was really shipped. +// +// A LONGER v1 payload is different and is always readable: within a version tag +// the layout is append-only, so a later build's file still starts with this +// binary's exact baseline. classify() reads that prefix and ignores the tail +// rather than rejecting the file — see the downgrade contract there. Any change +// that is not a pure append MUST bump MQTT_PREFS_VERSION instead. +// - PRE_OBSERVER (2736): stops before the observer tail (snmp_*/alert_*). +// - PRE_NEIGHBORS (2860): full observer tail, no neighbors fields yet. +// - PRE_FILTER (2864): neighbors tail, no per-slot packet filters. +// - FULL (2876): current baseline, with six uint16_t filter masks. +// +// FULL is the maximum written, not the default: MQTTPrefsCodec::payloadLenFor() +// keeps emitting PRE_FILTER while every slot holds the all-types default, so a +// node that never touches a filter stays readable by pre-filter firmware. See +// the rollback note there — /mqtt_prefs also carries the WiFi credentials. +static const size_t MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE = 2736; +static const size_t MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE = 2860; +static const size_t MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE = 2864; +static const size_t MQTT_PREFS_V1_FULL_PAYLOAD_SIZE = 2876; + +// /mqtt_prefs starts with a self-describing 8-byte header. Headerless files +// are deployed legacy layouts and continue to be distinguished by size. +static const uint8_t MQTT_PREFS_MAGIC[4] = {0xF5, 'M', 'Q', 'P'}; +static const uint16_t MQTT_PREFS_VERSION = 1; + +struct MQTTPrefsHeader { + uint8_t magic[4]; + uint16_t version; + uint16_t payload_len; +}; + +// 3-slot MQTTPrefs layout. Array dimensions changed in the current format, so +// it must be field-copied rather than read into MQTTPrefs directly. +struct ThreeSlotMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[3][24]; + char mqtt_slot_host[3][64]; + uint16_t mqtt_slot_port[3]; + char mqtt_slot_username[3][32]; + char mqtt_slot_password[3][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; + char mqtt_slot_token[3][48]; + char mqtt_slot_topic[3][96]; +}; + +// The earlier 3-slot format preceded token/topic fields. +struct ThreeSlotBaseMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[3][24]; + char mqtt_slot_host[3][64]; + uint16_t mqtt_slot_port[3]; + char mqtt_slot_username[3][32]; + char mqtt_slot_password[3][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; +}; + +// Headerless 6-slot layout shipped to the deployed flex fleet. It retains the +// removed `_legacy_*` block, so it too is field-copied into MQTTPrefs. +struct Legacy6SlotMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[MQTT_PREFS_SLOT_COUNT][24]; + char mqtt_slot_host[MQTT_PREFS_SLOT_COUNT][64]; + uint16_t mqtt_slot_port[MQTT_PREFS_SLOT_COUNT]; + char mqtt_slot_username[MQTT_PREFS_SLOT_COUNT][32]; + char mqtt_slot_password[MQTT_PREFS_SLOT_COUNT][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; + char mqtt_slot_token[MQTT_PREFS_SLOT_COUNT][48]; + char mqtt_slot_topic[MQTT_PREFS_SLOT_COUNT][96]; + char mqtt_slot_audience[MQTT_PREFS_SLOT_COUNT][64]; + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; +}; + +// Historical headerless 6-slot variants. They share a common prefix but only +// later files contain the appended audience, RX, and NTP fields. +static const size_t LEGACY6_BASE_SIZE = 2452; +static const size_t LEGACY6_AUDIENCE_SIZE = 2836; +static const size_t LEGACY6_AUDIENCE_RX_SIZE = 2840; + +// Frozen on-flash layouts; every firmware and native fixture build checks them. +static_assert(sizeof(MQTTPrefsHeader) == 8, "versioned /mqtt_prefs header must stay 8 bytes"); +static_assert(offsetof(MQTTPrefs, snmp_enabled) == MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE, + "v1 pre-observer /mqtt_prefs boundary changed"); +static_assert(sizeof(MQTTPrefs) == MQTT_PREFS_V1_FULL_PAYLOAD_SIZE, + "v1 /mqtt_prefs payload layout changed"); +// Lock the neighbors tail to the flex neighbors build's layout so a /mqtt_prefs +// written by either firmware is byte-for-byte interchangeable. The enable flag +// lands in the old struct's zeroed trailing padding (offset 2857), and the +// interval begins exactly at the pre-neighbors payload size (2860) so a +// pre-neighbors read stops right before it and the interval keeps its default. +static_assert(offsetof(MQTTPrefs, mqtt_neighbors_enabled) == 2857, + "neighbors enable flag must sit at the flex-compatible offset"); +static_assert(offsetof(MQTTPrefs, mqtt_neighbors_interval) == MQTT_PREFS_V1_PRE_NEIGHBORS_PAYLOAD_SIZE, + "neighbors interval offset must equal the pre-neighbors payload size"); +static_assert(offsetof(MQTTPrefs, mqtt_slot_packet_filter) == MQTT_PREFS_V1_PRE_FILTER_PAYLOAD_SIZE, + "packet filters must begin at the pre-filter payload boundary"); +static_assert(sizeof(OldMQTTPrefs) == 472, "frozen pre-slot /mqtt_prefs layout changed"); +static_assert(sizeof(PreWifiPowerOldMQTTPrefs) == 472, "frozen pre-WiFi-power /mqtt_prefs layout changed"); +static_assert(offsetof(OldMQTTPrefs, wifi_power_save) == 144, + "frozen post-WiFi-power discriminator offset changed"); +static_assert(offsetof(OldMQTTPrefs, timezone_string) == 145, + "frozen post-WiFi-power timezone offset changed"); +static_assert(offsetof(OldMQTTPrefs, timezone_offset) == 177, + "frozen post-WiFi-power UTC offset changed"); +static_assert(offsetof(OldMQTTPrefs, mqtt_server) == 178, + "frozen post-WiFi-power server offset changed"); +static_assert(offsetof(PreWifiPowerOldMQTTPrefs, timezone_string) == 144, + "frozen pre-WiFi-power timezone offset changed"); +static_assert(offsetof(PreWifiPowerOldMQTTPrefs, timezone_offset) == 176, + "frozen pre-WiFi-power UTC offset changed"); +static_assert(offsetof(PreWifiPowerOldMQTTPrefs, mqtt_server) == 177, + "frozen pre-WiFi-power server offset changed"); +static_assert(sizeof(ThreeSlotBaseMQTTPrefs) == 1032, "frozen early 3-slot /mqtt_prefs layout changed"); +static_assert(sizeof(ThreeSlotMQTTPrefs) == 1464, "frozen 3-slot /mqtt_prefs layout changed"); +static_assert(offsetof(ThreeSlotMQTTPrefs, mqtt_slot_token) == 1030, + "frozen 3-slot token offset changed"); +static_assert(offsetof(ThreeSlotMQTTPrefs, mqtt_slot_topic) == 1174, + "frozen 3-slot topic offset changed"); +static_assert(offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_audience) == LEGACY6_BASE_SIZE, + "frozen early 6-slot /mqtt_prefs prefix changed"); +static_assert(offsetof(Legacy6SlotMQTTPrefs, mqtt_rx_enabled) == LEGACY6_AUDIENCE_SIZE, + "frozen audience 6-slot /mqtt_prefs prefix changed"); +static_assert(offsetof(Legacy6SlotMQTTPrefs, mqtt_ntp_server) == 2837, + "frozen RX 6-slot /mqtt_prefs prefix changed"); +static_assert(sizeof(Legacy6SlotMQTTPrefs) == 2904, "frozen deployed-fleet /mqtt_prefs layout changed"); + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h new file mode 100644 index 0000000000..eb3366dc4a --- /dev/null +++ b/src/helpers/MQTTPresets.h @@ -0,0 +1,198 @@ +#pragma once + +#include +#include // strcmp/memcmp used by the inline preset helpers below + +// Maximum number of configurable MQTT connection slots (available to all builds for struct layout). +// Used in NodePrefs/MQTTPrefs for persistent storage — do NOT change without migration. +static const int MAX_MQTT_SLOTS = 6; + +// Runtime slot array size: fewer slots on non-PSRAM boards to save ~1.2KB of heap. +// Non-PSRAM boards are limited to 2 active connections (_max_active_slots), so 3 runtime +// slots (2 active + 1 spare for reconfiguration) is sufficient. +#if defined(BOARD_HAS_PSRAM) +static const int RUNTIME_MQTT_SLOTS = 6; +#else +static const int RUNTIME_MQTT_SLOTS = 3; +#endif + +#ifdef WITH_MQTT_BRIDGE + +enum MQTTAuthType : uint8_t { + MQTT_AUTH_NONE, // No authentication + MQTT_AUTH_USERPASS, // Username/password + MQTT_AUTH_JWT // Ed25519-signed JWT (device identity) +}; + +enum MQTTTopicStyle : uint8_t { + MQTT_TOPIC_MESHCORE, // meshcore/{iata}/{device_id}/{status|packets|raw} + MQTT_TOPIC_MESHRANK, // meshrank/uplink/{token}/{device_id}/{type} (no raw) +}; + +struct MQTTPresetDef { + const char* name; // Preset identifier: "analyzer-us", "analyzer-eu", "meshmapper", "meshrank", "waev", ... + const char* server_url; // Full URL including scheme: "wss://host:port/path" or "mqtts://host:port" + const char* jwt_audience; // JWT audience field (only for MQTT_AUTH_JWT, nullptr otherwise) + const char* ca_cert; // PEM CA certificate (nullptr to skip cert pinning) + MQTTAuthType auth_type; + MQTTTopicStyle topic_style; + unsigned long token_lifetime; // JWT token lifetime in seconds (0 = use default 86400) + bool allow_retain; // Whether the broker allows the MQTT retain flag + uint16_t keepalive; // MQTT keepalive in seconds (0 = library default 120s) + const char* userpass_username; // MQTT_AUTH_USERPASS: embedded username, or nullptr to use mqttN.username + const char* userpass_password; // MQTT_AUTH_USERPASS: embedded password, or nullptr to use mqttN.password +}; + +// Sentinel: resolve MQTT username from device public-key hex at connect time. +// Braces match topic placeholders ({device}/{iata}); never send this string to the broker. +static const char MQTT_USERPASS_USERNAME_PUBKEY[] = "{pubkey}"; + +static inline bool mqttPresetUsesDevicePubkeyUsername(const MQTTPresetDef* preset) { + return preset && preset->auth_type == MQTT_AUTH_USERPASS && + preset->userpass_username && + strcmp(preset->userpass_username, MQTT_USERPASS_USERNAME_PUBKEY) == 0; +} + +// True when USERPASS username must come from mqttN.username (null embedded username). +// "{pubkey}" is an embedded sentinel, so it does not need a slot username. +static inline bool mqttPresetNeedsSlotUsername(const MQTTPresetDef* preset) { + return preset && preset->auth_type == MQTT_AUTH_USERPASS && + !preset->userpass_username; +} + +// True when USERPASS password must come from mqttN.password (null embedded password). +static inline bool mqttPresetNeedsSlotPassword(const MQTTPresetDef* preset) { + return preset && preset->auth_type == MQTT_AUTH_USERPASS && + !preset->userpass_password; +} + +// True when preset uses MQTT_AUTH_USERPASS but at least one credential comes from slot prefs. +static inline bool mqttPresetNeedsSlotCredentials(const MQTTPresetDef* preset) { + return mqttPresetNeedsSlotUsername(preset) || mqttPresetNeedsSlotPassword(preset); +} + +// Google Trust Services - GTS Root R4 (used by LetsMesh Analyzer) +static const char GTS_ROOT_R4[] PROGMEM = + "-----BEGIN CERTIFICATE-----\n" + "MIIDejCCAmKgAwIBAgIQf+UwvzMTQ77dghYQST2KGzANBgkqhkiG9w0BAQsFADBX\n" + "MQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTEQMA4GA1UE\n" + "CxMHUm9vdCBDQTEbMBkGA1UEAxMSR2xvYmFsU2lnbiBSb290IENBMB4XDTIzMTEx\n" + "NTAzNDMyMVoXDTI4MDEyODAwMDA0MlowRzELMAkGA1UEBhMCVVMxIjAgBgNVBAoT\n" + "GUdvb2dsZSBUcnVzdCBTZXJ2aWNlcyBMTEMxFDASBgNVBAMTC0dUUyBSb290IFI0\n" + "MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE83Rzp2iLYK5DuDXFgTB7S0md+8Fhzube\n" + "Rr1r1WEYNa5A3XP3iZEwWus87oV8okB2O6nGuEfYKueSkWpz6bFyOZ8pn6KY019e\n" + "WIZlD6GEZQbR3IvJx3PIjGov5cSr0R2Ko4H/MIH8MA4GA1UdDwEB/wQEAwIBhjAd\n" + "BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB/zAd\n" + "BgNVHQ4EFgQUgEzW63T/STaj1dj8tT7FavCUHYwwHwYDVR0jBBgwFoAUYHtmGkUN\n" + "l8qJUC99BM00qP/8/UswNgYIKwYBBQUHAQEEKjAoMCYGCCsGAQUFBzAChhpodHRw\n" + "Oi8vaS5wa2kuZ29vZy9nc3IxLmNydDAtBgNVHR8EJjAkMCKgIKAehhxodHRwOi8v\n" + "Yy5wa2kuZ29vZy9yL2dzcjEuY3JsMBMGA1UdIAQMMAowCAYGZ4EMAQIBMA0GCSqG\n" + "SIb3DQEBCwUAA4IBAQAYQrsPBtYDh5bjP2OBDwmkoWhIDDkic574y04tfzHpn+cJ\n" + "odI2D4SseesQ6bDrarZ7C30ddLibZatoKiws3UL9xnELz4ct92vID24FfVbiI1hY\n" + "+SW6FoVHkNeWIP0GCbaM4C6uVdF5dTUsMVs/ZbzNnIdCp5Gxmx5ejvEau8otR/Cs\n" + "kGN+hr/W5GvT1tMBjgWKZ1i4//emhA1JG1BbPzoLJQvyEotc03lXjTaCzv8mEbep\n" + "8RqZ7a2CPsgRbuvTPBwcOMBBmuFeU88+FSBX6+7iP0il8b4Z0QFqIwwMHfs/L6K1\n" + "vepuoxtGzi4CZ68zJpiq1UvSqTbFJjtbD4seiMHl\n" + "-----END CERTIFICATE-----\n"; + +// ISRG Root X1 (used by MeshMapper - Let's Encrypt root CA) +static const char ISRG_ROOT_X1[] PROGMEM = + "-----BEGIN CERTIFICATE-----\n" + "MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\n" + "TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\n" + "cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\n" + "WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\n" + "ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\n" + "MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\n" + "h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n" + "0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\n" + "A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\n" + "T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\n" + "B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\n" + "B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\n" + "KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\n" + "OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\n" + "jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\n" + "qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\n" + "rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\n" + "HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\n" + "hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\n" + "ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n" + "3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\n" + "NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\n" + "ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\n" + "TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\n" + "jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\n" + "oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n" + "4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\n" + "mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\n" + "emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n" + "-----END CERTIFICATE-----\n"; + +// Number of built-in presets +static const int MQTT_PRESET_COUNT = 35; + +// Built-in preset definitions (stored in flash) +static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { + // name url audience rootCA auth topic tokenLife retain keepAlive user pass + { "analyzer-us", "wss://mqtt-us-v1.letsmesh.net:443/mqtt", "mqtt-us-v1.letsmesh.net", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "analyzer-eu", "wss://mqtt-eu-v1.letsmesh.net:443/mqtt", "mqtt-eu-v1.letsmesh.net", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "nz-analyzer", "wss://meshcore-mqtt-1.baird.io:443", "meshcore-mqtt-1.baird.io", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshmapper", "wss://mqtt.meshmapper.net:443/mqtt", "mqtt.meshmapper.net", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshrank", "mqtts://meshrank.net:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_NONE, MQTT_TOPIC_MESHRANK, 0, false, 0, nullptr, nullptr }, + // waev token_lifetime is 3300 (55 min) on purpose: the broker's real JWT TTL is + // 60 min, and claiming less keeps fresh tokens accepted even with ~5 min of fast + // device-clock skew (and off any exp-iat<=3600 boundary strictness). Do NOT + // "fix" this to 3600 — the renewal race is handled separately by + // tokenRenewalBufferSecs() in MQTTBridge, which renews another 5 min earlier. + { "waev", "wss://mqtt.waev.app:443/mqtt", "mqtt.waev.app", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 3300, false, 55, nullptr, nullptr }, + { "meshomatic", "wss://us-east.meshomatic.net:443/mqtt", "us-east.meshomatic.net", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "cascadiamesh", "wss://mqtt-v1.cascadiamesh.org:443/mqtt", "mqtt-v1.cascadiamesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "tennmesh", "mqtt://mqtt.tennmesh.com:1883", nullptr, nullptr, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, "mqttfeed", "tc2live" }, + { "nashmesh", "mqtt://mqtt.nashme.sh:1883", nullptr, nullptr, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, "meshdev", "large4cats"}, + { "ctmesh", "mqtt://mqtt.ctmesh.org:1883", nullptr, nullptr, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 60, "meshdev", "large4cats"}, + { "chimesh", "wss://mqtt.chimesh.org:443", "mqtt.chimesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshat.se", "wss://meshcore-mqtt.meshat.se:443", "meshcore-mqtt.meshat.se", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "eastidahomesh", "mqtt://live.eastidahomesh.com:1883", nullptr, nullptr, MQTT_AUTH_NONE, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "coloradomesh", "wss://mqtt.meshcore.coloradomesh.org:443","mqtt.meshcore.coloradomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "dutchmeshcore-1", "wss://collector1.dutchmeshcore.nl:443/mqtt", "collector1.dutchmeshcore.nl", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "dutchmeshcore-2", "wss://collector2.dutchmeshcore.nl:443/mqtt", "collector2.dutchmeshcore.nl", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshcore-ca-1", "wss://mqtt1.meshcore.ca:443/mqtt", "mqtt1.meshcore.ca", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshcore-ca-2", "wss://mqtt2.meshcore.ca:443/mqtt", "mqtt2.meshcore.ca", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshcore-fi", "wss://mc-mqtt.meshcore.fi:443/", "mc-mqtt.meshcore.fi", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "okimesh-1", "wss://mqtt1.okimesh.org:9002/mqtt", "mqtt1.okimesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "okimesh-2", "wss://mqtt2.okimesh.org:9002/mqtt", "mqtt2.okimesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "inwmesh", "mqtts://scope.inwmesh.org:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "bostonmesh", "wss://mqttmc01.bostonme.sh:443/mqtt", "mqttmc01.bostonme.sh", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "rflab", "wss://mqtt.rflab.io:443", "mqtt.rflab.io", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "ipnt.uk", "wss://mqtt.ipnt.uk:443", "mqtt.ipnt.uk", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "flmesh", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "corecomms", "wss://mqtt.corecomms.net:443/mqtt", "mqtt.corecomms.net", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshtexas", "wss://mqtt.meshtexas.org:443/mqtt", "mqtt.meshtexas.org", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + // Username is device pubkey hex at connect; password from mqttN.password. No TLS. + { "mesh-chaun14", "mqtt://mqtt.mesh.chaun14.fr:1884", nullptr, nullptr, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 60, MQTT_USERPASS_USERNAME_PUBKEY, nullptr }, + // LetsMesh-compatible JWT; TLS is Let's Encrypt (ISRG Root X1), not GTS. + { "wcmesh", "wss://mqtt.wcmesh.com:443", "mqtt.wcmesh.com", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "atvirastinklas","wss://mqtt-mc.atvirastinklas.lt:443", "mqtt-mc.atvirastinklas.lt", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + // JWT token auth; LE Gen-Y ECDSA chain (YE2 → Root YE → X2) still anchors at ISRG Root X1. + { "gomesh", "wss://mqtt.gomesh.dev:443", "mqtt.gomesh.dev", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "idahomesh", "wss://mqtt.idahomesh.org:443/mqtt", "mqtt.idahomesh.org", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "ntxmesh", "wss://ntxmesh.dhovin.me:8883", "ntxmesh.dhovin.me", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, +}; + +// Find a preset by name, returns nullptr if not found +static const MQTTPresetDef* findMQTTPreset(const char* name) { + if (!name || name[0] == '\0') return nullptr; + for (int i = 0; i < MQTT_PRESET_COUNT; i++) { + if (strcmp(name, MQTT_PRESETS[i].name) == 0) { + return &MQTT_PRESETS[i]; + } + } + return nullptr; +} + +// Slot preset name constants +static const char MQTT_PRESET_NONE[] = "none"; +static const char MQTT_PRESET_CUSTOM[] = "custom"; + +#endif diff --git a/src/helpers/MQTTReplyFormat.h b/src/helpers/MQTTReplyFormat.h new file mode 100644 index 0000000000..bdb343d70d --- /dev/null +++ b/src/helpers/MQTTReplyFormat.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +// Bounded, clamping printf-append for the fixed-size CLI reply buffers used by +// MQTTBridge's status/stats/diag formatters. Factored out of MQTTBridge so the +// bound is provable on the host instead of holding "by input-size accident" +// (see the A1 out-of-bounds-write finding, 2026-07-19 MQTT observer review). +// +// Appends `fmt...` to `buf` starting at offset `*pos`, then advances `*pos` by +// the number of characters actually written, CLAMPED to [0, bufsize-1]. Because +// snprintf returns the *would-have-written* length, the naive +// `*pos += snprintf(buf + *pos, bufsize - *pos, ...)` idiom can push `*pos` past +// `bufsize` after a truncated append; the next append then computes +// `bufsize - *pos` as a huge size_t and `buf + *pos` past the end, writing out of +// bounds. Clamping `*pos` here makes every subsequent append a safe no-op once +// the buffer is full. +// +// buf is always left NUL-terminated (vsnprintf guarantees this for bufsize > 0). +// No-ops on null buf/pos or bufsize == 0. A negative incoming *pos is treated as +// 0. Typical use: `int pos = 0;` then a sequence of replyAppendf() calls. +static inline void replyAppendf(char* buf, size_t bufsize, int* pos, const char* fmt, ...) { + if (!buf || !pos || bufsize == 0) return; + if (*pos < 0) *pos = 0; + // Full: no room for anything but the terminator. Keep buf NUL-terminated and + // leave *pos pinned at the last writable index. + if ((size_t)*pos >= bufsize - 1) { + *pos = (int)bufsize - 1; + buf[*pos] = '\0'; + return; + } + size_t remaining = bufsize - (size_t)*pos; + va_list args; + va_start(args, fmt); + int n = vsnprintf(buf + *pos, remaining, fmt, args); + va_end(args); + // Encoding error: vsnprintf still NUL-terminated buf + *pos; leave *pos as-is. + if (n < 0) return; + *pos += n; + if ((size_t)*pos >= bufsize) *pos = (int)bufsize - 1; // clamp truncated append +} diff --git a/src/helpers/MQTTRuntimeBufferLifecycle.h b/src/helpers/MQTTRuntimeBufferLifecycle.h new file mode 100644 index 0000000000..bb061a9efe --- /dev/null +++ b/src/helpers/MQTTRuntimeBufferLifecycle.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +// Small ownership helpers for MQTT runtime buffers. They intentionally keep +// each buffer independent: a failed allocation leaves that buffer null (so its +// caller can use its stack fallback) without discarding the other buffers. +namespace MQTTRuntimeBufferLifecycle { + +template +inline void* allocateIfMissing(void* buffer, size_t size, Allocator allocate) { + return buffer != nullptr ? buffer : allocate(size); +} + +template +inline void* release(void* buffer, Deallocator deallocate) { + if (buffer != nullptr) { + deallocate(buffer); + } + return nullptr; +} + +} // namespace MQTTRuntimeBufferLifecycle diff --git a/src/helpers/MQTTTopicRouter.h b/src/helpers/MQTTTopicRouter.h new file mode 100644 index 0000000000..4dd4b1b8f1 --- /dev/null +++ b/src/helpers/MQTTTopicRouter.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include + +#include "MQTTObserverValidation.h" +#include "MQTTTopicTemplate.h" + +// Pure MQTT publication-topic policy shared by MQTTBridge and the native tests. +// Keep these values aligned with MQTTBridge::MQTTMessageType; the bridge passes +// its enum value as an int so this helper stays independent of ESP/Arduino types. +enum MQTTPublicationType { + MQTT_PUBLICATION_STATUS = 0, + MQTT_PUBLICATION_PACKETS = 1, + MQTT_PUBLICATION_RAW = 2, + MQTT_PUBLICATION_NEIGHBORS = 3, +}; + +enum MQTTTopicRouteStyle { + MQTT_ROUTE_MESHCORE, + MQTT_ROUTE_MESHRANK, + MQTT_ROUTE_CUSTOM, +}; + +static inline bool mqttTopicSlotIndexValid(int index, size_t slot_count) { + return index >= 0 && (size_t)index < slot_count; +} + +static inline const char* mqttPublicationTypeName(int type) { + switch (type) { + case MQTT_PUBLICATION_STATUS: return "status"; + case MQTT_PUBLICATION_PACKETS: return "packets"; + case MQTT_PUBLICATION_RAW: return "raw"; + case MQTT_PUBLICATION_NEIGHBORS: return "neighbors"; + default: return NULL; + } +} + +static inline bool mqttWriteTopic(char* buf, size_t buf_size, const char* format, + const char* first, const char* second, + const char* third) { + if (!buf || buf_size == 0 || !format || !first || !second || !third) return false; + buf[0] = '\0'; + int written = snprintf(buf, buf_size, format, first, second, third); + return written > 0 && (size_t)written < buf_size; +} + +// Build the complete topic for one publication. MeshRank takes status, packets, +// and neighbors under meshrank/uplink/{token}/{device}/, using the same type +// suffixes as the MeshCore layout, and requires a per-slot token rather than an +// IATA. MeshCore routes require a configured IATA and device id. Custom +// templates may omit either placeholder, so their individual values are allowed +// to be empty. +static inline bool mqttBuildPublicationTopic(MQTTTopicRouteStyle style, int type, + const char* custom_template, + const char* iata, const char* device, + const char* token, + char* buf, size_t buf_size) { + if (!buf || buf_size == 0) return false; + buf[0] = '\0'; + + const char* type_name = mqttPublicationTypeName(type); + if (!type_name) return false; + + switch (style) { + case MQTT_ROUTE_MESHCORE: + if (!mqttIataValid(iata) || strcmp(iata, "XXX") == 0 || !device || device[0] == '\0') { + return false; + } + return mqttWriteTopic(buf, buf_size, "meshcore/%s/%s/%s", iata, device, type_name); + + case MQTT_ROUTE_MESHRANK: + // Raw is deliberately withheld: highest-volume topic, and the broker does + // not consume it. observer-firmware still sends it — keep this on merge. + if (type == MQTT_PUBLICATION_RAW || !token || token[0] == '\0' || + !device || device[0] == '\0') { + return false; + } + return mqttWriteTopic(buf, buf_size, "meshrank/uplink/%s/%s/%s", + token, device, type_name); + + case MQTT_ROUTE_CUSTOM: + return mqttSubstituteTopic(custom_template, iata, device, token, type_name, + buf, buf_size); + + default: + return false; + } +} + diff --git a/src/helpers/MQTTTopicTemplate.h b/src/helpers/MQTTTopicTemplate.h new file mode 100644 index 0000000000..e0af4d76c1 --- /dev/null +++ b/src/helpers/MQTTTopicTemplate.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +// Expand the {iata} {device} {token} {type} placeholders in a custom MQTT topic +// template. Factored out of MQTTBridge::substituteTopicTemplate so the (bounded) +// string expansion can be unit-tested on the host; the bridge passes its cached +// _iata / _device_id, the slot token, and the message-type string. +// +// Returns false on buffer overflow or an empty result. buf is always +// NUL-terminated. A null value substitutes as empty; an unknown "{...}" token is +// copied through verbatim. +static inline bool mqttSubstituteTopic(const char* tmpl, const char* iata, + const char* device, const char* token, + const char* type_str, char* buf, size_t buf_size) { + if (!buf || buf_size == 0) return false; + if (!iata) iata = ""; + if (!device) device = ""; + if (!token) token = ""; + if (!type_str) type_str = ""; + + size_t out = 0; + const char* p = tmpl ? tmpl : ""; + while (*p && out < buf_size - 1) { + const char* sub = NULL; + size_t adv = 0; + if (strncmp(p, "{iata}", 6) == 0) { + sub = iata; adv = 6; + } else if (strncmp(p, "{device}", 8) == 0) { + sub = device; adv = 8; + } else if (strncmp(p, "{token}", 7) == 0) { + sub = token; adv = 7; + } else if (strncmp(p, "{type}", 6) == 0) { + sub = type_str; adv = 6; + } + if (sub) { + size_t len = strlen(sub); + if (out + len >= buf_size) { + buf[out] = '\0'; // keep buf terminated even on the overflow path + return false; + } + memcpy(buf + out, sub, len); + out += len; + p += adv; + } else { + buf[out++] = *p++; + } + } + buf[out] = '\0'; + // The loop also stops when the output buffer is full. If input remains, + // report overflow just as we do for an oversized placeholder substitution; + // callers must never publish a silently truncated topic. + if (*p) return false; + return out > 0; +} diff --git a/src/helpers/MQTTWireScratch.h b/src/helpers/MQTTWireScratch.h new file mode 100644 index 0000000000..d5c793369f --- /dev/null +++ b/src/helpers/MQTTWireScratch.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +// Sizing and validation for the scratch buffers that hold a serialized packet. +// Pure so the boundary conditions can be tested on the host: the firmware-side +// callers are MQTTMessageBuilder::packetToHex() and MQTTBridge::publishPacket(). +namespace MQTTWireScratch { + +// A serialized packet is header(1) + transport codes(0|4) + path_len(1) + +// path(<= MAX_PATH_SIZE) + payload(<= MAX_PACKET_PAYLOAD). Packet::writeTo() +// returns uint8_t, so MAX_TRANS_UNIT is the hard ceiling. +static const size_t kWireBytes = MAX_TRANS_UNIT; +// Two uppercase hex chars per byte, plus the NUL. +static const size_t kWireHexChars = 2 * MAX_TRANS_UNIT + 1; + +static_assert(1 + 4 + 1 + MAX_PATH_SIZE + MAX_PACKET_PAYLOAD <= MAX_TRANS_UNIT, + "serialized packet no longer fits MAX_TRANS_UNIT — resize the wire scratch buffers"); + +// True when Packet::writeTo() can safely serialize `packet` into `dest_size` bytes. +// +// writeTo() trusts the packet's own length fields and cannot report an overrun (its +// return type is uint8_t), so the source fields must be checked as well as the +// destination: +// - payload_len drives an unchecked memcpy out of a MAX_PACKET_PAYLOAD array, and a +// corrupt value can still leave getRawLength() inside MAX_TRANS_UNIT. +// - path_len is written into a single wire byte, so anything above 255 is silently +// truncated and would disagree with getPathByteLen(). +// - the path encoding must be one writePath() will actually emit. It self-guards +// against overrunning the path array, but by writing nothing and returning 0, which +// is a correctness problem rather than a safety one: getRawLength() still counts the +// path, so an over-long or reserved encoding passes a destination-size check and +// then serializes to a truncated frame that gets published as the packet. The worst +// case is path_len 0xFF with no payload — 254 counted bytes, 2 bytes emitted. +// isValidPathLen() rejects both the reserved 4-byte hash size and any +// count * size above MAX_PATH_SIZE, and is the same predicate Packet::readFrom() +// applies to every received packet, so no decodable packet is turned away. +inline bool canSerialize(const mesh::Packet& packet, size_t dest_size) { + if (packet.payload_len > MAX_PACKET_PAYLOAD) return false; + if (packet.path_len > 0xFF) return false; + if (!mesh::Packet::isValidPathLen((uint8_t)packet.path_len)) return false; + const int raw_len = packet.getRawLength(); + return raw_len > 0 && (size_t)raw_len <= dest_size; +} + +} // namespace MQTTWireScratch diff --git a/src/helpers/NRF52Board.cpp b/src/helpers/NRF52Board.cpp index eb88c89744..bc3854e425 100644 --- a/src/helpers/NRF52Board.cpp +++ b/src/helpers/NRF52Board.cpp @@ -402,7 +402,7 @@ bool NRF52Board::getBootloaderVersion(char* out, size_t max_len) { return false; } -bool NRF52Board::startOTAUpdate(const char *id, char reply[]) { +bool NRF52Board::startOTAUpdate(const char *id, char reply[], bool force_ap) { // Config the peripheral connection with maximum bandwidth // more SRAM required by SoftDevice // Note: All config***() function must be called before begin() diff --git a/src/helpers/NRF52Board.h b/src/helpers/NRF52Board.h index dba15f974e..de716855c4 100644 --- a/src/helpers/NRF52Board.h +++ b/src/helpers/NRF52Board.h @@ -53,7 +53,7 @@ class NRF52Board : public mesh::MainBoard { virtual void shutdownPeripherals(); virtual void powerOff() override; virtual bool getBootloaderVersion(char* version, size_t max_len) override; - virtual bool startOTAUpdate(const char *id, char reply[]) override; + virtual bool startOTAUpdate(const char *id, char reply[], bool force_ap = false) override; virtual void sleep(uint32_t secs) override; bool isExternalPowered() override; diff --git a/src/helpers/RxReservePacketManager.h b/src/helpers/RxReservePacketManager.h new file mode 100644 index 0000000000..96d6682708 --- /dev/null +++ b/src/helpers/RxReservePacketManager.h @@ -0,0 +1,99 @@ +#pragma once + +#include +#include + +// Fork-owned (not upstream-tracked). Observer builds capture every received packet +// to MQTT, but RX processing needs a free pool packet first — Dispatcher::checkRecv() +// discards the received bytes before logRx() when allocNew() fails. Under duty-cycle +// throttling the outbound queue can park the entire pool waiting on TX budget, which +// starves RX allocation and silently caps MQTT capture at the TX rate (each completed +// TX frees exactly one packet for exactly one more RX). Parked retransmissions also +// absorb every budget refill, starving the node's own CLI responses/ACKs and making +// a heavily-throttled node un-administrable over the mesh. +// +// Two policies fix this, both confined to this manager: +// +// 1. Priority-aware shedding. Below the RX reserve, only low-priority outbound +// (priority > 1: multi-hop flood repeats, adverts, trace) is refused; the node's +// own responses/ACKs (pri 0) and login/PATH replies (pri 1) still queue. Below the +// smaller emergency floor everything is shed to keep capture alive. +// 2. Stale-packet expiry. A queued packet still untransmitted STALE_OUTBOUND_MS past +// its scheduled time is dropped at the next dequeue — a repeat delayed that long is +// noise (the flood has long since propagated), and a CLI response that old has +// already timed out at the client. Under normal load the queue drains in +// milliseconds and this never triggers; under throttle it frees the pool and lets +// fresh traffic (including admin responses) compete for the trickle of TX budget. +class RxReservePacketManager : public StaticPoolPacketManager { + int _rx_reserve, _emergency_floor; + int _cap; + // scheduled_for per queued packet, keyed by packet pointer. The pool is a fixed set + // of _cap Packet objects, so _cap slots cover every possible key with no eviction. + struct AgeEntry { mesh::Packet* pkt; uint32_t scheduled_for; }; + AgeEntry* _ages; + + static const uint32_t STALE_OUTBOUND_MS = 30000; + static const uint8_t MAX_PROTECTED_PRI = 1; // pri 0-1 = own responses/ACKs/replies + + void recordAge(mesh::Packet* packet, uint32_t scheduled_for) { + int empty = -1; + for (int i = 0; i < _cap; i++) { + if (_ages[i].pkt == packet) { _ages[i].scheduled_for = scheduled_for; return; } + if (empty < 0 && _ages[i].pkt == NULL) empty = i; + } + if (empty >= 0) { _ages[empty].pkt = packet; _ages[empty].scheduled_for = scheduled_for; } + } + + bool lookupAge(const mesh::Packet* packet, uint32_t* scheduled_for) const { + for (int i = 0; i < _cap; i++) { + if (_ages[i].pkt == packet) { *scheduled_for = _ages[i].scheduled_for; return true; } + } + return false; + } + +public: + RxReservePacketManager(int pool_size, int rx_reserve) + : StaticPoolPacketManager(pool_size), _rx_reserve(rx_reserve), + _emergency_floor(rx_reserve / 2), _cap(pool_size) { + _ages = new AgeEntry[pool_size]; + for (int i = 0; i < pool_size; i++) { _ages[i].pkt = NULL; _ages[i].scheduled_for = 0; } + } + + void queueOutbound(mesh::Packet* packet, uint8_t priority, uint32_t scheduled_for) override { + int free_count = getFreeCount(); + if (free_count < _emergency_floor + || (free_count < _rx_reserve && priority > MAX_PROTECTED_PRI)) { + MESH_DEBUG_PRINTLN("RxReservePacketManager: pool below RX reserve, shedding outbound (pri %d)", (int)priority); + free(packet); + return; + } + recordAge(packet, scheduled_for); + StaticPoolPacketManager::queueOutbound(packet, priority, scheduled_for); + } + + mesh::Packet* getNextOutbound(uint32_t now) override { + // Expire queued packets that have waited too long past their scheduled time. + for (int i = getOutboundTotal() - 1; i >= 0; i--) { + mesh::Packet* pkt = getOutboundByIdx(i); + uint32_t scheduled_for; + if (pkt && lookupAge(pkt, &scheduled_for) + && (int32_t)(now - scheduled_for) > (int32_t)STALE_OUTBOUND_MS) { + MESH_DEBUG_PRINTLN("RxReservePacketManager: dropping stale queued outbound"); + removeOutboundByIdx(i); + free(pkt); + } + } + return StaticPoolPacketManager::getNextOutbound(now); + } +}; + +// The packet manager for an app build: observer builds reserve a quarter of the pool +// for RX so MQTT capture survives duty-cycle throttling; non-observer builds keep the +// upstream pool behavior unchanged. +inline mesh::PacketManager* createObserverPacketManager(int pool_size) { +#ifdef WITH_MQTT_BRIDGE + return new RxReservePacketManager(pool_size, pool_size / 4); +#else + return new StaticPoolPacketManager(pool_size); +#endif +} diff --git a/src/helpers/SNMPAgent.cpp b/src/helpers/SNMPAgent.cpp new file mode 100644 index 0000000000..a70716f5bb --- /dev/null +++ b/src/helpers/SNMPAgent.cpp @@ -0,0 +1,123 @@ +#ifdef WITH_SNMP + +#include "SNMPAgent.h" +#include + +#define SNMP_PORT 161 + +MeshSNMPAgent::MeshSNMPAgent() + : _snmp("public"), + _running(false), + _uptime_secs(0), + _packets_recv(0), _packets_sent(0), _recv_errors(0), + _noise_floor(0), _last_rssi(0), _last_snr(0), + _sent_flood(0), _sent_direct(0), _recv_flood(0), _recv_direct(0), + _total_air_time_secs(0), + _mqtt_connected_slots(0), _mqtt_queue_depth(0), _mqtt_skipped_publishes(0), + _free_heap(0), _max_alloc(0), _internal_free(0), _psram_free(0), + _wifi_rssi(0) +{ + _firmware_version[0] = '\0'; + _node_name[0] = '\0'; +} + +void MeshSNMPAgent::begin(const char* community) { + if (_running) return; + + _snmp = SNMPAgent(community); + _snmp.setUDP(&_udp); + _snmp.begin(); + + // System group (.1.x.0) + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".1.1.0", &_uptime_secs); + _snmp.addReadOnlyStaticStringHandler(MESHCORE_OID_BASE ".1.2.0", _firmware_version, sizeof(_firmware_version)); + _snmp.addReadOnlyStaticStringHandler(MESHCORE_OID_BASE ".1.3.0", _node_name, sizeof(_node_name)); + + // Radio group (.2.x.0) + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.1.0", &_packets_recv); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.2.0", &_packets_sent); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.3.0", &_recv_errors); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.4.0", &_noise_floor); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.5.0", &_last_rssi); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.6.0", &_last_snr); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.7.0", &_sent_flood); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.8.0", &_sent_direct); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.9.0", &_recv_flood); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.10.0", &_recv_direct); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".2.11.0", &_total_air_time_secs); + + // MQTT group (.3.x.0) + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".3.1.0", &_mqtt_connected_slots); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".3.2.0", &_mqtt_queue_depth); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".3.3.0", &_mqtt_skipped_publishes); + + // Memory group (.4.x.0) + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".4.1.0", &_free_heap); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".4.2.0", &_max_alloc); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".4.3.0", &_internal_free); + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".4.4.0", &_psram_free); + + // Network group (.5.x.0) + _snmp.addIntegerHandler(MESHCORE_OID_BASE ".5.1.0", &_wifi_rssi); + + _snmp.sortHandlers(); + _running = true; +} + +void MeshSNMPAgent::loop() { + if (!_running) return; + + // Update memory and network stats locally (we're on Core 0 with WiFi) + _free_heap = (int)ESP.getFreeHeap(); + _max_alloc = (int)ESP.getMaxAllocHeap(); + _internal_free = (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL); +#ifdef BOARD_HAS_PSRAM + _psram_free = (int)heap_caps_get_free_size(MALLOC_CAP_SPIRAM); +#else + _psram_free = 0; +#endif + + if (WiFi.isConnected()) { + _wifi_rssi = (int)WiFi.RSSI(); + } + + _snmp.loop(); +} + +void MeshSNMPAgent::updateRadioStats( + uint32_t packets_recv, uint32_t packets_sent, uint32_t recv_errors, + int16_t noise_floor, int16_t last_rssi, int16_t last_snr, + uint32_t sent_flood, uint32_t sent_direct, + uint32_t recv_flood, uint32_t recv_direct, + uint32_t total_air_time_secs, uint32_t uptime_secs) { + _packets_recv = (int)packets_recv; + _packets_sent = (int)packets_sent; + _recv_errors = (int)recv_errors; + _noise_floor = (int)noise_floor; + _last_rssi = (int)last_rssi; + _last_snr = (int)last_snr; + _sent_flood = (int)sent_flood; + _sent_direct = (int)sent_direct; + _recv_flood = (int)recv_flood; + _recv_direct = (int)recv_direct; + _total_air_time_secs = (int)total_air_time_secs; + _uptime_secs = (int)uptime_secs; +} + +void MeshSNMPAgent::updateMQTTStats(int connected_slots, int queue_depth, int skipped_publishes) { + _mqtt_connected_slots = connected_slots; + _mqtt_queue_depth = queue_depth; + _mqtt_skipped_publishes = skipped_publishes; +} + +void MeshSNMPAgent::setNodeName(const char* name) { + strncpy(_node_name, name, sizeof(_node_name) - 1); + _node_name[sizeof(_node_name) - 1] = '\0'; +} + +void MeshSNMPAgent::setFirmwareVersion(const char* version) { + strncpy(_firmware_version, version, sizeof(_firmware_version) - 1); + _firmware_version[sizeof(_firmware_version) - 1] = '\0'; +} + +#endif // WITH_SNMP diff --git a/src/helpers/SNMPAgent.h b/src/helpers/SNMPAgent.h new file mode 100644 index 0000000000..c1bfe0f5be --- /dev/null +++ b/src/helpers/SNMPAgent.h @@ -0,0 +1,79 @@ +#pragma once + +#ifdef WITH_SNMP + +#include +#include +#include + +// Temporary private enterprise OID base — replace with registered PEN when available. +// All MeshCore OIDs live under this subtree. +#define MESHCORE_OID_BASE ".1.3.6.1.4.1.99999" + +// OID layout: +// .1.x.0 = system (uptime, version, node name) +// .2.x.0 = radio (packets, RSSI, SNR, noise floor, air time) +// .3.x.0 = mqtt (connected slots, queue depth, skipped publishes) +// .4.x.0 = memory (free heap, max alloc, internal free, PSRAM free) +// .5.x.0 = network (WiFi RSSI) + +class MeshSNMPAgent { +public: + MeshSNMPAgent(); + void begin(const char* community); + void loop(); + + // Called from the mesh task (Core 1) to push fresh stats into SNMP-visible variables. + // Copies are atomic for 32-bit aligned ints on ESP32, so no mutex needed. + void updateRadioStats(uint32_t packets_recv, uint32_t packets_sent, uint32_t recv_errors, + int16_t noise_floor, int16_t last_rssi, int16_t last_snr, + uint32_t sent_flood, uint32_t sent_direct, + uint32_t recv_flood, uint32_t recv_direct, + uint32_t total_air_time_secs, uint32_t uptime_secs); + + void updateMQTTStats(int connected_slots, int queue_depth, int skipped_publishes); + + void setNodeName(const char* name); + void setFirmwareVersion(const char* version); + + bool isRunning() const { return _running; } + +private: + WiFiUDP _udp; + SNMPAgent _snmp; + bool _running; + + // System OIDs + int _uptime_secs; + char _firmware_version[32]; + char _node_name[32]; + + // Radio OIDs + int _packets_recv; + int _packets_sent; + int _recv_errors; + int _noise_floor; + int _last_rssi; + int _last_snr; + int _sent_flood; + int _sent_direct; + int _recv_flood; + int _recv_direct; + int _total_air_time_secs; + + // MQTT OIDs + int _mqtt_connected_slots; + int _mqtt_queue_depth; + int _mqtt_skipped_publishes; + + // Memory OIDs (updated in loop() since we're on Core 0 with WiFi) + int _free_heap; + int _max_alloc; + int _internal_free; + int _psram_free; + + // Network OIDs + int _wifi_rssi; +}; + +#endif // WITH_SNMP diff --git a/src/helpers/SimpleMeshTables.h b/src/helpers/SimpleMeshTables.h index 956f36faa6..f0e6ca12a0 100644 --- a/src/helpers/SimpleMeshTables.h +++ b/src/helpers/SimpleMeshTables.h @@ -4,6 +4,16 @@ #ifdef ESP32 #include + // TFT_eSPI (pulled in by the tracker variants' display driver) defines + // FS_NO_GLOBALS, which suppresses FS.h's own `using fs::File`. Without this, + // File never reaches global scope and every TU that reaches FS.h through + // TFT_eSPI first fails with "'File' has not been declared" — here and in + // simple_repeater/MyMesh.h. Restore exactly what FS.h would have done. + // Cannot use fs::File explicitly instead: File is also the global type on the + // nRF52/RP2040 paths, which have no fs namespace. + #if defined(FS_NO_GLOBALS) + using fs::File; + #endif #endif #define MAX_PACKET_HASHES (128+32) diff --git a/src/helpers/StatsFormatHelper.h b/src/helpers/StatsFormatHelper.h index bf619133e9..93dc808239 100644 --- a/src/helpers/StatsFormatHelper.h +++ b/src/helpers/StatsFormatHelper.h @@ -34,6 +34,24 @@ class StatsFormatHelper { ); } + template + static void formatRadioDiag(char* reply, + mesh::Radio* radio, + RadioDriverType& driver, + mesh::MillisecondClock& ms, + uint16_t err_flags, + bool has_outbound) { + uint8_t st = radio->getRadioState(); + const char* state_name = (st & 16) ? "INT_READY" : (st == 0 ? "IDLE" : (st == 1 ? "RX" : (st == 3 ? "TX_WAIT" : "?"))); + unsigned long last_rx = radio->getLastRecvMillis(); + unsigned long ago_secs = (last_rx > 0) ? (ms.getMillis() - last_rx) / 1000 : 0; + sprintf(reply, + "state=%d(%s), recv=%u, sent=%u, errors=%u, err_flags=%u, outbound=%s, last_rx=%lus ago", + st, state_name, + driver.getPacketsRecv(), driver.getPacketsSent(), driver.getPacketsRecvErrors(), + err_flags, has_outbound ? "yes" : "no", ago_secs); + } + template static void formatPacketStats(char* reply, RadioDriverType& driver, diff --git a/src/helpers/TxtDataHelpers.cpp b/src/helpers/TxtDataHelpers.cpp index d327931fde..f994c9decc 100644 --- a/src/helpers/TxtDataHelpers.cpp +++ b/src/helpers/TxtDataHelpers.cpp @@ -1,4 +1,18 @@ #include "TxtDataHelpers.h" +#include + +void StrHelper::stripSurroundingQuotes(char* str, size_t buf_sz) { + if (!str || buf_sz == 0) return; + size_t len = strlen(str); + if (len == 0) return; + if (str[0] == '"' || str[0] == '\'') { + memmove(str, str + 1, len); + len--; + } + if (len > 0 && (str[len - 1] == '"' || str[len - 1] == '\'')) { + str[len - 1] = '\0'; + } +} void StrHelper::strncpy(char* dest, const char* src, size_t buf_sz) { while (buf_sz > 1 && *src) { diff --git a/src/helpers/TxtDataHelpers.h b/src/helpers/TxtDataHelpers.h index ece494f291..d9e82b23a1 100644 --- a/src/helpers/TxtDataHelpers.h +++ b/src/helpers/TxtDataHelpers.h @@ -13,6 +13,8 @@ class StrHelper { public: static void strncpy(char* dest, const char* src, size_t buf_sz); static void strzcpy(char* dest, const char* src, size_t buf_sz); // pads with trailing nulls + /** Remove one leading and one trailing ASCII " or ' if present (in-place). No-op if empty. */ + static void stripSurroundingQuotes(char* str, size_t buf_sz); static const char* ftoa(float f); static const char* ftoa3(float f); //Converts float to string with 3 decimal places static bool isBlank(const char* str); diff --git a/src/helpers/WebConfigBatch.h b/src/helpers/WebConfigBatch.h new file mode 100644 index 0000000000..5532bbad2d --- /dev/null +++ b/src/helpers/WebConfigBatch.h @@ -0,0 +1,310 @@ +#pragma once + +#include // size_t / NULL for the reply classifiers below +#include + +// Fork-owned, dependency-free spec for the WebConfig "config batch / reboot / +// stop" decision + timing core, plus its host tests (test/test_webconfig_batch/). +// +// This is the Phase 6 counterpart of MQTTLifecycle.h: a PURE state machine that +// captures exactly what src/helpers/esp32/WebConfigServer.cpp decides today, so +// the POST-accept / drain / result-read / reboot / stop transitions can be +// exercised deterministically without AsyncWebServer, ArduinoJson, WiFi, or the +// FreeRTOS mutex/refcount. +// +// WIRED: WebConfigServer.cpp calls these functions directly, so they are +// load-bearing and the host tests cover production behavior rather than a +// parallel copy of it. MAX_BATCH and STOP_WARN_MS in WebConfigServer.h alias +// kMaxBatch/kStopWarnMs here, so the constants cannot drift either. +// +// Two deliberate asymmetries remain between this spec and its caller: +// +// 1. finishRebootAt() returns 0 for "no reboot scheduled", but the caller must +// only ASSIGN _reboot_at when the result is non-zero. _reboot_at is not +// solely batch-owned — the manual /api/reboot route arms it from the +// async_tcp task, possibly while a batch is still draining — so an +// unconditional assign would silently cancel a manual reboot. +// 2. classifyPost() is consulted in two phases by handleConfigPost, because the +// change count is only known after the `set` map is parsed, and parsing must +// not precede the Replay/Busy answer (a replayed POST carrying a bad key +// must still receive its 202). +// +// The file:line references below point at the behavior each function mirrors. +// +// Behavior source (all line refs against WebConfigServer.{h,cpp} at the time of +// writing): constants at .h:90-96,155-161; POST accept at .cpp:610-719; drain at +// .cpp:289-334; result read at .cpp:721-791; reboot fire at .cpp:262-265; +// isRebootPending at .cpp:70-74; stop gating at .cpp:185-255. +namespace WebConfigBatch { + +// Constants, verbatim from WebConfigServer. +static const int kMaxBatch = 24; // .h:90 MAX_BATCH +static const uint32_t kDrainPacingMs = 25; // .cpp:296 inter-command gap +static const uint32_t kRebootFallbackMs = 30000; // .cpp:331 drain-finish fallback +static const uint32_t kRebootConfirmMs = 3000; // .cpp:784 first result-read arm +static const uint32_t kStopWarnMs = 10000; // .h:95 STOP_WARN_MS + +// The batch lifecycle. A fresh POST moves Idle->Pending; the drainer moves +// Pending->Done; Done stays re-readable until the next POST claims the slot; +// finalizeTeardown() resets to Idle. +enum class State : uint8_t { + Idle = 0, + Pending, + Done, +}; + +// millis() idioms. elapsedMs uses unsigned wraparound (correct across one 32-bit +// rollover). scheduleAt mirrors the production wrap-guard: every _reboot_at / +// _stop_warn_at assignment does `if (t == 0) t = 1;` so 0 keeps meaning +// "unscheduled" even when the deadline lands exactly on the rollover boundary. +static inline uint32_t elapsedMs(uint32_t now, uint32_t then) { return now - then; } +static inline uint32_t scheduleAt(uint32_t now, uint32_t delay) { + const uint32_t t = now + delay; + return t == 0 ? 1u : t; +} +// Signed wrap-safe "deadline reached", matching the production +// `(int32_t)(now - deadline) >= 0` comparisons. +static inline bool deadlineReached(uint32_t now, uint32_t deadline) { + return (int32_t)(now - deadline) >= 0; +} + +// -------------------------------------------------------------------------- +// POST accept classification (.cpp:637-718). Precedence, verbatim from the +// source: an in-flight/finished batch with the SAME reqid is an idempotent +// replay (commands are NOT re-applied); a DIFFERENT reqid while a batch is still +// PENDING is rejected as busy; otherwise a batch with no changes and no reboot +// is a no-op, and anything else is accepted. Note the asymmetry: a different +// reqid while DONE is NOT busy — the new batch overwrites the DONE slot. +// +// Assumes the request already passed reqid grammar (WebConfigKeys::wcIsValidReqId) +// and per-key allowlist/secret validation, which are covered by test_webconfig_keys. +// -------------------------------------------------------------------------- +enum class PostOutcome : uint8_t { + Replay, // 202; reqid matches the current batch, commands not re-applied + Busy, // 409; a different batch is still PENDING + Accept, // 202; a new batch is accepted (from Idle, or overwriting a Done slot) + NoChanges, // 400; nothing to do (no changes and no reboot requested) +}; + +static inline PostOutcome classifyPost(State state, bool reqid_matches_current, + int change_count, bool reboot_after) { + if (state != State::Idle && reqid_matches_current) return PostOutcome::Replay; + if (state == State::Pending) return PostOutcome::Busy; // reqid differs (match handled above) + if (change_count <= 0 && !reboot_after) return PostOutcome::NoChanges; + return PostOutcome::Accept; +} + +// The state string a Replay body reports mirrors the batch state (.cpp:640): +// "done" when the matched batch already finished, else "pending". +static inline const char* replayStateName(State state) { + return state == State::Done ? "done" : "pending"; +} + +// -------------------------------------------------------------------------- +// Drain (.cpp:289-333). One command per tick. +// -------------------------------------------------------------------------- + +// The drainer waits only BETWEEN commands: never before the first (batch_next +// == 0 runs immediately and fires onConfigBatchStart), never after the last, and +// otherwise until the 25 ms pacing gap elapses. The pacing compare is SIGNED to +// mirror the source verbatim (.cpp:296 `(int32_t)(now - _batch_last_cmd) < 25`), +// matching deadlineReached()'s signedness; for all reachable inputs (elapsed +// 0..25 ms) it is identical to the unsigned form. +static inline bool drainMustWait(int batch_next, int batch_count, + uint32_t now, uint32_t last_cmd_ms) { + return batch_next > 0 && batch_next < batch_count && + (int32_t)(now - last_cmd_ms) < (int32_t)kDrainPacingMs; +} + +// all_ok is a sticky AND across command replies; a reply counts as ok iff it +// begins with "OK" (.cpp:314). Once false it stays false. +static inline bool nextAllOk(bool prev_all_ok, bool reply_is_ok) { + return prev_all_ok && reply_is_ok; +} + +// The batch is finished once the post-increment drain index reaches the count +// (.cpp:319-321). +static inline bool drainFinished(int batch_next_after_increment, int batch_count) { + return batch_next_after_increment >= batch_count; +} + +// On finish, a reboot-requested + all-ok batch arms the 30 s fallback deadline; +// a partially-failed batch (all_ok == false) never reboots (.cpp:324-333). +// Returns the reboot_at deadline, or 0 for "no reboot scheduled". +static inline uint32_t finishRebootAt(bool batch_reboot, bool batch_all_ok, uint32_t now) { + return (batch_reboot && batch_all_ok) ? scheduleAt(now, kRebootFallbackMs) : 0u; +} + +// -------------------------------------------------------------------------- +// Result read (.cpp:738-786). +// -------------------------------------------------------------------------- +enum class ResultOutcome : uint8_t { + Idle, // 200 "idle" — no batch; any valid reqid is echoed + Unknown, // 404 — a batch exists but the reqid does not match it + Pending, // 200 "pending" + Done, // 200 "done" (+ per-command results) +}; + +static inline ResultOutcome classifyResult(State state, bool reqid_matches_current) { + if (state == State::Idle) return ResultOutcome::Idle; // no reqid check while idle + if (!reqid_matches_current) return ResultOutcome::Unknown; + return state == State::Pending ? ResultOutcome::Pending : ResultOutcome::Done; +} + +// The "reboot" flag reported in a Done body (.cpp:767): only a fully-OK, +// reboot-requested batch advertises a pending reboot. +static inline bool doneReportsReboot(bool batch_reboot, bool batch_all_ok) { + return batch_reboot && batch_all_ok; +} + +// The first Done read arms the confirmed (3 s) reboot exactly once (.cpp:777-786): +// the !already_armed guard makes later reads idempotent, so polling cannot push +// the deadline out. When this returns true the caller sets armed = true and +// reboot_at = confirmRebootAt(now). +static inline bool shouldArmConfirmReboot(State state, bool batch_reboot, + bool batch_all_ok, bool already_armed) { + return state == State::Done && batch_reboot && batch_all_ok && !already_armed; +} +static inline uint32_t confirmRebootAt(uint32_t now) { + return scheduleAt(now, kRebootConfirmMs); +} + +// -------------------------------------------------------------------------- +// CLI sequences (/api/cli). The terminal shares this one deferred-command slot +// with config saves rather than owning a second MAX_BATCH array: both drain on +// the loop task, both are single-slot, and a duplicate would cost ~8 KB of +// permanently resident RAM. Sharing also makes a save and a CLI run mutually +// exclusive, which they must be. +// +// Two things differ from a config save: +// 1. results stream. A save's results appear only when the whole batch is +// Done; a CLI read hands back whatever has executed so far, so a pasted +// sequence fills the terminal command by command. +// 2. the reboot is not requested by a `reboot` flag on the request but by the +// word `reboot` appearing in the sequence. It is deferred rather than +// executed, because Board::reboot() does not return and would take the node +// down before the client could read a single result. +// -------------------------------------------------------------------------- + +// Results returned by one read. Bounds the JSON document built on the +// async_tcp task; a longer sequence pages across successive reads. +static const int kCliResultPage = 8; + +static inline int cliPageCount(int from, int produced, int page) { + const int pending = produced - from; + if (pending <= 0) return 0; + return pending > page ? page : pending; +} + +// "done" means the client has been handed every result, not merely that +// execution finished: the last page may still be unread, and a client that +// stops polling at "done" would lose it. +static inline bool cliReadIsFinal(State state, int from, int page_count, int total) { + return state == State::Done && from + page_count >= total; +} + +// A trailing `reboot` is withheld when any command in the sequence failed, +// exactly as a config save's is. The operator asked for the reboot, but +// rebooting into a half-applied config — over a link they may not get back — +// is the worse failure, and the result body reports the refusal. +static inline bool cliRebootAllowed(bool has_reboot, bool all_ok) { + return has_reboot && all_ok; +} + +// CommonCLI has no single failure convention. Testing only for an "Err" prefix +// let five other shapes through as success — including "Unknown command" and +// "unknown config: x", the two an operator hits most — which coloured them +// green AND let a queued reboot proceed after them. +// +// Every shape below is a literal from CommonCLI.cpp / CommonCLI_Observer.cpp. +// This list is the fragile part of the design: a new failure string added there +// is silently a success here. Which is exactly why it must not be what decides +// whether to reboot — see cliReplyConfirmsWrite. +static inline bool cliReplyIsFailure(const char* r) { + if (r == NULL || r[0] == 0) return false; // empty is normalised to "OK" + static const char* const kPrefixes[] = { + "Err", "ERR", "err", // "Err - ", "ERR: ", "Error: " + "(ERR", // "(ERR: clock cannot go backwards)" + "Unknown command", + "unknown config", + "??", // "??: " from the get fallthrough + "Can't find", // "Can't find GPS" + }; + for (size_t i = 0; i < sizeof(kPrefixes) / sizeof(kPrefixes[0]); i++) { + const char* p = kPrefixes[i]; + size_t n = 0; + while (p[n]) n++; + bool match = true; + for (size_t j = 0; j < n; j++) { + if (r[j] != p[j]) { match = false; break; } + } + if (match) return true; + } + // "File system erase: Err" reports the failure at the END of the reply. + for (size_t i = 0; r[i]; i++) { + if (r[i] == ':' && r[i + 1] == ' ' && r[i + 2] == 'E' && r[i + 3] == 'r' && + r[i + 4] == 'r') return true; + } + return false; +} + +// Whether a command's reply is allowed to gate the deferred reboot. +// +// Only writes are, and only writes have a reply convention worth trusting: +// every setter answers with an "OK" prefix. Diagnostics do not — `memory` +// answers "Free: ...", a getter answers "> value" — so letting them gate would +// mean guessing, and guessing wrong here either strands the operator (a +// harmless `memory` blocks their reboot) or reboots into a config that did not +// apply. The question the gate exists to answer is narrower than "did anything +// fail": it is "did every setting I asked for actually take". +static inline bool cliReplyGatesReboot(const char* cmd) { + if (cmd == NULL) return false; + const char* set = "set "; + const char* pwd = "password "; + bool is_set = true, is_pwd = true; + for (int i = 0; i < 4; i++) if (cmd[i] != set[i]) { is_set = false; break; } + for (int i = 0; i < 9; i++) if (cmd[i] != pwd[i]) { is_pwd = false; break; } + return is_set || is_pwd; +} + +// A write took effect iff its reply starts with "OK" — the one convention every +// setter in CommonCLI actually keeps. +static inline bool cliWriteSucceeded(const char* reply) { + return reply != NULL && reply[0] == 'O' && reply[1] == 'K'; +} + +// -------------------------------------------------------------------------- +// Reboot fire (.cpp:262-265) and isRebootPending (.cpp:70-74). +// -------------------------------------------------------------------------- +static inline bool rebootDue(uint32_t reboot_at, uint32_t now) { + return reboot_at != 0 && deadlineReached(now, reboot_at); +} + +// isRebootPending() reports true only for a config-save reboot in the Done +// state, so the manual /api/reboot path (batch_reboot == false) is deliberately +// NOT reported as pending even though _reboot_at is set. +static inline bool isConfigRebootPending(uint32_t reboot_at, bool batch_reboot, State state) { + return reboot_at != 0 && batch_reboot && state == State::Done; +} + +// -------------------------------------------------------------------------- +// Stop gating (.cpp:244-255). Teardown waits indefinitely for in-flight async +// handlers to drain (refs == 0); the STOP_WARN_MS timer only triggers a one-time +// diagnostic — it never forces teardown. +// -------------------------------------------------------------------------- +enum class StopAction : uint8_t { + Finalize, // refs == 0: finalizeTeardown() may run now + Warn, // refs > 0 and the warn deadline passed, not yet warned: log once + Wait, // refs > 0: keep the session alive and wait +}; + +static inline StopAction stopStep(uint32_t handler_refs, bool already_warned, + uint32_t stop_warn_at, uint32_t now) { + if (handler_refs == 0) return StopAction::Finalize; + if (!already_warned && stop_warn_at != 0 && deadlineReached(now, stop_warn_at)) { + return StopAction::Warn; + } + return StopAction::Wait; +} + +} // namespace WebConfigBatch diff --git a/src/helpers/WebConfigKeys.h b/src/helpers/WebConfigKeys.h new file mode 100644 index 0000000000..47fd414949 --- /dev/null +++ b/src/helpers/WebConfigKeys.h @@ -0,0 +1,120 @@ +#pragma once + +#include +#include "MQTTPresets.h" // MAX_MQTT_SLOTS + +// Classification of the config keys the web portal is allowed to drive through +// the CLI `set` handlers. Factored out of WebConfigServer.cpp so the allowlist +// and the (attacker-facing) key parsing can be unit-tested on the host without +// pulling in the whole ESP32 web server (see test/test_webconfig_keys). +// +// Everything here is pure string logic. The functions are `static inline` so +// each translation unit that includes this gets its own copy (there are only +// two: WebConfigServer.cpp and the test), avoiding any ODR concern. + +// Keys mapping to CLI `set ` handlers. Everything not listed here +// is rejected, so a crafted request can't reach arbitrary commands (`erase`, +// etc.) through the batch. The portal's admin-password field is classified +// separately, see wcIsAdminPasswordKey below. +static const char* const WC_ALLOWED_SET_KEYS[] = { + // NodePrefs (radio / node) + "name", "lat", "lon", "radio", "tx", "af", "rxdelay", "txdelay", + "cad", "radio.rxgain", "repeat", "advert.interval", "flood.advert.interval", + "flood.max", "flood.max.advert", "flood.max.unscoped", "loop.detect", + // MQTTPrefs (WiFi / MQTT / misc observer) + "wifi.ssid", "wifi.pwd", "wifi.powersave", + "mqtt.origin", "mqtt.iata", "mqtt.status", "mqtt.packets", "mqtt.raw", + "mqtt.tx", "mqtt.rx", "mqtt.interval", "mqtt.neighbors", "mqtt.neighbors.interval", + "mqtt.ntp", "mqtt.owner", "mqtt.email", + "timezone", "timezone.offset", "snmp", "snmp.community", +}; +static const char* const WC_ALLOWED_SLOT_KEYS[] = { + "preset", "server", "port", "username", "password", "token", "topic", "audience", + "filter", +}; + +// True when `key` is a well-formed per-slot key ("mqttN." with N in +// 1..MAX_MQTT_SLOTS). The shortest such key is "mqttN.x" (7 chars), and this +// probes key[4..6], so the length guard must come first — an attacker-supplied +// "mqtt" or "m" would otherwise read past the terminator. +static inline bool wcIsSlotKeyPrefix(const char* key) { + return strlen(key) >= 7 && memcmp(key, "mqtt", 4) == 0 + && key[4] >= '1' && key[4] <= ('0' + MAX_MQTT_SLOTS) && key[5] == '.'; +} + +static inline bool wcIsAllowedSetKey(const char* key) { + for (size_t i = 0; i < sizeof(WC_ALLOWED_SET_KEYS) / sizeof(WC_ALLOWED_SET_KEYS[0]); i++) { + if (strcmp(key, WC_ALLOWED_SET_KEYS[i]) == 0) return true; + } + // mqtt<1-6>. + if (wcIsSlotKeyPrefix(key)) { + for (size_t i = 0; i < sizeof(WC_ALLOWED_SLOT_KEYS) / sizeof(WC_ALLOWED_SLOT_KEYS[0]); i++) { + if (strcmp(&key[6], WC_ALLOWED_SLOT_KEYS[i]) == 0) return true; + } + } + return false; +} + +// The admin password maps to the top-level `password` command, not a setter, so +// it is classified apart from the `set` allowlist. It is the only key that gets +// this treatment, which is what keeps the allowlist the sole route to `set` and +// leaves no general path from a batch to arbitrary top-level CLI commands. +static inline bool wcIsAdminPasswordKey(const char* key) { + return strcmp(key, "password") == 0; +} + +static inline bool wcIsValidAdminPassword(const char* value) { + if (value == NULL) return false; + const size_t len = strlen(value); + if (len == 0 || len > 15) return false; // NodePrefs::password[16], including NUL + for (size_t i = 0; i < len; i++) { + if (value[i] == '\r' || value[i] == '\n') return false; // reject, never silently strip + } + return true; +} + +// Keys carrying a secret whose stored value is masked with the placeholder in +// the UI; a POST echoing the placeholder for one of these is dropped (unchanged). +static inline bool wcIsSecretKey(const char* key) { + if (strcmp(key, "wifi.pwd") == 0) return true; + if (wcIsSlotKeyPrefix(key) + && (strcmp(&key[6], "password") == 0 || strcmp(&key[6], "token") == 0)) return true; + return false; +} + +// CommonCLI answers a secret getter in plaintext only for the serial console +// (sender_timestamp 0) and masks it for remote callers. The web CLI executes +// with sender_timestamp 0 — that is what makes `erase`, `stats-*` and `set freq` +// reachable — so it would otherwise inherit the serial console's plaintext +// answers for an HTTP request. This says which `get` commands must be masked +// back down, restoring the distinction for a caller not at the serial port. +// +// Writing these has always been possible from the portal; reading them never +// was, because handleConfigGet masks them (wcIsSecretKey). The two are different +// capabilities: replacing a WiFi password does not reveal the current one, and +// replacing an identity does not reveal the existing private key. +static inline bool wcIsSecretReadCommand(const char* cmd) { + if (strncmp(cmd, "get ", 4) != 0) return false; + const char* key = cmd + 4; + while (*key == ' ') key++; + if (strcmp(key, "prv.key") == 0) return true; // this node's identity + if (strcmp(key, "guest.password") == 0) return true; + if (strcmp(key, "alert.psk") == 0) return true; + if (strcmp(key, "bridge.secret") == 0) return true; + return wcIsSecretKey(key); // wifi.pwd, mqttN.password, mqttN.token +} + +// Browser-generated request IDs are exactly eight random bytes encoded as +// hexadecimal. Keeping the grammar deliberately small makes the ID safe to +// echo in JSON/logs and prevents an empty or truncated ID from weakening the +// save/result correlation contract. +static inline bool wcIsValidReqId(const char* reqid) { + if (reqid == NULL || strlen(reqid) != 16) return false; + for (size_t i = 0; i < 16; i++) { + char c = reqid[i]; + if (!((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'))) return false; + } + return true; +} diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp new file mode 100644 index 0000000000..735999e97e --- /dev/null +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -0,0 +1,4389 @@ +#include "MQTTBridge.h" +#include "../MQTTConnectionPolicy.h" +#include "../MQTTMessageBuilder.h" +#include "../MQTTPacketQueuePolicy.h" +#include "../MQTTReplyFormat.h" +#include "../MQTTRuntimeBufferLifecycle.h" +#include "../MQTTTopicRouter.h" +#include "../TxtDataHelpers.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef WITH_SNMP +#include "../SNMPAgent.h" +#endif + +#ifdef ESP_PLATFORM +#include +#include +#include +#include +#include +#include +#include +#endif + +// Effective MQTT origin: empty mqtt_origin follows node_name; otherwise mqtt_origin override (quotes stripped). +static void applyEffectiveOrigin(const NodePrefs* np, const MQTTPrefs* obs, char* dest, size_t dest_size) { + if (!np || !obs || !dest || dest_size == 0) return; + if (obs->mqtt_origin[0] == '\0') { + strncpy(dest, np->node_name, dest_size - 1); + } else { + strncpy(dest, obs->mqtt_origin, dest_size - 1); + } + dest[dest_size - 1] = '\0'; + StrHelper::stripSurroundingQuotes(dest, dest_size); +} + +static const char* const kNtpBuiltinFallbacks[] = { + "pool.ntp.org", + "time.google.com", + "time.cloudflare.com", + "time.aws.com", + "time.nist.gov", +}; +static constexpr size_t kNtpBuiltinFallbackCount = + sizeof(kNtpBuiltinFallbacks) / sizeof(kNtpBuiltinFallbacks[0]); +static_assert(MQTTBridge::kMaxNtpServers >= 1 + (int)kNtpBuiltinFallbackCount, + "kMaxNtpServers must hold the custom primary plus all built-in fallbacks"); + +static bool ntpHostnameEquals(const char* a, const char* b) { + if (!a || !b) return false; + return strcasecmp(a, b) == 0; +} + +static void fillNtpServerList(const MQTTPrefs* prefs, const char* servers[], int& count) { + count = 0; + if (prefs && prefs->mqtt_ntp_server[0] != '\0') { + servers[count++] = prefs->mqtt_ntp_server; + } + for (size_t i = 0; i < kNtpBuiltinFallbackCount && count < MQTTBridge::kMaxNtpServers; i++) { + const char* fb = kNtpBuiltinFallbacks[i]; + bool dup = false; + for (int j = 0; j < count; j++) { + if (ntpHostnameEquals(servers[j], fb)) { + dup = true; + break; + } + } + if (!dup) { + servers[count++] = fb; + } + } +} + +const char* MQTTBridge::effectiveNtpPrimary(const MQTTPrefs* obs) { + if (obs && obs->mqtt_ntp_server[0] != '\0') { + return obs->mqtt_ntp_server; + } + return kNtpBuiltinFallbacks[0]; +} + +void MQTTBridge::refreshOriginFromPrefs() { + if (!_prefs) return; + applyEffectiveOrigin(_prefs, _obs, _origin, sizeof(_origin)); +} + +void MQTTBridge::getEffectiveMqttOrigin(const NodePrefs* np, const MQTTPrefs* obs, char* buf, size_t buf_size) { + if (!buf || buf_size == 0) return; + if (!np || !obs) { + buf[0] = '\0'; + return; + } + applyEffectiveOrigin(np, obs, buf, buf_size); +} + +// Helper function to check if WiFi credentials are valid +static bool isWiFiConfigValid(const MQTTPrefs* obs) { + // Check if WiFi SSID is configured (not empty) + if (!obs || strlen(obs->wifi_ssid) == 0) { + return false; + } + + // WiFi password can be empty for open networks, so we don't check it + + return true; +} + +#ifdef WITH_MQTT_BRIDGE + +// A custom slot endpoint is complete if a port is set, or if the host is a +// full URI with a scheme (esp-mqtt applies scheme default ports, and the URI +// builder in setupSlot() preserves any embedded port/path). +static bool customEndpointComplete(const char* host, uint16_t port) { + return host[0] != '\0' && (port != 0 || strstr(host, "://") != nullptr); +} + +bool MQTTBridge::isConfigValid(const MQTTPrefs* obs) { + if (!obs || !isWiFiConfigValid(obs)) return false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + const char* preset_name = obs->mqtt_slot_preset[i]; + if (preset_name[0] == '\0' || strcmp(preset_name, MQTT_PRESET_NONE) == 0) continue; + if (strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0) { + if (customEndpointComplete(obs->mqtt_slot_host[i], obs->mqtt_slot_port[i])) return true; + } else if (findMQTTPreset(preset_name) != nullptr) { + return true; + } + } + return false; +} + +// Optional embedded CA bundle symbols produced by board_build.embed_files. +// Weak linkage keeps non-bundle builds linkable and allows runtime fallback. +extern const uint8_t rootca_crt_bundle_start[] asm("_binary_src_certs_x509_crt_bundle_bin_start") __attribute__((weak)); +extern const uint8_t rootca_crt_bundle_end[] asm("_binary_src_certs_x509_crt_bundle_bin_end") __attribute__((weak)); + +// Track whether the global cert bundle has been loaded into s_crt_bundle. +// Loading must happen exactly once to avoid a use-after-free race when multiple +// TLS slots are set up in sequence (each connect() launches an async task). +static bool s_ca_bundle_loaded = false; + +// PSRAM-aware allocation: prefer PSRAM on ESP32 when BOARD_HAS_PSRAM, fallback to internal heap or malloc. +// Use psram_free() for any pointer returned by psram_malloc(). +static void* psram_malloc(size_t size) { + if (size == 0) return nullptr; +#if defined(ESP_PLATFORM) && defined(BOARD_HAS_PSRAM) + void* p = heap_caps_malloc(size, MALLOC_CAP_SPIRAM); + if (p != nullptr) return p; + p = heap_caps_malloc(size, MALLOC_CAP_INTERNAL); + return p; +#else + return malloc(size); +#endif +} + +static void* psram_calloc(size_t n, size_t size) { + if (n == 0 || size == 0) return nullptr; +#if defined(ESP_PLATFORM) && defined(BOARD_HAS_PSRAM) + void* p = heap_caps_calloc(n, size, MALLOC_CAP_SPIRAM); + if (p != nullptr) return p; + return heap_caps_calloc(n, size, MALLOC_CAP_INTERNAL); +#else + return calloc(n, size); +#endif +} + +static void psram_free(void* ptr) { + if (ptr == nullptr) return; +#if defined(ESP_PLATFORM) + heap_caps_free(ptr); +#else + free(ptr); +#endif +} + +static void* psram_realloc(void* ptr, size_t new_size) { + if (new_size == 0) { + psram_free(ptr); + return nullptr; + } +#if defined(ESP_PLATFORM) && defined(BOARD_HAS_PSRAM) + void* p = heap_caps_realloc(ptr, new_size, MALLOC_CAP_SPIRAM); + if (p != nullptr) return p; + // A block that fell back to internal DRAM on allocation (PSRAM exhausted) cannot + // be grown in PSRAM; retry there rather than reporting failure. + return heap_caps_realloc(ptr, new_size, MALLOC_CAP_INTERNAL); +#else + return realloc(ptr, new_size); +#endif +} + +// Shared JSON document pools follow the same PSRAM-first policy as the bridge's +// text buffers. ArduinoJson calls reallocate() when shrinking its pool list and +// asserts the result is non-null for a shrink, which both branches above satisfy. +void* MQTTBridge::JsonScratchAllocator::allocate(size_t size) { + return psram_malloc(size); +} + +void MQTTBridge::JsonScratchAllocator::deallocate(void* ptr) { + psram_free(ptr); +} + +void* MQTTBridge::JsonScratchAllocator::reallocate(void* ptr, size_t new_size) { + return psram_realloc(ptr, new_size); +} + +// Time (millis()) when WiFi was last seen connected; 0 when disconnected. Used for get wifi.status uptime. +static unsigned long s_wifi_connected_at = 0; + +// Last WiFi disconnect reason (from ESP-IDF event). Used for get wifi.status diagnostics. +static uint8_t s_wifi_disconnect_reason = 0; +static unsigned long s_wifi_disconnect_time = 0; + +#ifdef MQTT_MEMORY_DEBUG +// #region agent log +static void agentLogHeap(const char* location, const char* message, const char* hypothesisId, + size_t free_h, size_t max_alloc, unsigned long internal_free, unsigned long spiram_free) { + char buf[320]; + snprintf(buf, sizeof(buf), + "{\"sessionId\":\"debug-session\",\"location\":\"%s\",\"message\":\"%s\",\"hypothesisId\":\"%s\"," + "\"data\":{\"free\":%u,\"max_alloc\":%u,\"internal_free\":%lu,\"spiram_free\":%lu},\"timestamp\":%lu}", + location, message, hypothesisId, (unsigned)free_h, (unsigned)max_alloc, internal_free, spiram_free, + (unsigned long)millis()); + Serial.println(buf); +} +// #endregion +#endif + +// Singleton for formatMqttStatusReply (set in begin(), cleared in end()) +static MQTTBridge* s_mqtt_bridge_instance = nullptr; + +unsigned long MQTTBridge::getWifiConnectedAtMillis() { + return s_wifi_connected_at; +} + +#if defined(WITH_MQTT_NEIGHBORS) +// Compact "time remaining" for the `get mqtt.status` nbr field: "3h12m" / "12m" / "45s". +static void formatDuration(char* buf, size_t len, uint32_t secs) { + if (!buf || len == 0) return; + uint32_t h = secs / 3600; + uint32_t m = (secs % 3600) / 60; + if (h > 0) { + snprintf(buf, len, "%uh%um", (unsigned)h, (unsigned)m); + } else if (m > 0) { + snprintf(buf, len, "%um", (unsigned)m); + } else { + snprintf(buf, len, "%us", (unsigned)secs); + } +} +#endif + +void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPrefs* obs) { + if (buf == nullptr || bufsize == 0) return; + const char* msgs = (obs && obs->mqtt_status_enabled) ? "on" : "off"; + if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) { + snprintf(buf, bufsize, "> msgs: %s (bridge not running)", msgs); + return; + } + MQTTBridge* b = s_mqtt_bridge_instance; + + // Build per-slot status strings (compact format to fit 160-byte reply buffer) + // Only show configured slots, skip "none" slots + int q = 0; +#ifdef ESP_PLATFORM + if (b->_packet_queue_handle != nullptr) { + q = (int)uxQueueMessagesWaiting(b->_packet_queue_handle); + } +#else + q = b->_queue_count; +#endif + + // replyAppendf clamps pos into the buffer on every call, so no per-append + // guard or trailing clamp is needed (see MQTTReplyFormat.h / A1). + int pos = 0; + replyAppendf(buf, bufsize, &pos, "> msgs: %s", msgs); + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + const MQTTSlot& slot = b->_slots[i]; + const char* name = nullptr; + const char* state = nullptr; + + if (!slot.enabled && slot.preset) { + name = slot.preset->name; + state = "inactive"; + } else if (!slot.enabled) { + continue; // Skip unconfigured slots + } else if (!b->isSlotReady(i)) { + name = slot.preset ? slot.preset->name : "custom"; + state = "wait"; + } else if (slot.connected) { + name = slot.preset ? slot.preset->name : "custom"; + state = "ok"; + } else if (slot.circuit_breaker_tripped) { + name = slot.preset ? slot.preset->name : "custom"; + state = "fail"; + } else { + name = slot.preset ? slot.preset->name : "custom"; + state = "disc"; + } + replyAppendf(buf, bufsize, &pos, ", %d: %s (%s)", i + 1, name, state); + } + replyAppendf(buf, bufsize, &pos, ", q:%d", q); + +#if defined(WITH_MQTT_NEIGHBORS) + // Periodic neighbors: time to next publish + how the last one went. + if (obs && obs->mqtt_neighbors_enabled) { + char when[16]; + switch (b->_neighbors_phase.load(std::memory_order_relaxed)) { + case NBR_ACTIVE: strcpy(when, "active"); break; + case NBR_DUE: strcpy(when, "due"); break; + default: + formatDuration(when, sizeof(when), + b->_neighbors_secs_until_next.load(std::memory_order_relaxed)); + break; + } + const char* last; + switch (b->_neighbors_last_result.load(std::memory_order_relaxed)) { + case NBR_RESULT_OK: last = "ok"; break; + case NBR_RESULT_FAIL: last = "failed"; break; + default: last = "none"; break; + } + replyAppendf(buf, bufsize, &pos, ", nbr: %s/%s", when, last); + } +#endif +} + +// On-demand publish-health + heap snapshot for the `get mqtt.stats` CLI command. +// Same data as the (MQTT_MEMORY_DEBUG-only) periodic logMemoryStatus() line, but +// returned as a reply instead of logged. Per-slot "sN=ok/err": ok = cumulative +// accepted publishes, err = cumulative failures (socket error / network timeout). +// Outbox should read ~0 (QoS0 publishes synchronously); a rising err isolates a +// broker whose uplink is dropping writes. +void MQTTBridge::formatMqttStatsReply(char* buf, size_t bufsize) { + if (buf == nullptr || bufsize == 0) return; + if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) { + snprintf(buf, bufsize, "> (bridge not running)"); + return; + } + MQTTBridge* b = s_mqtt_bridge_instance; + + int q = 0; +#ifdef ESP_PLATFORM + if (b->_packet_queue_handle != nullptr) { + q = (int)uxQueueMessagesWaiting(b->_packet_queue_handle); + } +#else + q = b->_queue_count; +#endif + + size_t outbox_total = 0; + unsigned long outbox_drops = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (b->_slots[i].client) { + outbox_total += b->_slots[i].client->getOutboxSize(); + outbox_drops += b->_slots[i].client->getOutboxDrops(); + } + } + + // drops=/: outbox-cap drops vs. memory-pressure skips. + int pos = 0; + replyAppendf(buf, bufsize, &pos, "> Free=%d Max=%d q:%d/%d Outbox=%u drops=%lu/%d", + (int)ESP.getFreeHeap(), (int)ESP.getMaxAllocHeap(), + q, MAX_QUEUE_SIZE, (unsigned)outbox_total, + outbox_drops, b->_skipped_publishes); + // filt=: packets the per-slot type filters rejected before the queue. + // Omitted while zero so an unfiltered node's reply keeps its former length — + // the per-slot list below is what usually gets clamped away first. + if (b->_filtered_packets > 0) { + replyAppendf(buf, bufsize, &pos, " filt=%lu", b->_filtered_packets); + } + replyAppendf(buf, bufsize, &pos, " |"); + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (!b->_slots[i].enabled || !b->_slots[i].client) continue; + replyAppendf(buf, bufsize, &pos, " s%d=%lu/%lu", i + 1, + b->_slots[i].client->getPublishOk(), + b->_slots[i].client->getPublishErr()); + } +} + +// Structured per-slot status for the webconfig stats endpoint. Same state +// derivation as formatMqttStatusReply above. Returns false for out-of-range, +// unconfigured, or bridge-not-running slots. +bool MQTTBridge::getSlotStatusSnapshot(int slot_index, SlotStatusSnapshot* out) { + if (out == nullptr || slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return false; + if (s_mqtt_bridge_instance == nullptr || !s_mqtt_bridge_instance->_initialized) return false; + MQTTBridge* b = s_mqtt_bridge_instance; + const MQTTSlot& slot = b->_slots[slot_index]; + + if (!slot.enabled && slot.preset) { + out->name = slot.preset->name; + out->state = "inactive"; + } else if (!slot.enabled) { + return false; // unconfigured slot + } else { + out->name = slot.preset ? slot.preset->name : "custom"; + if (!b->isSlotReady(slot_index)) { + out->state = "wait"; + } else if (slot.connected) { + out->state = "ok"; + } else if (slot.circuit_breaker_tripped) { + out->state = "fail"; + } else { + out->state = "disc"; + } + } + out->publish_ok = slot.client ? slot.client->getPublishOk() : 0; + out->publish_err = slot.client ? slot.client->getPublishErr() : 0; + // Lets the portal show why a healthy slot is quiet. + out->filter_mask = b->_obs ? b->_obs->mqtt_slot_packet_filter[slot_index] + : MQTTPacketFilter::kAllPacketTypes; + return true; +} + +int MQTTBridge::getMaxActiveSlots() { + // Each WSS/TLS connection needs ~40KB for mbedTLS buffers. Without PSRAM even + // 3 concurrent connections would exhaust internal heap, so cap at 2; with + // PSRAM cap at 5 (6 configurable but 5 active max). +#if defined(ESP_PLATFORM) && defined(BOARD_HAS_PSRAM) + return psramFound() ? 5 : 2; +#else + return 2; +#endif +} + +uint8_t MQTTBridge::getLastWifiDisconnectReason() { return s_wifi_disconnect_reason; } +unsigned long MQTTBridge::getLastWifiDisconnectTime() { return s_wifi_disconnect_time; } + +unsigned long MQTTBridge::getSlotCurrentOutageStartMs(int slot_index) const { + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return 0; + return _slots[slot_index].current_outage_started_ms; +} + +bool MQTTBridge::isSlotEnabledAndAttempted(int slot_index) const { + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return false; + const MQTTSlot& s = _slots[slot_index]; + return s.enabled && s.initial_connect_done; +} + +const char* MQTTBridge::getSlotPresetName(int slot_index) const { + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return "?"; + const MQTTSlot& s = _slots[slot_index]; + if (s.preset && s.preset->name) return s.preset->name; + if (!s.enabled) return MQTT_PRESET_NONE; + return MQTT_PRESET_CUSTOM; +} + +const char* MQTTBridge::wifiReasonStr(uint8_t reason) { + switch (reason) { + case 2: return "auth expired"; + case 4: return "assoc timeout"; + case 8: return "AP disconnected"; + case 15: return "4-way handshake timeout"; + case 18: return "group cipher mismatch"; + case 40: return "cipher suite rejected"; + case 49: return "invalid PMKID"; + case 61: return "AP BSS management"; + case 88: return "AP BSS management"; + case 168: return "AP band-steering kick"; + case 34: return "AP state mismatch (class 3 frame)"; + case 39: return "SSID not found"; + case 63: return "SA query timeout (PMF)"; + case 200: return "signal lost"; + case 201: return "security mismatch"; + case 202: return "auth mode rejected"; + case 204: return "handshake timeout"; + default: return nullptr; + } +} + +const char* MQTTBridge::tlsErrorStr(int32_t err) { + switch (err) { + case 0x8001: return "DNS failed"; + case 0x8002: return "socket error"; + case 0x8004: return "connect refused"; + case 0x8006: return "TLS timeout"; + case 0x8008: return "connection timeout"; + case 0x800B: return "cert verify failed"; + case 0x8010: return "mbedTLS error"; + case 0x801A: return "TLS handshake failed"; + default: return nullptr; + } +} + +void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) { + if (!buf || bufsize == 0) return; + if (!s_mqtt_bridge_instance || !s_mqtt_bridge_instance->_initialized) { + snprintf(buf, bufsize, "> mqtt%d: bridge not running", slot_index + 1); + return; + } + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) { + snprintf(buf, bufsize, "> invalid slot"); + return; + } + + MQTTBridge* b = s_mqtt_bridge_instance; + const MQTTSlot& slot = b->_slots[slot_index]; + + // Determine state string + const char* state; + if (!slot.enabled && !slot.preset && slot.host[0] == '\0') { + snprintf(buf, bufsize, "> mqtt%d: not configured", slot_index + 1); + return; + } else if (!slot.enabled) { + state = "inactive"; + } else if (!b->isSlotReady(slot_index)) { + // Same classification as `get mqtt.status` and getSlotStatusSnapshot(): the slot + // is configured but missing a token/IATA/credential, so it was never set up and + // has no client yet. Previously reported "disc", which read as a network fault. + state = "wait"; + } else if (!slot.client) { + // Ready to connect but the client object could not be allocated. + state = "no client"; + } else if (slot.connected) { + state = "ok"; + } else if (slot.circuit_breaker_tripped) { + state = "fail"; + } else { + state = "disc"; + } + + // replyAppendf clamps pos on every call, so the chained appends below can't + // walk past the reply buffer even if the accumulated text exceeds it (A1). + // A non-default filter is the one healthy-looking reason for a slot to stop + // publishing, so it has to appear — but appended last. replyAppendf clamps at + // the 160-byte reply, and an error tail (TLS + mbedTLS + errno + age) can + // already reach ~117 chars, so putting the filter first would push the + // operator's diagnostic detail off the end of a failing slot's line. + const uint16_t filter_mask = b->_obs ? b->_obs->mqtt_slot_packet_filter[slot_index] + : MQTTPacketFilter::kAllPacketTypes; + char filter_text[MQTTPacketFilter::kFilterTextSize]; + const bool show_filter = filter_mask != MQTTPacketFilter::kAllPacketTypes && + MQTTPacketFilter::format(filter_mask, filter_text, sizeof(filter_text)); + + int pos = 0; + replyAppendf(buf, bufsize, &pos, "> mqtt%d: %s", slot_index + 1, state); + if (slot.disconnect_count > 0) { + replyAppendf(buf, bufsize, &pos, ", dc:%lu", (unsigned long)slot.disconnect_count); + if (slot.first_disconnect_time > 0) { + unsigned long first_disc_age_sec = (millis() - slot.first_disconnect_time) / 1000; + replyAppendf(buf, bufsize, &pos, ", first_disc:%lus", first_disc_age_sec); + } + } + + // Connected with no errors: nothing more to say about the connection. + if (slot.connected && slot.last_error_time == 0) { + replyAppendf(buf, bufsize, &pos, ", no errors"); + } else if (slot.last_error_time > 0) { + // TLS error with human-friendly description + if (slot.last_tls_err != 0) { + const char* desc = tlsErrorStr(slot.last_tls_err); + if (desc) { + replyAppendf(buf, bufsize, &pos, ", %s (0x%04X)", desc, (unsigned)slot.last_tls_err); + } else { + replyAppendf(buf, bufsize, &pos, ", tls:0x%04X", (unsigned)slot.last_tls_err); + } + } + // mbedTLS stack error (shown as negative hex per convention) + if (slot.last_tls_stack_err != 0) { + replyAppendf(buf, bufsize, &pos, ", mbedtls:-0x%04X", (unsigned)(-slot.last_tls_stack_err)); + } + // Socket errno + if (slot.last_sock_errno != 0) { + replyAppendf(buf, bufsize, &pos, ", sock:%d", slot.last_sock_errno); + } + // Time ago + unsigned long ago_sec = (millis() - slot.last_error_time) / 1000; + if (ago_sec < 60) { + replyAppendf(buf, bufsize, &pos, ", %lus ago", ago_sec); + } else if (ago_sec < 3600) { + replyAppendf(buf, bufsize, &pos, ", %lum ago", ago_sec / 60); + } else { + replyAppendf(buf, bufsize, &pos, ", %luh ago", ago_sec / 3600); + } + } else if (!slot.connected) { + replyAppendf(buf, bufsize, &pos, ", no error info"); + } + + // Appended last so it never displaces connection diagnostics. replyAppendf + // clamps rather than overflows, but a clipped type list is worse than no + // list: "…,13,14," parses as a real, different allowlist, and this is the + // one line an operator reads to find out why a slot is quiet. So the exact + // text is only emitted when it fits whole; otherwise fall back to a count, + // which cannot be misread. `get mqttN.filter` always has the exact value. + if (show_filter) { + static const int kFilterLabelLen = (int)sizeof(", filter:") - 1; + const int remaining = pos < (int)bufsize ? (int)bufsize - 1 - pos : 0; + const uint8_t allowed = MQTTPacketFilter::countTypes(filter_mask); + const int compact_len = kFilterLabelLen + (allowed >= 10 ? 5 : 4); // "N/16" + if (kFilterLabelLen + (int)strlen(filter_text) <= remaining) { + replyAppendf(buf, bufsize, &pos, ", filter:%s", filter_text); + } else if (compact_len <= remaining) { + replyAppendf(buf, bufsize, &pos, ", filter:%u/16", (unsigned)allowed); + } + // Neither fits: the slot is reporting so much error detail that the filter + // is the least useful field on the line. Omit it rather than mislead. + } +} + +// Bounded cooperative-stop timeout for end() (see MQTTLifecycle::Coordinator). +// Phase 0 hardware characterization (2026-07-19, Heltec V3 non-PSRAM + V4 PSRAM, +// see STABILITY_TESTABILITY_HANDOFF.md): a real mbedTLS/wss client teardown +// (disconnect + esp_mqtt_client_destroy) takes ~5-6 s per CONNECTED slot, applied +// SEQUENTIALLY in destroySlotClients(). So the safe timeout scales with the +// number of slots being torn down, not a single constant: a flat 8 s tripped the +// dirty/force-kill fallback on a healthy 2-slot non-PSRAM node (~11-12 s) and a +// normal 3-slot PSRAM node (~16 s), which withholds OTA on healthy devices. +// +// The budget below gives generous headroom (~8 s/slot vs the ~5-6 s measured) +// plus a fixed base for WiFi/queue/buffer teardown. Headroom is nearly free: +// end() returns as soon as the task acks (it checks _stop_acked before ticking +// the timeout), so a larger bound does NOT slow a healthy stop — it only length- +// ens the wait before force-killing a genuinely wedged task. The timeout is set +// per stop in end() via computeStopTimeoutMs() based on the enabled-slot count. +static const uint32_t MQTT_STOP_TIMEOUT_BASE_MS = 5000; // fixed teardown overhead +static const uint32_t MQTT_STOP_TIMEOUT_PER_SLOT_MS = 8000; // ~5-6 s measured + headroom + +// Slot-scaled cooperative-stop timeout. `slots` is the number of MQTT slots that +// will be torn down (enabled/connected); clamped to >=1 so a zero-slot bridge +// still budgets for the base teardown. +static inline uint32_t mqttStopTimeoutForSlots(int slots) { + if (slots < 1) slots = 1; + return MQTT_STOP_TIMEOUT_BASE_MS + MQTT_STOP_TIMEOUT_PER_SLOT_MS * (uint32_t)slots; +} + +// --------------------------------------------------------------------------- +// Constructor +// --------------------------------------------------------------------------- +MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mgr, mesh::RTCClock *rtc, mesh::LocalIdentity *identity) + : BridgeBase(prefs, mgr, rtc), + _obs(obs), + _queue_count(0), + _last_status_publish(0), _last_status_retry(0), _status_interval(300000), + _ntp_client(_ntp_udp, effectiveNtpPrimary(obs), 0, 60000), _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), + _ntp_force_requested(false), _ntp_force_done(false), _ntp_force_result(false), + _ntp_diag_requested(false), _ntp_diag_done(false), _ntp_diag_count(0), + // Default to UTC; setRules() will be called from syncTimeWithNTP when a + // non-UTC timezone string is configured. Timezone has no default ctor, + // so we must pass rules here. + _timezone_storage(TimeChangeRule{"UTC", Last, Sun, Mar, 0, 0}, TimeChangeRule{"UTC", Last, Sun, Mar, 0, 0}), + _timezone(&_timezone_storage), +#if defined(BOARD_HAS_PSRAM) + _last_raw_data(nullptr), +#endif + _last_raw_len(0), _last_snr(0), _last_rssi(0), _last_raw_timestamp(0), +#if defined(BOARD_HAS_PSRAM) + _json_scratch_buffer(nullptr), +#endif + _identity(identity), + _cached_has_connected_slots(false), + _last_memory_check(0), _skipped_publishes(0), + _last_no_broker_log(0), _queue_disconnected_since(0), + _last_config_warning(0), + _dispatcher(nullptr), _radio(nullptr), _board(nullptr), _ms(nullptr), +#ifdef WITH_SNMP + _snmp_agent(nullptr), +#endif + _last_wifi_check(0), _last_wifi_status(WL_DISCONNECTED), _wifi_status_initialized(false), + _wifi_disconnected_time(0), _last_wifi_reconnect_attempt(0), _wifi_reconnect_backoff_attempt(0), + _last_slot_reconnect_ms(0) +#ifdef ESP_PLATFORM + , _packet_queue_handle(nullptr), _mqtt_task_handle(nullptr), + _packet_queue_storage(nullptr) +#else + , _queue_head(0), _queue_tail(0) +#endif + // Cooperative lifecycle: _lifecycle_ops must be constructed before + // _lifecycle (declaration order guarantees this) so the reference binds. + // Seed with the worst-case (max runtime slots) budget; end() recomputes the + // slot-scaled timeout before each stop via setStopTimeoutMs(). + , _lifecycle_ops(this), _lifecycle(_lifecycle_ops, mqttStopTimeoutForSlots(RUNTIME_MQTT_SLOTS)) +{ + // Initialize default values + strncpy(_origin, "MeshCore-Repeater", sizeof(_origin) - 1); + strncpy(_iata, "XXX", sizeof(_iata) - 1); + strncpy(_device_id, "DEVICE_ID_PLACEHOLDER", sizeof(_device_id) - 1); + strncpy(_firmware_version, "unknown", sizeof(_firmware_version) - 1); + strncpy(_board_model, "unknown", sizeof(_board_model) - 1); + strncpy(_build_date, "unknown", sizeof(_build_date) - 1); + _status_enabled = true; + _packets_enabled = true; + _raw_enabled = false; + _rx_enabled = true; + _tx_mode = 0; + + // Initialize all slots to empty/disabled state + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + memset(&_slots[i], 0, sizeof(MQTTSlot)); + _slots[i].enabled = false; + _slots[i].client = nullptr; + _slots[i].preset = nullptr; + // auth_token == nullptr after memset above — allocated on first token creation + _slots[i].connected = false; + _slots[i].initial_connect_done = false; + _slots[i].token_expires_at = 0; + _slots[i].last_token_renewal = 0; + _slots[i].reconnect_backoff = 0; + _slots[i].max_backoff_failures = 0; + _slots[i].circuit_breaker_tripped = false; + _slots[i].last_reconnect_attempt = 0; + _slots[i].last_log_time = 0; + _slots[i].port = 1883; + _slot_reconfigure_pending[i] = false; + _status_publish_pending[i] = false; + } + + // Reset CLI-requested forced NTP sync handshake (bridge object is reused across restarts) + _ntp_force_requested = false; + _ntp_force_done = false; + _ntp_force_result = false; + + // Reset CLI-requested NTP diagnostic handshake + _ntp_diag_requested = false; + _ntp_diag_done = false; + _ntp_diag_count = 0; + +#if defined(WITH_MQTT_NEIGHBORS) + // Neighbors publish handoff (buffer allocated in begin() after PSRAM probe). + // std::atomic has no value-initializing default ctor pre-C++20, so set them here. + _neighbors_json_buffer = nullptr; + _neighbors_publish_len = 0; + _neighbors_publish_pending.store(false, std::memory_order_relaxed); + _neighbors_last_result.store(NBR_RESULT_NONE, std::memory_order_relaxed); + _neighbors_phase.store(NBR_SCHEDULED, std::memory_order_relaxed); + _neighbors_secs_until_next.store(0, std::memory_order_relaxed); +#endif + + // Initialize JWT username + _jwt_username[0] = '\0'; + + // Initialize packet queue (FreeRTOS queue will be created in begin()) + #ifdef ESP_PLATFORM + // Queue and mutex will be created in begin() + #else + // Initialize circular buffer for non-ESP32 platforms + memset(_packet_queue, 0, sizeof(_packet_queue)); +#if defined(BOARD_HAS_PSRAM) + for (int i = 0; i < MAX_QUEUE_SIZE; i++) { + _packet_queue[i].has_raw_data = false; + } +#endif + #endif + + // Non-PSRAM boards keep the raw cache inline for the bridge lifetime. + // PSRAM boards allocate their runtime buffers in begin(), after PSRAM has + // been probed/initialized, and release them in end(). + #if !defined(BOARD_HAS_PSRAM) + memset(_last_raw_data, 0, sizeof(_last_raw_data)); + #endif + // The shared JSON document needs no setup here: its pools are allocated lazily on + // the first publish through _json_allocator and released by releaseRuntimeBuffers(). +} + +void MQTTBridge::allocateRuntimeBuffers() { + #if defined(BOARD_HAS_PSRAM) + // Keep each allocation independent. A nullptr is deliberately retained on + // failure: status/packet publish paths already use stack fallbacks, and the + // next begin() will retry only the missing buffer. + _last_raw_data = static_cast(MQTTRuntimeBufferLifecycle::allocateIfMissing( + _last_raw_data, LAST_RAW_DATA_SIZE, psram_malloc)); + _json_scratch_buffer = static_cast(MQTTRuntimeBufferLifecycle::allocateIfMissing( + _json_scratch_buffer, PUBLISH_JSON_BUFFER_SIZE, psram_malloc)); + MQTT_DEBUG_PRINTLN("Runtime buffers: raw=%s json=%s", + _last_raw_data ? "PSRAM" : "unavailable", + _json_scratch_buffer ? "PSRAM" : "stack fallback"); + #endif + +#if defined(WITH_MQTT_NEIGHBORS) + // Persistent neighbors JSON buffer, heap-allocated on every board: too large to + // keep inline in the bridge object the way the non-PSRAM status/packet buffers + // are. psram_malloc() falls back to internal DRAM, so this works without PSRAM. + // Unlike status/packet there is no stack fallback — a nullptr simply disables + // publishing (requestPublishNeighbors/publishNeighbors both no-op on nullptr). + _neighbors_json_buffer = static_cast(MQTTRuntimeBufferLifecycle::allocateIfMissing( + _neighbors_json_buffer, NEIGHBORS_JSON_BUFFER_SIZE, psram_malloc)); + MQTT_DEBUG_PRINTLN("Neighbors buffer: %s", + _neighbors_json_buffer ? "ready" : "unavailable"); +#endif +} + +void MQTTBridge::releaseRuntimeBuffers() { + #if defined(BOARD_HAS_PSRAM) + _last_raw_data = static_cast(MQTTRuntimeBufferLifecycle::release( + _last_raw_data, psram_free)); + _json_scratch_buffer = static_cast(MQTTRuntimeBufferLifecycle::release( + _json_scratch_buffer, psram_free)); + #endif + + // Drop the shared document's pools with the buffers. clear() destroys every pool + // and resets the list to its inline array; the next publish reallocates. Holding + // 4 KB of pool across a stopped bridge is pure overhead. + _json_scratch_doc.clear(); + +#if defined(WITH_MQTT_NEIGHBORS) + // Paired with the unconditional allocation in allocateRuntimeBuffers(). + _neighbors_json_buffer = static_cast(MQTTRuntimeBufferLifecycle::release( + _neighbors_json_buffer, psram_free)); + _neighbors_publish_len = 0; + _neighbors_publish_pending.store(false, std::memory_order_release); +#endif + + // Never pair a newly allocated raw buffer with metadata from a prior bridge + // run. This also makes non-PSRAM restarts discard their stale raw cache. + _last_raw_len = 0; + _last_snr = 0; + _last_rssi = 0; + _last_raw_timestamp = 0; +} + +// --------------------------------------------------------------------------- +// begin() +// --------------------------------------------------------------------------- +void MQTTBridge::begin() { + MQTT_DEBUG_PRINTLN("Initializing MQTT Bridge..."); + + // Idempotent start (Phase 5): a second begin() on an already-running bridge + // would re-run allocation and re-create the task, leaking the previous + // queue/task. Guard here instead of relying on caller discipline. + if (_initialized) { + MQTT_DEBUG_PRINTLN("MQTT Bridge already running - begin() ignored"); + return; + } + + // PSRAM diagnostic - helps debug memory fragmentation on boards with external RAM + #ifdef BOARD_HAS_PSRAM + { + bool psram_available = psramFound(); + size_t psram_size = 0; + size_t psram_free = 0; + if (psram_available) { + psram_size = ESP.getPsramSize(); + psram_free = ESP.getFreePsram(); + } + MQTT_DEBUG_PRINTLN("PSRAM: found=%s, size=%u, free=%u", + psram_available ? "YES" : "NO", psram_size, psram_free); + if (!psram_available) { + MQTT_DEBUG_PRINTLN("PSRAM: board has PSRAM flag but psramFound()=false. " + "Trying explicit psramInit()..."); + bool init_result = psramInit(); + MQTT_DEBUG_PRINTLN("PSRAM: psramInit() returned %s", init_result ? "true" : "false"); + if (init_result) { + psram_size = ESP.getPsramSize(); + psram_free = ESP.getFreePsram(); + MQTT_DEBUG_PRINTLN("PSRAM: after init - size=%u, free=%u", psram_size, psram_free); + } + } + // Log internal heap for comparison + MQTT_DEBUG_PRINTLN("PSRAM: internal_free=%u, internal_max_alloc=%u", + heap_caps_get_free_size(MALLOC_CAP_INTERNAL), + heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL)); + } + #else + MQTT_DEBUG_PRINTLN("PSRAM: not configured for this board (no BOARD_HAS_PSRAM)"); + #endif + + // Limit active slots based on available memory (see getMaxActiveSlots()). + _max_active_slots = getMaxActiveSlots(); + MQTT_DEBUG_PRINTLN("Max active slots: %d", _max_active_slots); + + // Check if WiFi credentials are configured first + if (!isWiFiConfigValid(_obs)) { + MQTT_DEBUG_PRINTLN("MQTT Bridge initialization skipped - WiFi credentials not configured"); + return; + } + + // These are begin()/end()-scoped on PSRAM targets. Allocation happens after + // the PSRAM probe above so a late psramInit() has taken effect. + allocateRuntimeBuffers(); + + refreshOriginFromPrefs(); + + strncpy(_iata, _obs->mqtt_iata, sizeof(_iata) - 1); + _iata[sizeof(_iata) - 1] = '\0'; + + StrHelper::stripSurroundingQuotes(_iata, sizeof(_iata)); + + // Convert IATA code to uppercase (IATA codes are conventionally uppercase) + for (int i = 0; _iata[i]; i++) { + _iata[i] = toupper(_iata[i]); + } + + // Initial snapshot of the publish toggles. NOTE: the publish hot paths read + // these live from _obs->mqtt_* (status/packets/raw/rx/tx) so a CLI/web `set` + // takes effect without a bridge restart; these members are kept only for + // startup logging/back-compat and are not the source of truth. + _status_enabled = _obs->mqtt_status_enabled; + _packets_enabled = _obs->mqtt_packets_enabled; + _raw_enabled = _obs->mqtt_raw_enabled; + _rx_enabled = _obs->mqtt_rx_enabled; + _tx_mode = _obs->mqtt_tx_enabled; // 0=off, 1=all, 2=advert + // Set status interval to 5 minutes (300000 ms), or use preference if set and valid + if (_obs->mqtt_status_interval >= 1000 && _obs->mqtt_status_interval <= 3600000) { + _status_interval = _obs->mqtt_status_interval; + } else { + // Invalid or uninitialized value - fix it in preferences and use default + _obs->mqtt_status_interval = 300000; // Fix the preference value + _status_interval = 300000; // 5 minutes default + } + + // Check for configuration mismatch: bridge.source=tx but mqtt.tx=off + checkConfigurationMismatch(); + + MQTT_DEBUG_PRINTLN("Config: Origin=%s, IATA=%s, Device=%s", _origin, _iata, _device_id); + + // Apply slot presets from preferences + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + const char* preset_name = _obs->mqtt_slot_preset[i]; + if (preset_name[0] != '\0' && strcmp(preset_name, MQTT_PRESET_NONE) != 0) { + if (strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0) { + // Custom broker: copy host/port/username/password from prefs + _slots[i].preset = nullptr; + strncpy(_slots[i].host, _obs->mqtt_slot_host[i], sizeof(_slots[i].host) - 1); + _slots[i].host[sizeof(_slots[i].host) - 1] = '\0'; + if (strlen(_slots[i].host) == 0) { + MQTT_DEBUG_PRINTLN("MQTT%d: custom preset has no server configured, disabling", i + 1); + _slots[i].enabled = false; + continue; + } + _slots[i].enabled = true; + _slots[i].port = _obs->mqtt_slot_port[i]; + strncpy(_slots[i].username, _obs->mqtt_slot_username[i], sizeof(_slots[i].username) - 1); + _slots[i].username[sizeof(_slots[i].username) - 1] = '\0'; + strncpy(_slots[i].password, _obs->mqtt_slot_password[i], sizeof(_slots[i].password) - 1); + _slots[i].password[sizeof(_slots[i].password) - 1] = '\0'; + strncpy(_slots[i].audience, _obs->mqtt_slot_audience[i], sizeof(_slots[i].audience) - 1); + _slots[i].audience[sizeof(_slots[i].audience) - 1] = '\0'; + } else { + const MQTTPresetDef* preset = findMQTTPreset(preset_name); + if (preset) { + _slots[i].enabled = true; + _slots[i].preset = preset; + if (mqttPresetNeedsSlotCredentials(preset)) { + strncpy(_slots[i].username, _obs->mqtt_slot_username[i], sizeof(_slots[i].username) - 1); + _slots[i].username[sizeof(_slots[i].username) - 1] = '\0'; + strncpy(_slots[i].password, _obs->mqtt_slot_password[i], sizeof(_slots[i].password) - 1); + _slots[i].password[sizeof(_slots[i].password) - 1] = '\0'; + } + } else { + MQTT_DEBUG_PRINTLN("MQTT%d: unknown preset '%s', disabling", i + 1, preset_name); + _slots[i].enabled = false; + } + } + } + } + + // Log slot configuration + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled) { + if (_slots[i].preset) { + MQTT_DEBUG_PRINTLN("MQTT%d: preset=%s", i + 1, _slots[i].preset->name); + } else { + MQTT_DEBUG_PRINTLN("MQTT%d: custom=%s:%d", i + 1, _slots[i].host, _slots[i].port); + } + } else { + MQTT_DEBUG_PRINTLN("MQTT%d: none", i + 1); + } + } + + #ifdef ESP_PLATFORM + // Create FreeRTOS queue; use PSRAM storage when available + #ifdef BOARD_HAS_PSRAM + _packet_queue_storage = (uint8_t*)psram_malloc(MAX_QUEUE_SIZE * sizeof(QueuedPacket)); + if (_packet_queue_storage != nullptr) { + _packet_queue_handle = xQueueCreateStatic(MAX_QUEUE_SIZE, sizeof(QueuedPacket), _packet_queue_storage, &_packet_queue_struct); + } else { + _packet_queue_handle = nullptr; + } + #else + // Non-PSRAM: use inline class-member storage with static queue creation. + // Eliminates a separate heap allocation, reducing startup fragmentation. + _packet_queue_storage = _packet_queue_inline; + _packet_queue_handle = xQueueCreateStatic(MAX_QUEUE_SIZE, sizeof(QueuedPacket), + _packet_queue_storage, &_packet_queue_struct); + #endif + if (_packet_queue_handle == nullptr) { + _packet_queue_handle = xQueueCreate(MAX_QUEUE_SIZE, sizeof(QueuedPacket)); + } + if (_packet_queue_handle == nullptr) { + MQTT_DEBUG_PRINTLN("Failed to create packet queue!"); + #if defined(BOARD_HAS_PSRAM) + psram_free(_packet_queue_storage); + #endif + _packet_queue_storage = nullptr; + releaseRuntimeBuffers(); + return; + } + + // Create FreeRTOS task for MQTT/WiFi processing on Core 0 + #ifndef MQTT_TASK_CORE + #define MQTT_TASK_CORE 0 + #endif + #ifndef MQTT_TASK_STACK_SIZE + #define MQTT_TASK_STACK_SIZE 8192 + #endif + #ifndef MQTT_TASK_PRIORITY + #define MQTT_TASK_PRIORITY 1 + #endif + + // Task stack: dynamic allocation (internal RAM). A PSRAM-backed stack was tried and + // reverted — it resets some boards (e.g. Heltec V4) when the task runs from PSRAM. + _mqtt_task_handle = nullptr; + // Clear the cooperative-stop handshake before the new task starts reading it. + // deliverStop() leaves _stop_requested latched true after a stop cycle, so a + // restart must reset it or the fresh task would self-terminate immediately. + _stop_requested = false; + _stop_acked = false; + BaseType_t create_result = xTaskCreatePinnedToCore( + mqttTask, + "MQTTBridge", + MQTT_TASK_STACK_SIZE, + this, + MQTT_TASK_PRIORITY, + &_mqtt_task_handle, + MQTT_TASK_CORE + ); + if (create_result != pdPASS) _mqtt_task_handle = nullptr; + if (_mqtt_task_handle == nullptr) { + MQTT_DEBUG_PRINTLN("Failed to create MQTT task!"); + vQueueDelete(_packet_queue_handle); + _packet_queue_handle = nullptr; + #if defined(BOARD_HAS_PSRAM) + psram_free(_packet_queue_storage); + #endif + _packet_queue_storage = nullptr; + releaseRuntimeBuffers(); + return; + } + + MQTT_DEBUG_PRINTLN("MQTT task created on Core %d", MQTT_TASK_CORE); + #else + // Non-ESP32: Initialize WiFi directly (no task) + WiFi.mode(WIFI_STA); + WiFi.setAutoReconnect(true); + WiFi.setAutoConnect(true); + WiFi.begin(_obs->wifi_ssid, _obs->wifi_password); + + // NOTE: Slot setup deferred until after NTP sync in loop() + #endif + + // MQTT client objects are NOT allocated here. setupSlot() creates one on a slot's + // first setup, so unconfigured and capped-off slots never cost their ~1.3 KB of + // internal DRAM. Once created a client lives for the bridge's lifetime, so the + // reconfigure/reconnect paths still reuse the same mbedTLS context instead of + // churning ~40 KB of internal heap per cycle. + + // Sync the lifecycle Coordinator to Running now that all resources exist and + // the task is created. Driven only on the success path: the failure rollbacks + // above already free what they acquired and leave the bridge Stopped, so we + // must not also fire the state machine's release effect there (double free). + // A fresh start also clears any dirty-stop latch (re-enabling OTA flashing). + _lifecycle.requestStart(); // Stopped -> Starting (startTask() is a no-op here) + _lifecycle.onTaskStarted(); // Starting -> Running + + _initialized = true; + s_mqtt_bridge_instance = this; + MQTT_DEBUG_PRINTLN("MQTT Bridge initialized"); +} + +// --------------------------------------------------------------------------- +// end() +// --------------------------------------------------------------------------- +void MQTTBridge::end() { + MQTT_DEBUG_PRINTLN("Stopping MQTT Bridge..."); + + // Idempotent stop: nothing to tear down if we never started (or already stopped). + if (!_initialized) { + MQTT_DEBUG_PRINTLN("MQTT Bridge already stopped - end() ignored"); + return; + } + + // Stop new diagnostic reads through the singleton before teardown begins. + s_mqtt_bridge_instance = nullptr; + + // Size the stop timeout to the work about to happen: each enabled slot's + // mbedTLS/wss client takes ~5-6 s to disconnect + destroy, sequentially (Phase + // 0 hardware characterization). A flat bound force-killed healthy multi-slot + // nodes and withheld OTA; the slot-scaled budget lets a normal teardown ack + // cleanly. Count enabled slots (the ones that connect); a disabled slot's + // client destroys quickly. Must run BEFORE requestStop() arms the window. + int stop_slots = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled) stop_slots++; + } + _lifecycle.setStopTimeoutMs(mqttStopTimeoutForSlots(stop_slots)); + MQTT_DEBUG_PRINTLN("MQTT stop: %d enabled slot(s), timeout %lu ms", + stop_slots, (unsigned long)_lifecycle.stopTimeoutMs()); + + // Cooperative shutdown (Phase 5). Request the stop, then let the lifecycle + // Coordinator drive it. On ESP32 the MQTT task (Core 0) tears down its own + // clients where the mbedTLS contexts live and acknowledges via _stop_acked; + // the queue/buffer release happens inside LifecycleOps::releaseResources() + // once the Coordinator reaches Stopped (clean ack OR the reviewed timeout + // fallback). This replaces the former blind vTaskDelete that could kill the + // task mid-mbedTLS and then free client buffers on a corrupted heap. + _lifecycle.requestStop(); // Running -> StopRequested; deliverStop() sets _stop_requested + +#ifdef ESP_PLATFORM + // Wait (bounded) for the task to acknowledge. tick() synthesizes the timeout + // fallback if the task never acks. Checking the ack first each iteration means + // a stop that completes right as the timeout expires is still treated as clean. + while (_lifecycle.isStopInProgress()) { + if (_stop_acked) { + _lifecycle.onTaskStopped(); // StopRequested -> Stopped (clean): releaseResources() + break; + } + _lifecycle.tick(); // may fire StopTimedOut -> Stopped (dirty): releaseResources() + if (!_lifecycle.isStopInProgress()) break; + vTaskDelay(pdMS_TO_TICKS(20)); + } +#else + // Non-ESP32: the bridge runs cooperatively in loop(); there is no separate + // task to signal. Drive straight to a clean Stopped and let releaseResources() + // perform the (unchanged) synchronous teardown. + _stop_acked = true; + _lifecycle.onTaskStopped(); +#endif + + // Timezone is inline class storage (_timezone_storage) — nothing to delete. + // The shared JSON document's pools were freed by releaseRuntimeBuffers() above. + _initialized = false; + _slots_setup_done = false; // Reset so deferred setup runs again on next begin() + MQTT_DEBUG_PRINTLN("MQTT Bridge stopped (%s)", + _lifecycle.stopTimedOut() ? "forced/timeout - OTA blocked" : "clean"); +} + +// --------------------------------------------------------------------------- +// LifecycleOps — binds MQTTLifecycle::Ops (the pure, host-tested spec) to the +// FreeRTOS / PsychicMqttClient runtime. Every method runs on the loop task +// (Core 1): the Coordinator that calls them is driven only from begin()/end(). +// --------------------------------------------------------------------------- +uint32_t MQTTBridge::LifecycleOps::nowMs() { + return (uint32_t)millis(); +} + +void MQTTBridge::LifecycleOps::startTask() { + // No-op: begin() owns task/queue/buffer creation and its rollback paths. The + // Coordinator is synced to Running there via requestStart()/onTaskStarted(). +} + +void MQTTBridge::LifecycleOps::deliverStop() { + // Clear any stale ack before raising the request (same ordering as the NTP + // handshake: clear the done-flag, then set the request). The MQTT task polls + // _stop_requested at the top of mqttTaskLoop(). + _b->_stop_acked = false; + _b->_stop_requested = true; +} + +void MQTTBridge::LifecycleOps::releaseResources() { + MQTTBridge* b = _b; +#ifdef ESP_PLATFORM + // stopTimedOut() is set before this effect fires (Coordinator::dispatch), so + // it reliably distinguishes a clean ack from the timeout fallback. + const bool dirty = b->_lifecycle.stopTimedOut(); + if (dirty && !b->_stop_acked) { + // Reviewed fallback: the task never acknowledged (likely wedged in mbedTLS). + // Force-kill it and tear down clients here on Core 1 — the pre-cooperative + // behavior — accepting the heap risk. The dirty latch keeps OTA flashing + // blocked (canFlashAfterStop() == false) so firmware is never written after + // this path. + if (b->_mqtt_task_handle != nullptr) { + vTaskDelete(b->_mqtt_task_handle); + } + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) b->teardownSlot(i); + b->destroySlotClients(); + } + // Clean path (or a task that acked right at the deadline): the MQTT task + // already disconnected/deleted its clients on Core 0 and self-terminated, so + // we must NOT touch slots here (that would be a cross-core double-delete). + // Just drop our handle reference; FreeRTOS reclaims the self-deleted task's + // dynamically-allocated stack/TCB in the idle task. + b->_mqtt_task_handle = nullptr; + + // Drain and delete the FreeRTOS packet queue (value-copied packets, no + // external pointers to clean up). Safe on Core 1: not a TLS resource. + if (b->_packet_queue_handle != nullptr) { + QueuedPacket queued; + while (xQueueReceive(b->_packet_queue_handle, &queued, 0) == pdTRUE) { + b->_queue_count--; + } + vQueueDelete(b->_packet_queue_handle); + b->_packet_queue_handle = nullptr; + } + #if defined(BOARD_HAS_PSRAM) + psram_free(b->_packet_queue_storage); + #endif + b->_packet_queue_storage = nullptr; +#else + // Non-ESP32 circular buffer + synchronous client teardown (unchanged behavior). + for (int i = 0; i < b->_queue_count; i++) { + int index = (b->_queue_head + i) % MAX_QUEUE_SIZE; + memset(&b->_packet_queue[index], 0, sizeof(QueuedPacket)); + } + b->_queue_count = 0; + b->_queue_head = 0; + b->_queue_tail = 0; + memset(b->_packet_queue, 0, sizeof(b->_packet_queue)); + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) b->teardownSlot(i); + b->destroySlotClients(); +#endif + + b->releaseRuntimeBuffers(); +} + +void MQTTBridge::LifecycleOps::onStopComplete(bool clean) { + MQTT_DEBUG_PRINTLN("MQTT stop %s", clean + ? "acknowledged (clean)" + : "TIMED OUT (dirty; OTA flashing withheld)"); +} + +// --------------------------------------------------------------------------- +// FreeRTOS task entry point +// --------------------------------------------------------------------------- +#ifdef ESP_PLATFORM +void MQTTBridge::mqttTask(void* parameter) { + MQTTBridge* bridge = static_cast(parameter); + if (bridge) { + bridge->mqttTaskLoop(); + } + // Task should never return, but if it does, delete itself + vTaskDelete(nullptr); +} + +void MQTTBridge::initializeWiFiInTask() { + MQTT_DEBUG_PRINTLN("Initializing WiFi in MQTT task..."); + + // Initialize WiFi + WiFi.mode(WIFI_STA); + + // Enable automatic reconnection - ESP32 will handle reconnection automatically + WiFi.setAutoReconnect(true); + WiFi.setAutoConnect(true); + + // Set up WiFi event handlers for better diagnostics and immediate disconnection + // detection. Register ONCE — the bridge is reused across restarts (e.g. stopped + // for `ota check`/`ota update`, or `set mqtt…` reconfigure) and WiFi.onEvent() + // never removes prior callbacks, so re-registering leaks handlers and duplicates + // every log line. + if (!_wifi_event_registered) { + WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { + switch(event) { + case ARDUINO_EVENT_WIFI_STA_GOT_IP: + MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); + // Set flag to trigger NTP sync from loop() instead of doing it here + if (!_ntp_synced && !_ntp_sync_pending) { + _ntp_sync_pending = true; + } + break; + case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: + s_wifi_disconnect_reason = info.wifi_sta_disconnected.reason; + s_wifi_disconnect_time = millis(); + MQTT_DEBUG_PRINTLN("WiFi disconnected: reason %d", s_wifi_disconnect_reason); + break; + default: + break; + } + }); + _wifi_event_registered = true; + } + + // Only (re)start the WiFi association if it isn't already up. end() leaves the + // STA link connected, so on a restart (e.g. after `ota check`) calling + // WiFi.begin() again forces a needless disconnect/reconnect — which also races + // the MQTT task's first DNS lookup (getaddrinfo fails until WiFi/DNS recovers). + // When already connected, the deferred slot setup still fires in mqttTaskLoop() + // because _ntp_synced persists across end() (only _slots_setup_done is reset). + if (WiFi.status() != WL_CONNECTED) { + WiFi.begin(_obs->wifi_ssid, _obs->wifi_password); + } else if (!_ntp_synced && !_ntp_sync_pending) { + _ntp_sync_pending = true; // already connected but never synced — kick NTP now + } + + // NOTE: Slot setup is deferred until after NTP sync in mqttTaskLoop(). + // JWT-auth slots need valid timestamps for token creation, and connecting + // before NTP sync just wastes heap on TLS handshakes that will be rejected. + + MQTT_DEBUG_PRINTLN("WiFi initialization started in task"); +} + +// --------------------------------------------------------------------------- +// mqttTaskLoop() - main loop running on Core 0 +// --------------------------------------------------------------------------- +void MQTTBridge::mqttTaskLoop() { + // Initialize WiFi first + initializeWiFiInTask(); + + // Wait a bit for WiFi to start connecting + vTaskDelay(pdMS_TO_TICKS(1000)); + + // Main task loop + #ifdef MQTT_MEMORY_DEBUG + static unsigned long last_agent_log = 0; + #endif + while (true) { + // Cooperative stop (Phase 5). end() on the loop task (Core 1) set this flag. + // Tear down our own clients HERE on Core 0 — where the mbedTLS/transport + // state lives — instead of letting Core 1 free them after a blind + // vTaskDelete. Acknowledge LAST so end() only frees the queue/buffers once + // this teardown has completed, then self-terminate via the mqttTask() + // trampoline (vTaskDelete(nullptr)). + if (_stop_requested) { + MQTT_DEBUG_PRINTLN("MQTT task: cooperative stop - tearing down clients on Core 0"); + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + teardownSlot(i); + } + destroySlotClients(); + _stop_acked = true; // release semantics: set only after teardown is done + return; + } + + #ifdef MQTT_MEMORY_DEBUG + // #region agent log + unsigned long now_loop = millis(); + if (now_loop - last_agent_log >= 60000) { + last_agent_log = now_loop; + size_t free_h = ESP.getFreeHeap(); + size_t max_alloc = ESP.getMaxAllocHeap(); + unsigned long internal_f = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + unsigned long spiram_f = 0; + #ifdef BOARD_HAS_PSRAM + spiram_f = heap_caps_get_free_size(MALLOC_CAP_SPIRAM); + #endif + agentLogHeap("MQTTBridge.cpp:mqttTaskLoop", "mqtt_loop_60s", "H5", free_h, max_alloc, internal_f, spiram_f); + } + // #endregion + #endif + + unsigned long now = millis(); + + // Periodic heap + publish-health snapshot. Gated behind MQTT_MEMORY_DEBUG (a + // dedicated diagnostics flag, NOT enabled on production or plain MQTT_DEBUG builds) + // so it stays off by default — the same data is available on demand via the + // `get mqtt.stats` CLI command (formatMqttStatsReply / logMemoryStatus()). + #ifdef MQTT_MEMORY_DEBUG + static unsigned long last_mem_log = 0; + if (now - last_mem_log >= 30000) { + last_mem_log = now; + logMemoryStatus(); + } + #endif + + bool wifi_just_connected = handleWiFiConnection(now); + if (wifi_just_connected) { + // WiFi recovered — reset last_reconnect_attempt for disconnected slots so they + // retry immediately rather than waiting up to 5 min for backoff timers to expire. + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].initial_connect_done && !_slots[i].connected) { + _slots[i].last_reconnect_attempt = 0; + } + } + } + + // Check for pending NTP sync (triggered from WiFi event handler) + if (_ntp_sync_pending && WiFi.status() == WL_CONNECTED) { + _ntp_sync_pending = false; + syncTimeWithNTP(); + } + + // Retry NTP every 30s if initial sync failed (slots can't start without valid time) + if (!_ntp_synced && WiFi.status() == WL_CONNECTED) { + static unsigned long last_ntp_retry = 0; + if (now - last_ntp_retry >= 30000) { + last_ntp_retry = now; + syncTimeWithNTP(); + } + } + + // Process a CLI-requested forced NTP sync (queued from Core 1). Running it here + // keeps all NTP I/O on Core 0; requestForcedNtpSync() blocks the CLI thread until + // we publish the outcome below. + if (_ntp_force_requested) { + _ntp_force_requested = false; + // primary_only: validate just the server that was set, so a typo fails fast. + bool ok = syncTimeWithNTP(true, /*primary_only=*/true); + _ntp_force_result = ok; + _ntp_force_done = true; // set last so the waiter sees a consistent result + } + + // Process a CLI-requested NTP connectivity diagnostic (queued from Core 1). + // Probe-only — never touches the system clock. + if (_ntp_diag_requested) { + _ntp_diag_requested = false; + runNtpDiagProbe(); + _ntp_diag_done = true; // set last so the waiter sees populated results + } + + // Deferred slot setup: wait until NTP is synced so JWT tokens get valid timestamps. + // This avoids wasted TLS handshakes that get rejected due to bad token times. + if (_ntp_synced && !_slots_setup_done) { + _slots_setup_done = true; + + // Redirect mbedTLS allocations to PSRAM to save ~40KB internal heap per TLS connection. + // This is critical when running 3 concurrent WSS connections. + #if defined(BOARD_HAS_PSRAM) + mbedtls_platform_set_calloc_free(psram_calloc, psram_free); + MQTT_DEBUG_PRINTLN("mbedTLS allocator redirected to PSRAM"); + #endif + + MQTT_DEBUG_PRINTLN("NTP synced, setting up MQTT slots (max %d active)...", _max_active_slots); + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled) { + if (!canActivateSlot(i)) { + MQTT_DEBUG_PRINTLN("MQTT%d skipped: max active slots (%d) reached", i + 1, _max_active_slots); + _slots[i].enabled = false; // Disable so other loops skip it + continue; + } + char reason[80]; + if (!isSlotReady(i, reason, sizeof(reason))) { + MQTT_DEBUG_PRINTLN("MQTT%d not ready - run '%s' to connect", i + 1, reason); + continue; + } + // A slot that fails to activate consumes no position and stays enabled, so + // maintainSlotConnections() retries it and a later healthy broker is not + // starved by it on a capped board. + if (!setupSlot(i)) continue; + // Stagger connections: 5s between slots to avoid simultaneous TLS handshakes + // which compete for ~40KB internal heap each + if (i < RUNTIME_MQTT_SLOTS - 1) { + vTaskDelay(pdMS_TO_TICKS(5000)); + } + } + } + } + + // Process pending slot reconfigures (queued from CLI on Core 1) + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slot_reconfigure_pending[i]) { + _slot_reconfigure_pending[i] = false; + MQTT_DEBUG_PRINTLN("Applying deferred reconfigure for MQTT%d (preset: %s)", i + 1, _obs->mqtt_slot_preset[i]); + applySlotPreset(i, _obs->mqtt_slot_preset[i]); + } + } + + // Publish on-connect status for slots whose onConnect callback fired since + // the last loop. Raised on the esp-mqtt event task, consumed here on the + // bridge task so the shared status doc/buffer/origin are only ever touched + // from Core 0 (see the onConnect handler / A2). Clear before publishing so a + // reconnect during the publish re-arms for the next loop rather than being + // lost; publishStatusToSlot() re-checks slot.connected and no-ops if dropped. + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_status_publish_pending[i]) { + _status_publish_pending[i] = false; + publishStatusToSlot(i); + } + } + + // Maintain slot connections (token renewal, reconnect with backoff) + maintainSlotConnections(); + + // Process packet queue + processPacketQueue(); + +#if defined(WITH_MQTT_NEIGHBORS) + // Consume a pending neighbors snapshot handed over by the mesh (Core 1). + // The pending flag stays raised across the whole publish so a second + // request is rejected until this one completes (see requestPublishNeighbors). + if (_neighbors_publish_pending.load(std::memory_order_acquire)) { + bool ok = publishNeighbors(); + _neighbors_last_result.store(ok ? NBR_RESULT_OK : NBR_RESULT_FAIL, + std::memory_order_relaxed); + // MQTT_DEBUG_PRINTLN concatenates its format as a string literal, so the + // argument must be a literal, not a ternary expression. + if (ok) { + MQTT_DEBUG_PRINTLN("Neighbors published"); + } else { + MQTT_DEBUG_PRINTLN("Neighbors publish failed"); + } + _neighbors_publish_pending.store(false, std::memory_order_release); + } +#endif + +#ifdef WITH_SNMP + // SNMP agent loop — process incoming UDP requests + if (_snmp_agent) { + if (!_snmp_agent->isRunning() && WiFi.isConnected() && _obs->snmp_enabled) { + _snmp_agent->begin(_obs->snmp_community); + MQTT_DEBUG_PRINTLN("SNMP agent started on port 161 (community: %s)", _obs->snmp_community); + } + if (_snmp_agent->isRunning()) { + // Update MQTT stats from this core + int connected = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].connected) connected++; + } + _snmp_agent->updateMQTTStats(connected, _queue_count, _skipped_publishes); + _snmp_agent->loop(); + } + } +#endif + + // Periodic configuration check (throttled to avoid spam) + checkConfigurationMismatch(); + + // Periodic NTP refresh (every hour) — lightweight, non-blocking. + // Uses async SNTP instead of the heavy syncTimeWithNTP() which blocks Core 0 + // for up to 20+ seconds with DNS lookups, UDP sockets, and retry loops. + if (WiFi.status() == WL_CONNECTED && now - _last_ntp_sync > 3600000) { + refreshNTP(); + } + + // Publish status updates (handle millis() overflow correctly). + // Read the toggle live from prefs (like mqtt.packets/rx/tx below) so a + // CLI/web `set mqtt.status` change applies without a bridge restart. + if (_obs->mqtt_status_enabled) { + bool has_destinations = _cached_has_connected_slots; + + // Early exit if no destinations - skip all the expensive logic below + if (!has_destinations) { + if (_last_status_retry != 0) { + _last_status_retry = 0; + } + } else { + bool should_publish = false; + + // First, check if we need to respect retry interval (prevents spam when publish keeps failing) + if (_last_status_retry != 0) { + unsigned long retry_elapsed = (now >= _last_status_retry) ? + (now - _last_status_retry) : + (ULONG_MAX - _last_status_retry + now + 1); + if (retry_elapsed < STATUS_RETRY_INTERVAL) { + should_publish = false; + } else { + should_publish = true; + } + } else { + if (_last_status_publish == 0) { + should_publish = true; + } else { + unsigned long elapsed = (now >= _last_status_publish) ? + (now - _last_status_publish) : + (ULONG_MAX - _last_status_publish + now + 1); + should_publish = (elapsed >= _status_interval); + } + } + + if (should_publish) { + if (_last_status_publish != 0) { + unsigned long elapsed = (now >= _last_status_publish) ? + (now - _last_status_publish) : + (ULONG_MAX - _last_status_publish + now + 1); + MQTT_DEBUG_PRINTLN("Status publish timer expired (elapsed: %lu ms, interval: %lu ms)", elapsed, _status_interval); + } else { + MQTT_DEBUG_PRINTLN("Status publish attempt (first publish or retry)"); + } + + _last_status_retry = now; + if (publishStatus()) { + _last_status_publish = now; + _last_status_retry = 0; + MQTT_DEBUG_PRINTLN("Status published successfully, next publish in %lu ms", _status_interval); + } else { + MQTT_DEBUG_PRINTLN("Status publish failed, will retry in %lu ms", STATUS_RETRY_INTERVAL); + } + } + } + } + + // Update cached connection status periodically (every 5 seconds) + // This ensures cache stays accurate even if callbacks miss updates + static unsigned long last_slot_status_update = 0; + if (now - last_slot_status_update > 5000) { + updateCachedConnectionStatus(); + last_slot_status_update = now; + } + + // Adaptive delay: 5 ms when packets are queued, 50 ms when idle. + // The previous "status approaching" check (widening to 5 ms for 10 s before each status + // publish) caused 2 000 unnecessary wakeups per interval; the 50 ms idle tick catches + // the status deadline with at most 50 ms of extra latency, which is irrelevant at a + // 5-minute interval. + vTaskDelay(pdMS_TO_TICKS(_queue_count > 0 ? 5 : 50)); + } +} +#endif + +// --------------------------------------------------------------------------- +// Slot management +// --------------------------------------------------------------------------- + +// Allocate this slot's PsychicMqttClient and register its persistent callbacks. +// Called from setupSlot(), i.e. only for a slot that is enabled, within the active +// cap, and ready to connect — a client is ~1.3 KB of internal DRAM and does nothing +// at all until setupSlot() runs (the reconnect ladder is gated on +// initial_connect_done), so slots that are unconfigured or capped off never get one. +// +// Once created the object lives until destroySlotClients(): reconfiguring a slot +// (preset change, JWT renewal, reconnect) reuses it, so the mbedTLS context and its +// ~40 KB of internal-heap buffers are allocated once instead of every reconfigure. +// That context is created by connect(), not by this constructor, so deferring the +// allocation to first use costs nothing beyond the object itself. +bool MQTTBridge::ensureSlotClient(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + MQTTSlot& slot = _slots[index]; + if (slot.client != nullptr) return true; + + // nothrow: this framework builds with C++ exceptions enabled, so a plain new would + // throw on exhaustion and panic the node. A slot that cannot get a client should + // degrade to the "no client" diag state instead. + slot.client = new (std::nothrow) PsychicMqttClient(); + if (slot.client == nullptr) { + MQTT_DEBUG_PRINTLN("MQTT%d: out of memory allocating client", index + 1); + return false; + } + slot.client->setAutoReconnect(false); // we handle reconnect with our own backoff + + slot.client->onConnect([this, index](bool sessionPresent) { + MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1); + _slots[index].connected = true; + // NOTE: reconnect_backoff / max_backoff_failures are NOT reset here. + // A CONNACK alone doesn't prove the link is healthy — a broker that + // accepts and then drops within seconds would reset the ladder every + // cycle and retry at the 10 s rung forever, and each retry is a full + // TLS session alloc/free (~40 KB of internal-heap churn, a known + // fragmentation driver). The ladder is instead cleared by + // maintainSlotConnection() once the connection has stayed up for + // BACKOFF_STABLE_RESET_MS, so flapping endpoints keep their earned + // backoff level. The breaker itself does clear now: while connected + // the diag/status must not claim the slot gave up, and the next + // disconnect should be governed by the (still-elevated) ladder. + _slots[index].connected_at_ms = millis(); + _slots[index].circuit_breaker_tripped = false; + _slots[index].last_tls_err = 0; + _slots[index].last_tls_stack_err = 0; + _slots[index].last_sock_errno = 0; + _slots[index].last_error_time = 0; + _slots[index].current_outage_started_ms = 0; // clear current-outage timer for AlertReporter + updateCachedConnectionStatus(); // bool store — safe from this (esp-mqtt) task + // This callback runs on the client's esp-mqtt event task, not the bridge + // task. Do NOT build/publish status here: publishStatusToSlot() writes the + // shared _json_scratch_doc/_json_scratch_buffer/_origin that the periodic + // publishStatus() uses on the bridge task, and two slots' callbacks could + // race each other over them. Marshal the publish onto the bridge task via a + // per-slot flag (see mqttTaskLoop consumer / A2). + _status_publish_pending[index] = true; + }); + slot.client->onDisconnect([this, index](bool sessionPresent) { + MQTT_DEBUG_PRINTLN("MQTT%d disconnected", index + 1); + _slots[index].disconnect_count++; + if (_slots[index].first_disconnect_time == 0) { + _slots[index].first_disconnect_time = millis(); + } + if (_slots[index].current_outage_started_ms == 0) { + _slots[index].current_outage_started_ms = millis(); + } + _slots[index].connected = false; + _slots[index].connected_at_ms = 0; // stability clock only runs while connected + updateCachedConnectionStatus(); + }); + slot.client->onError([this, index](esp_mqtt_error_codes error) { + _slots[index].last_tls_err = error.esp_tls_last_esp_err; + _slots[index].last_tls_stack_err = error.esp_tls_stack_err; + _slots[index].last_sock_errno = error.esp_transport_sock_errno; + _slots[index].last_error_time = millis(); + if (error.error_type == MQTT_ERROR_TYPE_CONNECTION_REFUSED) { + // Broker rejected the MQTT CONNECT itself — not a transport failure. + // return code: 1=protocol, 2=client-id rejected, 3=server unavailable, + // 4=bad username/password, 5=not authorized. Codes 3/4/5 point at a + // server-side lockout or auth problem rather than the network. + MQTT_DEBUG_PRINTLN("MQTT%d connection refused by broker (return code=%d)", + index + 1, (int)error.connect_return_code); + } else if (error.esp_tls_last_esp_err != 0 || error.esp_tls_stack_err != 0 || error.esp_transport_sock_errno != 0) { + MQTT_DEBUG_PRINTLN("MQTT%d error: tls=%d, tls_stack=%d, sock=%d, type=%d", + index + 1, error.esp_tls_last_esp_err, error.esp_tls_stack_err, + error.esp_transport_sock_errno, error.error_type); + } else { + MQTT_DEBUG_PRINTLN("MQTT%d error: type=%d", index + 1, error.error_type); + } + }); + return true; +} + +// Allocate this slot's JWT token buffer. Called only from createSlotAuthToken(), the +// sole writer, so a slot on a non-JWT preset (or no preset at all) never allocates. +// +// PSRAM where the board has it (psram_malloc falls back to internal DRAM otherwise), +// which is what moves the token off internal heap for slots that DO use JWT. Safe +// because the only readers are CPU copies on the bridge task: JWTHelper memcpy's the +// token in here, and esp-mqtt copies it out of _mqtt_cfg into its own internal-DRAM +// storage when connect() applies the config. No DMA, no ISR, and no cache-disabled +// window -- unlike the PSRAM task stack that reset Heltec V4 boards. +bool MQTTBridge::ensureSlotAuthToken(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + MQTTSlot& slot = _slots[index]; + const bool fresh = (slot.auth_token == nullptr); + slot.auth_token = static_cast(MQTTRuntimeBufferLifecycle::allocateIfMissing( + slot.auth_token, AUTH_TOKEN_SIZE, psram_malloc)); + if (slot.auth_token == nullptr) { + MQTT_DEBUG_PRINTLN("MQTT%d: out of memory allocating auth token", index + 1); + return false; + } + // Initialise only a newly allocated buffer. Clearing on every call would discard a + // valid token at the start of each renewal, so a renewal that then failed inside + // JWTHelper would leave the slot with an empty password where it previously kept + // working credentials (JWTHelper writes the token only on success). + if (fresh) slot.auth_token[0] = '\0'; + return true; +} + +// Safe only once this slot's client is gone: setCredentials() gave the client this +// pointer, and esp-mqtt re-reads it from _mqtt_cfg on any later connect() that +// re-applies a dirtied config. See the MQTTSlot::auth_token comment. +void MQTTBridge::releaseSlotAuthToken(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; + MQTTSlot& slot = _slots[index]; + slot.auth_token = static_cast( + MQTTRuntimeBufferLifecycle::release(slot.auth_token, psram_free)); + slot.token_expires_at = 0; + slot.last_token_renewal = 0; +} + +void MQTTBridge::destroySlotClients() { + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + MQTTSlot& slot = _slots[i]; + if (slot.client != nullptr) { + if (slot.client->connected()) { + slot.client->disconnect(); + } + #ifdef ESP_PLATFORM + vTaskDelay(pdMS_TO_TICKS(50)); + #else + delay(50); + #endif + delete slot.client; + slot.client = nullptr; + } + // Unconditional: only now is the token unreachable from the client's stored + // config, and a token without a client would otherwise leak. + releaseSlotAuthToken(i); + } +} + +int MQTTBridge::activatedSlotCount() const { + int n = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].initial_connect_done) n++; + } + return n; +} + +bool MQTTBridge::canActivateSlot(int index) const { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + // Already holding a position (a reconfigure of a live slot) — no new position needed. + if (_slots[index].enabled && _slots[index].initial_connect_done) return true; + return activatedSlotCount() < _max_active_slots; +} + +// Returns true only when the slot reached connect(). A false result leaves the slot +// enabled but not activated, so it holds no active-slot position and +// maintainSlotConnections() will retry it — the allocation failures below are transient +// memory conditions, not permanent misconfiguration. +bool MQTTBridge::setupSlot(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + MQTTSlot& slot = _slots[index]; + + if (!slot.enabled) { + teardownSlot(index); + return false; + } + + // Every failure below is a real attempt, so stamp it: the retry interval in + // maintainSlotConnections() measures from last_reconnect_attempt, which starts at 0 + // and is re-zeroed by teardownSlot(). Left unstamped, the gate degenerates to + // "uptime >= SLOT_SETUP_RETRY_INTERVAL" and a failure past that point is retried on + // the very next maintenance pass — the same task iteration, for a live reconfigure. + // The reconnect ladder never reads this field for an unactivated slot (it is gated + // on initial_connect_done), so stamping here cannot perturb reconnect timing. + + // First setup for this slot allocates its persistent client; later ones reuse it. + if (!ensureSlotClient(index)) { + MQTT_DEBUG_PRINTLN("MQTT%d: client allocation failed - will retry", index + 1); + slot.last_reconnect_attempt = millis(); + return false; + } + + // Reconfigure path: if we're re-applying (e.g. after a preset change), stop + // the existing connection cleanly first. The client object (and its mbedTLS + // context) is reused; setCredentials / setServer below overwrite the config + // fields in place before connect() restarts the ESP-IDF client. + if (slot.initial_connect_done) { + if (slot.client->connected()) { + slot.client->disconnect(); + } + // Clear TLS verification fields so a stale CA-bundle attach or cert + // pointer from a prior preset doesn't override the new one. + esp_mqtt_client_config_t* cfg = slot.client->getMqttConfig(); + #if ESP_IDF_VERSION_MAJOR == 5 + cfg->broker.verification.certificate = nullptr; + cfg->broker.verification.certificate_len = 0; + cfg->broker.verification.crt_bundle_attach = nullptr; + cfg->credentials.username = nullptr; + cfg->credentials.authentication.password = nullptr; + #else + cfg->cert_pem = nullptr; + cfg->cert_len = 0; + cfg->crt_bundle_attach = nullptr; + cfg->username = nullptr; + cfg->password = nullptr; + #endif + if (slot.auth_token) slot.auth_token[0] = '\0'; + slot.connected = false; + slot.token_expires_at = 0; + slot.last_token_renewal = 0; + slot.reconnect_backoff = 0; + slot.max_backoff_failures = 0; + slot.circuit_breaker_tripped = false; + slot.last_reconnect_attempt = 0; + } + + bool uses_jwt = (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT) || slot.audience[0] != '\0'; + optimizeMqttClientConfig(slot.client, uses_jwt); // sets keepalive (45s PSRAM, 75s non-PSRAM) + #ifndef MQTT_FORCE_KEEPALIVE_45 + #if defined(BOARD_HAS_PSRAM) + if (slot.preset && slot.preset->keepalive > 0) { + slot.client->setKeepAlive(slot.preset->keepalive); // preset overrides default + } + #else + // Non-PSRAM: keep the longer 75s default to reduce TLS churn. + // Preset keepalive (55s) is more aggressive than needed behind Cloudflare. + #endif + #endif + + if (slot.preset) { + // Preset-based slot + slot.client->setServer(slot.preset->server_url); + if (slot.preset->ca_cert) { + slot.client->setCACert(slot.preset->ca_cert); + } + + // A JWT slot with no usable token would connect unauthenticated and be rejected. + // Stay unactivated instead, so the retry path tries again — the failure is either + // a transient token-buffer allocation or a JWTHelper error, not a config problem. + if (slot.preset->auth_type == MQTT_AUTH_JWT) { + if (!createSlotAuthToken(index) || !slot.auth_token || slot.auth_token[0] == '\0') { + MQTT_DEBUG_PRINTLN("MQTT%d: no usable JWT token - will retry", index + 1); + slot.last_reconnect_attempt = millis(); + return false; + } + slot.client->setCredentials(_jwt_username, slot.auth_token); + } else if (slot.preset->auth_type == MQTT_AUTH_USERPASS) { + const char* user = nullptr; + const char* pass = slot.preset->userpass_password + ? slot.preset->userpass_password + : slot.password; + if (mqttPresetUsesDevicePubkeyUsername(slot.preset)) { + user = _device_id; // never send "{pubkey}" literally + } else if (slot.preset->userpass_username) { + user = slot.preset->userpass_username; + } else if (slot.username[0] != '\0') { + user = slot.username; + } + if (user && user[0] != '\0' && pass && pass[0] != '\0') { + slot.client->setCredentials(user, pass); + } + } + } else { + // Custom broker slot — build persistent URI + // If host already has a scheme (mqtt://, mqtts://, ws://, wss://), preserve the full URI + // (including optional path/query) and only inject :port when the authority has no explicit port. + // Otherwise, infer protocol from port number. + bool has_scheme = (strncmp(slot.host, "mqtt://", 7) == 0 || + strncmp(slot.host, "mqtts://", 8) == 0 || + strncmp(slot.host, "ws://", 5) == 0 || + strncmp(slot.host, "wss://", 6) == 0); + if (has_scheme) { + const char* authority = strstr(slot.host, "://"); + authority = authority ? authority + 3 : slot.host; + const char* path = strchr(authority, '/'); + const char* authority_end = path ? path : slot.host + strlen(slot.host); + bool has_explicit_port = false; + + // Detect host:port in URI authority (IPv6 literals in [addr]:port are supported). + if (authority < authority_end) { + if (*authority == '[') { + const char* close = (const char*)memchr(authority, ']', authority_end - authority); + if (close && (close + 1) < authority_end && *(close + 1) == ':') { + has_explicit_port = true; + } + } else { + const char* colon = (const char*)memchr(authority, ':', authority_end - authority); + if (colon != nullptr) { + has_explicit_port = true; + } + } + } + + if (has_explicit_port || slot.port == 0) { + snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s", slot.host); + } else { + const size_t authority_len = (size_t)(authority_end - slot.host); + snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%.*s:%u%s", + (int)authority_len, + slot.host, + (unsigned)slot.port, + path ? path : ""); + } + } else { + const char* proto = "mqtt"; + if (slot.port == 8883) { + proto = "mqtts"; + } else if (slot.port == 443) { + proto = "wss"; + } + snprintf(slot.broker_uri, sizeof(slot.broker_uri), "%s://%s:%d", proto, slot.host, slot.port); + } + slot.client->setServer(slot.broker_uri); + MQTT_DEBUG_PRINTLN("MQTT%d custom broker URI: %s (host='%s', port=%u)", + index + 1, slot.broker_uri, slot.host, (unsigned)slot.port); + + // Custom TLS/WSS slots need a CA bundle for server verification. + // The bundle is loaded into the global s_crt_bundle exactly once to avoid + // a use-after-free race: connect() launches an async FreeRTOS task, and + // calling setCACertBundle() again from a later slot would free the global + // crts array while a prior slot's TLS handshake may still be reading it. + bool needs_tls = (strncmp(slot.broker_uri, "mqtts://", 8) == 0 || + strncmp(slot.broker_uri, "wss://", 6) == 0); + if (needs_tls) { + if (!s_ca_bundle_loaded) { + size_t bundle_len = 0; + if (rootca_crt_bundle_start != nullptr && + rootca_crt_bundle_end != nullptr && + rootca_crt_bundle_end > rootca_crt_bundle_start) { + bundle_len = static_cast(rootca_crt_bundle_end - rootca_crt_bundle_start); + } + + if (bundle_len > 0) { + MQTT_DEBUG_PRINTLN("MQTT global CA bundle init: embedded bundle (%u bytes)", + (unsigned)bundle_len); + // Load the bundle into the global s_crt_bundle via the first client. + // This is a one-time operation; subsequent clients reuse via attachArduinoCACertBundle. + slot.client->setCACertBundle(rootca_crt_bundle_start, bundle_len); + s_ca_bundle_loaded = true; + } else { + MQTT_DEBUG_PRINTLN("MQTT%d TLS: no embedded cert bundle available", index + 1); + } + } else { + // Global bundle already loaded — just attach the callback for this client. + slot.client->attachArduinoCACertBundle(true); + } + MQTT_DEBUG_PRINTLN("MQTT%d TLS verify: CA bundle %s", index + 1, + s_ca_bundle_loaded ? "active" : "unavailable"); + } else { + MQTT_DEBUG_PRINTLN("MQTT%d custom broker uses non-TLS transport", index + 1); + } + + // Custom slot authentication: JWT if audience is set, else username/password + if (slot.audience[0] != '\0') { + // JWT auth for custom slot — same rule as the preset JWT path above. + if (!createSlotAuthToken(index) || !slot.auth_token || slot.auth_token[0] == '\0') { + MQTT_DEBUG_PRINTLN("MQTT%d: no usable JWT token - will retry", index + 1); + slot.last_reconnect_attempt = millis(); + return false; + } + slot.client->setCredentials(_jwt_username, slot.auth_token); + MQTT_DEBUG_PRINTLN("MQTT%d custom broker using JWT auth (audience: %s)", index + 1, slot.audience); + } else if (strlen(slot.username) > 0) { + slot.client->setCredentials(slot.username, slot.password); + } + } + + slot.client->connect(); + slot.initial_connect_done = true; + return true; +} + +// Disconnect the slot's MQTT client and clear per-connection state, but leave +// the client object alive so a subsequent setupSlot() can reuse its mbedTLS +// context. This is called both on reconfigure (preset change) and at shutdown; +// destruction of the underlying client happens once in destroySlotClients(). +void MQTTBridge::teardownSlot(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; + MQTTSlot& slot = _slots[index]; + + if (slot.client && slot.client->connected()) { + slot.client->disconnect(); + #ifdef ESP_PLATFORM + vTaskDelay(pdMS_TO_TICKS(50)); + #else + delay(50); + #endif + } + + // Invalidate the token but keep the buffer: the client survives teardown and still + // holds this pointer in its config (see MQTTSlot::auth_token). + if (slot.auth_token) slot.auth_token[0] = '\0'; + slot.connected = false; + slot.initial_connect_done = false; + slot.broker_uri[0] = '\0'; + slot.token_expires_at = 0; + slot.last_token_renewal = 0; + slot.reconnect_backoff = 0; + slot.max_backoff_failures = 0; + slot.circuit_breaker_tripped = false; + slot.last_reconnect_attempt = 0; + slot.last_log_time = 0; + slot.last_deferred_log_ms = 0; +} + +void MQTTBridge::maintainSlotConnections() { + if (!_identity) return; + + // Check WiFi status first + if (WiFi.status() != WL_CONNECTED) return; + + unsigned long now_millis = millis(); + unsigned long current_time = time(nullptr); + bool time_synced = (current_time >= 1000000000); // After year 2001 + + // JWT tokens require valid timestamps + unsigned long clock_sec = current_time; + bool can_do_jwt = MQTTConnectionPolicy::jwtClockAvailable( + _ntp_synced, static_cast(clock_sec)); + + // Count connected slots to inform reconnect decisions + int connected_count = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].connected) connected_count++; + } + + // Only allow one reconnect attempt per maintenance cycle to avoid + // multiple simultaneous TLS handshakes blocking the network stack. + // Time-based guard: block reconnects if any slot reconnected within the last 15 s, + // ensuring the previous TLS handshake (and its Core-0-expensive completion events) + // finish before the next slot begins its own handshake. + bool reconnect_attempted_this_cycle = MQTTConnectionPolicy::reconnectGuardActive( + static_cast(now_millis), static_cast(_last_slot_reconnect_ms)); + // Only allow one full teardown+setup per cycle to limit heap fragmentation + // when multiple slots fail simultaneously + bool teardown_attempted_this_cycle = false; + + // At most one deferred setup retry per cycle: a successful one ends in connect(), so + // this shares the "no simultaneous TLS handshakes" rule the reconnect guard enforces. + bool setup_retry_this_cycle = false; + + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (!_slots[i].enabled) continue; + + // JWT slots need time sync before we can manage tokens + bool slot_jwt = (_slots[i].preset && _slots[i].preset->auth_type == MQTT_AUTH_JWT) || + (!_slots[i].preset && _slots[i].audience[0] != '\0'); + if (slot_jwt && !can_do_jwt) { + continue; + } + + // Enabled but never activated: setupSlot() failed on a client or token allocation, + // or on token creation. The ladder below is gated on initial_connect_done and would + // never revisit it, and maintenance used to skip clientless slots entirely, so + // without this the slot stayed dead until a reconfigure or reboot. Only retried + // after the initial pass has run, so the NTP-deferred setup order is preserved. + if (!_slots[i].initial_connect_done) { + if (_slots_setup_done && !setup_retry_this_cycle && !reconnect_attempted_this_cycle && + isSlotReady(i) && canActivateSlot(i) && + MQTTConnectionPolicy::elapsedMs(static_cast(now_millis), + static_cast(_slots[i].last_reconnect_attempt)) + >= SLOT_SETUP_RETRY_INTERVAL) { + _slots[i].last_reconnect_attempt = now_millis; + setup_retry_this_cycle = true; + MQTT_DEBUG_PRINTLN("MQTT%d retrying deferred setup (int_heap=%d)", i + 1, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + if (setupSlot(i)) { + // A successful setup ends in connect(), so it spends this cycle's single + // handshake allowance as well as arming the 15 s cross-slot guard. Without + // the local flag, a disconnected slot later in this same pass would start a + // second concurrent TLS handshake — the contention the guard exists to + // prevent, and most damaging here because a failed allocation is why we are + // retrying at all. A failed setup launches nothing and so spends only + // setup_retry_this_cycle. + _last_slot_reconnect_ms = now_millis; + reconnect_attempted_this_cycle = true; + } + } + continue; + } + if (!_slots[i].client) continue; + + maintainSlotConnection(i, now_millis, current_time, time_synced, reconnect_attempted_this_cycle, teardown_attempted_this_cycle); + } +} + +void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted) { + MQTTSlot& slot = _slots[index]; + + // Forgive past failures only after the connection has proven stable. + // 2 minutes covers at least one keepalive round-trip (keepalive is 75 s), + // so a link that can't survive a single keepalive period never resets the + // ladder. Flapping endpoints therefore stay at their earned backoff rung + // (worst case the 300 s rung / 30-minute breaker probes) instead of + // hammering full TLS handshakes at the 10 s rung — see the onConnect + // handler in ensureSlotClient() for why this doesn't happen on CONNACK. + if (slot.connected && + (slot.reconnect_backoff != 0 || slot.max_backoff_failures != 0) && + MQTTConnectionPolicy::stableConnection(static_cast(now_millis), + static_cast(slot.connected_at_ms))) { + MQTT_DEBUG_PRINTLN("MQTT%d stable for %lus - clearing reconnect backoff (was level %d)", + index + 1, (now_millis - slot.connected_at_ms) / 1000UL, slot.reconnect_backoff); + slot.reconnect_backoff = 0; + slot.max_backoff_failures = 0; + } + + // JWT token renewal (for preset JWT slots and custom slots with audience set) + bool slot_uses_jwt = (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT) || + (!slot.preset && slot.audience[0] != '\0'); + if (slot_uses_jwt) { + // Renew (and below, reconnect) this many seconds before the token's exp + // claim. Scaled to the slot's token lifetime — see renewalBufferSecs() + // for why a flat 60 s lost the renewal race against brokers that enforce + // exp on live sessions (waev's 55-minute tokens). + const unsigned long renewal_buffer = MQTTConnectionPolicy::renewalBufferSecs( + static_cast(slotTokenLifetime(index))); + bool token_needs_renewal = MQTTConnectionPolicy::tokenNeedsRenewal( + time_synced, static_cast(current_time), + static_cast(slot.token_expires_at), + static_cast(renewal_buffer)); + + // Throttle renewal attempts to once per minute + bool can_attempt_renewal = MQTTConnectionPolicy::renewalAttemptAllowed( + static_cast(now_millis), static_cast(slot.last_token_renewal)); + + if (token_needs_renewal && can_attempt_renewal) { + slot.last_token_renewal = now_millis; + + unsigned long old_token_expires_at = slot.token_expires_at; + + if (createSlotAuthToken(index)) { + MQTT_DEBUG_PRINTLN("MQTT%d token renewed", index + 1); + + // Bounce the connection while WE control the timing whenever the old + // token is inside the renewal buffer — waiting for the broker to + // enforce exp mid-session means a FIN plus a trip through the backoff + // ladder instead of one clean reconnect. Same buffer as the renewal + // trigger above, so a renewal implies a proactive reconnect. + bool old_token_expired_or_imminent = !time_synced || + (old_token_expires_at == 0) || + (current_time >= old_token_expires_at) || + (time_synced && old_token_expires_at >= 1000000000 && + current_time >= (old_token_expires_at - renewal_buffer)); + + if (old_token_expired_or_imminent || !slot.client->connected()) { + // Disconnect + reconnect with fresh credentials, reusing existing client + // to avoid internal heap leak/fragmentation from destroy/create cycles + MQTT_DEBUG_PRINTLN("MQTT%d token renewal: reconnecting with fresh credentials", index + 1); + if (slot.client->connected()) { + slot.client->disconnect(); // stops the client internally + } + slot.client->setCredentials(_jwt_username, slot.auth_token); + slot.client->connect(); // restart stopped client; reconnect() fails silently on a stopped client + reconnect_attempted = true; + _last_slot_reconnect_ms = now_millis; + MQTT_DEBUG_PRINTLN("MQTT%d int_heap=%d at token renewal reconnect", index + 1, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); + } else { + // Token renewed but old one still valid — just update credentials for next reconnect + slot.client->setCredentials(_jwt_username, slot.auth_token); + } + } else { + MQTT_DEBUG_PRINTLN("MQTT%d token renewal failed", index + 1); + slot.token_expires_at = 0; + } + return; // Token renewal handled connect; skip backoff logic below + } + } + + // Phase 4 (MQTT memory-defrag): the MIN_TLS_HEAP preflight was a workaround + // for the fragmentation caused by per-reconnect mbedTLS allocations. With + // persistent clients (Phase 1), the mbedTLS context is allocated once at + // startup and the preflight is no longer necessary. + + // Periodic probe for circuit-breaker-tripped slots (recovery from transient outages) + // Attempts a single reconnect every 30 minutes to see if the server has come back + if (slot.circuit_breaker_tripped && !reconnect_attempted) { + unsigned long probe_elapsed = MQTTConnectionPolicy::elapsedMs( + static_cast(now_millis), static_cast(slot.last_reconnect_attempt)); + if (MQTTConnectionPolicy::circuitBreakerProbeDue( + static_cast(now_millis), static_cast(slot.last_reconnect_attempt))) { + slot.last_reconnect_attempt = now_millis; + reconnect_attempted = true; + _last_slot_reconnect_ms = now_millis; + MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (attempting single reconnect after %lu ms, int_heap=%d)", index + 1, probe_elapsed, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); + if (slot_uses_jwt) { + // Regenerate or refresh token, then reconnect the persistent client. + // Reaching the ladder at all means setupSlot() ran, so the client object + // and its mbedTLS context are live and no full setup is needed here. + if (createSlotAuthToken(index)) { + slot.client->setCredentials(_jwt_username, slot.auth_token); + MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (fresh token)", index + 1); + } + slot.client->reconnect(); + } else { + slot.client->reconnect(); + } + // If the connect callback fires and sets slot.connected = true, + // it will clear circuit_breaker_tripped via the onConnect handler + } + } + + // Reconnect with exponential backoff (for disconnected slots that already have valid config) + // Only one reconnect per maintenance cycle to prevent TLS handshakes from blocking other slots + if (!slot.connected && slot.initial_connect_done && !slot.circuit_breaker_tripped && !reconnect_attempted) { + if (MQTTConnectionPolicy::reconnectDue( + static_cast(now_millis), static_cast(slot.last_reconnect_attempt), + slot.reconnect_backoff, static_cast(index))) { + slot.last_reconnect_attempt = now_millis; + MQTTConnectionPolicy::BackoffAdvance advance = MQTTConnectionPolicy::advanceBackoff( + slot.reconnect_backoff, slot.max_backoff_failures); + slot.reconnect_backoff = advance.reconnect_backoff; + slot.max_backoff_failures = advance.max_backoff_failures; + slot.circuit_breaker_tripped = advance.circuit_breaker_tripped; + if (!advance.should_reconnect) { + MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker tripped after %d failures at max backoff - stopping reconnect attempts. Reconfigure slot to retry.", index + 1, slot.max_backoff_failures); + return; + } + MQTT_DEBUG_PRINTLN("MQTT%d reconnecting (backoff level %d, failures at max: %d, int_heap=%d)", index + 1, slot.reconnect_backoff, slot.max_backoff_failures, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); + reconnect_attempted = true; + _last_slot_reconnect_ms = now_millis; + if (slot_uses_jwt) { + // Always lightweight reconnect on the persistent client. A stale/expired + // token is handled by regenerating it in place and updating credentials + // — no teardown is needed because the client and its mbedTLS context + // persist for the bridge lifetime. + if (createSlotAuthToken(index)) { + slot.client->setCredentials(_jwt_username, slot.auth_token); + MQTT_DEBUG_PRINTLN("MQTT%d reconnect (fresh token, backoff %d)", index + 1, slot.reconnect_backoff); + } else { + MQTT_DEBUG_PRINTLN("MQTT%d reconnect (token refresh failed, backoff %d)", index + 1, slot.reconnect_backoff); + } + slot.client->reconnect(); + } else { + // Non-JWT slots — lightweight reconnect on existing client. + MQTT_DEBUG_PRINTLN("MQTT%d reconnect (non-JWT, backoff %d)", index + 1, slot.reconnect_backoff); + slot.client->reconnect(); + } + } + } +} + +// Effective JWT lifetime for a slot: the preset's token_lifetime (or the 24 h +// default for custom/audience slots), minus the per-slot expiry stagger that +// keeps multiple JWT slots from renewing/reconnecting simultaneously. This is +// the exact value createSlotAuthToken() puts in the token's exp claim, so the +// renewal scheduling in maintainSlotConnection() can be derived from it. +unsigned long MQTTBridge::slotTokenLifetime(int index) const { + const MQTTSlot& slot = _slots[index]; + unsigned long base_lifetime = MQTTConnectionPolicy::kDefaultJwtLifetimeSecs; + if (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT && slot.preset->token_lifetime > 0) { + base_lifetime = slot.preset->token_lifetime; + } + return MQTTConnectionPolicy::jwtLifetimeSecs( + static_cast(base_lifetime), static_cast(index)); +} + +// How early (seconds before the token's exp claim) to renew the token AND +// proactively bounce the connection with fresh credentials. exp and the +// renewal schedule are locked together (both derive from slotTokenLifetime), +// so this buffer is the ONLY margin between "device re-authenticates" and +// "broker enforces exp and FIN-closes the session mid-stream" — shortening a +// preset's token_lifetime moves both times together and cannot widen it. +// The old flat 60 s lost that race whenever the device clock ran slow, or a +// single renewal attempt failed (the 60 s renewal throttle then ate the whole +// margin) — observed on the waev preset, whose 55-minute tokens are the only +// ones short enough for brokers to enforce exp against a live session. +// lifetime/10 with a 60 s floor and 300 s cap: 24 h tokens renew 5 min early +// (unchanged in practice), waev renews ~5 min early with ~5 throttled retry +// windows, and degenerate short lifetimes still renew inside their validity. +bool MQTTBridge::createSlotAuthToken(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + MQTTSlot& slot = _slots[index]; + if (!_identity) return false; + + // Determine JWT audience: preset takes priority, then custom slot audience field + const char* audience = nullptr; + if (slot.preset && slot.preset->auth_type == MQTT_AUTH_JWT) { + audience = slot.preset->jwt_audience; + } else if (slot.audience[0] != '\0') { + audience = slot.audience; + } + if (!audience || audience[0] == '\0') return false; + + // This slot is confirmed JWT, so it needs the token buffer. Allocated on first use + // and kept thereafter; every caller already treats false as "no usable token". + if (!ensureSlotAuthToken(index)) return false; + + // Ensure JWT username is set + if (_jwt_username[0] == '\0') { + char public_key_hex[65]; + mesh::Utils::toHex(public_key_hex, _identity->pub_key, PUB_KEY_SIZE); + snprintf(_jwt_username, sizeof(_jwt_username), "v1_%s", public_key_hex); + } + + // Prepare owner key + const char* owner_key = nullptr; + char owner_key_uppercase[65]; + if (_obs->mqtt_owner_public_key[0] != '\0') { + strncpy(owner_key_uppercase, _obs->mqtt_owner_public_key, sizeof(owner_key_uppercase) - 1); + owner_key_uppercase[sizeof(owner_key_uppercase) - 1] = '\0'; + for (int i = 0; owner_key_uppercase[i]; i++) { + owner_key_uppercase[i] = toupper(owner_key_uppercase[i]); + } + owner_key = owner_key_uppercase; + } + + char client_version[64]; + getClientVersion(client_version, sizeof(client_version)); + const char* email = (_obs->mqtt_email[0] != '\0') ? _obs->mqtt_email : nullptr; + + unsigned long current_time = time(nullptr); + unsigned long expires_in = slotTokenLifetime(index); // preset/default lifetime minus per-slot stagger + bool time_synced = (current_time >= 1000000000); + + if (JWTHelper::createAuthToken( + *_identity, audience, + 0, expires_in, slot.auth_token, AUTH_TOKEN_SIZE, + owner_key, client_version, email)) { + slot.token_expires_at = time_synced ? (current_time + expires_in) : 0; + return true; + } + + slot.token_expires_at = 0; + return false; +} + +bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload, size_t payload_len, bool retained, uint8_t qos) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + MQTTSlot& slot = _slots[index]; + if (!slot.client || !slot.connected) { + unsigned long now = millis(); + if (now - slot.last_log_time > SLOT_LOG_INTERVAL) { + slot.last_log_time = now; + MQTT_DEBUG_PRINTLN("MQTT%d not connected - skipping publish", index + 1); + } + return false; + } + + // Publish path by QoS: + // - QoS 0 (high-rate packets/raw): SYNCHRONOUS (async=false → esp_mqtt_client_publish), + // which writes straight to the socket. The async/outbox path drains only one queued + // item per esp-mqtt task loop (~1 msg/s/conn, gated by the 1s poll_read), so under + // even light packet load the outbox pins at its cap and drops ~20-30%. A synchronous + // write bypasses that drain ceiling entirely and does not store in the outbox. It can + // block the (Core-0, prio-1) MQTT task on a stalled socket, but only up to + // network_timeout_ms (lowered in optimizeMqttClientConfig); mesh RX (Core 1) and the + // WiFi/TCP stack (higher-prio system tasks) are unaffected, and a failed write flips + // the slot to disconnected so subsequent packets skip it. + // - QoS 1 (low-rate retained status): async, so it keeps the durable outbox + retransmit. + // + // Return convention: QoS 0 sync publish returns msg_id == 0 on success (no PUBACK + // tracking). Negative values (-1 write/failure) are the only actual failures; the queue + // retry/drop path below handles them. + bool async = (qos > 0); + int result = slot.client->publish(topic, qos, retained, payload, (int)payload_len, async); + if (result < 0) { + // QoS0 packet/raw publishes are best-effort and may be retried from the + // bridge queue; avoid logging transient first-attempt failures here. + if (qos > 0) { + static unsigned long last_fail_log = 0; + unsigned long now = millis(); + if (now - last_fail_log > 60000) { + MQTT_DEBUG_PRINTLN("MQTT%d publish failed (result=%d qos=%u)", index + 1, result, (unsigned)qos); + last_fail_log = now; + } + } + return false; + } + return true; +} + +bool MQTTBridge::publishToAllSlots(const char* topic, const char* payload, size_t payload_len, bool retained, uint8_t qos) { + bool published = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].client && _slots[i].connected) { + if (publishToSlot(i, topic, payload, payload_len, retained, qos)) { + published = true; + } + } + } + return published; +} + +// --------------------------------------------------------------------------- +// Topic building - resolves the correct topic for a given slot and message type. +// Presets use hardcoded topic logic; custom slots support user-defined templates. +// --------------------------------------------------------------------------- +bool MQTTBridge::substituteTopicTemplate(const char* tmpl, MQTTMessageType type, int slot_index, char* buf, size_t buf_size) { + return mqttBuildPublicationTopic(MQTT_ROUTE_CUSTOM, (int)type, tmpl, + _iata, _device_id, _obs->mqtt_slot_token[slot_index], + buf, buf_size); +} + +bool MQTTBridge::buildTopicForSlot(int index, MQTTMessageType type, char* topic_buf, size_t buf_size) { + static_assert( + static_cast(MSG_STATUS) == MQTT_PUBLICATION_STATUS && + static_cast(MSG_PACKETS) == MQTT_PUBLICATION_PACKETS && + static_cast(MSG_RAW) == MQTT_PUBLICATION_RAW && + static_cast(MSG_NEIGHBORS) == MQTT_PUBLICATION_NEIGHBORS, + "topic router enum drift"); + + if (!mqttTopicSlotIndexValid(index, RUNTIME_MQTT_SLOTS)) return false; + const MQTTSlot& slot = _slots[index]; + + // Preset slots: use hardcoded topic logic + if (slot.preset) { + MQTTTopicRouteStyle style = (slot.preset->topic_style == MQTT_TOPIC_MESHRANK) + ? MQTT_ROUTE_MESHRANK : MQTT_ROUTE_MESHCORE; + return mqttBuildPublicationTopic(style, (int)type, nullptr, + _iata, _device_id, _obs->mqtt_slot_token[index], + topic_buf, buf_size); + } + + // Custom slots: use topic template if set, otherwise default meshcore format + if (_obs->mqtt_slot_topic[index][0] != '\0') { + return substituteTopicTemplate(_obs->mqtt_slot_topic[index], type, index, topic_buf, buf_size); + } + // Default: meshcore format + return mqttBuildPublicationTopic(MQTT_ROUTE_MESHCORE, (int)type, nullptr, + _iata, _device_id, _obs->mqtt_slot_token[index], + topic_buf, buf_size); +} + +void MQTTBridge::publishStatusToSlot(int index) { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return; + MQTTSlot& slot = _slots[index]; + if (!slot.client || !slot.connected) return; + + refreshOriginFromPrefs(); + + // Build per-slot topic (handles IATA check for meshcore, token check for meshrank) + char status_topic[128]; + if (!buildTopicForSlot(index, MSG_STATUS, status_topic, sizeof(status_topic))) { + return; // Slot is missing required topic configuration + } + + // Reuse pre-allocated buffer to avoid heap alloc/free churn under memory pressure. + // _json_scratch_doc/_json_scratch_buffer/_origin are shared with publishStatus() and + // with the packet/raw paths; every one of them runs only on the bridge task (this + // function is reached solely via the _status_publish_pending consumer in + // mqttTaskLoop, never from the onConnect callback thread — see A2), so the accesses + // are serialized and need no mutex. + #if defined(BOARD_HAS_PSRAM) + char fallback_status_buffer[STATUS_JSON_BUFFER_SIZE]; + char* json_buffer = (_json_scratch_buffer != nullptr) ? _json_scratch_buffer : fallback_status_buffer; + #else + char* json_buffer = _json_scratch_buffer; + #endif + + char origin_id[65]; + char timestamp[40]; + char radio_info[64]; + + // Status timestamp: UTC with explicit +00:00 offset, same as packet/raw JSON + // `timestamp` (system clock is UTC — SNTP offset 0; prefs Timezone is separate). + struct timeval now_tv; + gettimeofday(&now_tv, nullptr); + MQTTMessageBuilder::formatIsoTimestampForMqtt(now_tv.tv_sec, now_tv.tv_usec, _timezone, timestamp, sizeof(timestamp)); + + snprintf(radio_info, sizeof(radio_info), "%.6f,%.1f,%d,%d", + _prefs->freq, _prefs->bw, _prefs->sf, _prefs->cr); + + strncpy(origin_id, _device_id, sizeof(origin_id) - 1); + origin_id[sizeof(origin_id) - 1] = '\0'; + + char client_version[64]; + getClientVersion(client_version, sizeof(client_version)); + + // Collect stats on-demand if sources are available + int battery_mv = -1; + int uptime_secs = -1; + int errors = -1; + int noise_floor = -999; + int tx_air_secs = -1; + int rx_air_secs = -1; + int recv_errors = -1; + int packets_sent = -1; + int packets_received = -1; + + if (_board) battery_mv = _board->getBattMilliVolts(); + if (_ms) uptime_secs = _ms->getMillis() / 1000; + if (_dispatcher) { + errors = _dispatcher->getErrFlags(); + tx_air_secs = _dispatcher->getTotalAirTime() / 1000; + rx_air_secs = _dispatcher->getReceiveAirTime() / 1000; + packets_sent = (int)(_dispatcher->getNumSentFlood() + _dispatcher->getNumSentDirect()); + packets_received = (int)(_dispatcher->getNumRecvFlood() + _dispatcher->getNumRecvDirect()); + } + if (_radio) { + noise_floor = (int16_t)_radio->getNoiseFloor(); + recv_errors = (int)_radio->getPacketsRecvErrors(); + } + + // Internal heap free (for diagnosing repeater hangs from internal heap exhaustion) + int internal_heap_free = (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + + int len = MQTTMessageBuilder::buildStatusMessage( + _json_scratch_doc, + _origin, origin_id, _board_model, _firmware_version, radio_info, + client_version, "online", timestamp, json_buffer, STATUS_JSON_BUFFER_SIZE, + battery_mv, uptime_secs, errors, _queue_count, noise_floor, + tx_air_secs, rx_air_secs, recv_errors, internal_heap_free, + packets_sent, packets_received, + _prefs->disable_fwd ? "off" : "on" + ); + + if (len > 0) { + // Honor the preset's retain policy, matching publishStatus() — brokers that + // set allow_retain=false (e.g. waev) reject retained publishes, so this + // on-connect status must not force retain=true. Custom slots default to + // non-retained here too, keeping both status paths consistent. + bool use_retain = slot.preset ? slot.preset->allow_retain : false; + int result = slot.client->publish(status_topic, 1, use_retain, json_buffer, len); + if (result <= 0) { + MQTT_DEBUG_PRINTLN("MQTT%d status publish failed", index + 1); + } + } +} + +void MQTTBridge::updateCachedConnectionStatus() { + bool any_connected = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].connected) { + any_connected = true; + break; + } + } + _cached_has_connected_slots = any_connected; +} + +bool MQTTBridge::isAnySlotConnected() { + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].enabled && _slots[i].connected) { + return true; + } + } + return false; +} + +void MQTTBridge::setSlotPreset(int slot_index, const char* preset_name) { + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return; + + // On ESP32, teardown/setup involves TLS and must run on the MQTT task (Core 0). + // Set a flag so the MQTT task picks it up on its next loop iteration. + #ifdef ESP_PLATFORM + if (_mqtt_task_handle != nullptr) { + _slot_reconfigure_pending[slot_index] = true; + MQTT_DEBUG_PRINTLN("MQTT%d reconfigure queued (preset: %s)", slot_index + 1, preset_name); + return; + } + #endif + + // Non-ESP32 or bridge not yet started: apply directly + applySlotPreset(slot_index, preset_name); +} + +void MQTTBridge::applySlotPreset(int slot_index, const char* preset_name) { + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return; + MQTTSlot& slot = _slots[slot_index]; + + teardownSlot(slot_index); + + if (strcmp(preset_name, MQTT_PRESET_NONE) == 0 || preset_name[0] == '\0') { + slot.enabled = false; + slot.preset = nullptr; + return; + } + + if (strcmp(preset_name, MQTT_PRESET_CUSTOM) == 0) { + slot.preset = nullptr; + // Re-sync every custom field from prefs (same copy begin() does at startup) + // so a CLI/web edit to the host, port, credentials, or JWT audience is + // actually picked up on reconfigure. Previously this branch reused the + // stale slot fields, so e.g. changing mqttN.server or mqttN.username had no + // effect on the live connection. Token and topic are read live from _obs in + // setupSlot()/buildTopicForSlot(), so they don't need copying here. + strncpy(slot.host, _obs->mqtt_slot_host[slot_index], sizeof(slot.host) - 1); + slot.host[sizeof(slot.host) - 1] = '\0'; + slot.port = _obs->mqtt_slot_port[slot_index]; + strncpy(slot.username, _obs->mqtt_slot_username[slot_index], sizeof(slot.username) - 1); + slot.username[sizeof(slot.username) - 1] = '\0'; + strncpy(slot.password, _obs->mqtt_slot_password[slot_index], sizeof(slot.password) - 1); + slot.password[sizeof(slot.password) - 1] = '\0'; + strncpy(slot.audience, _obs->mqtt_slot_audience[slot_index], sizeof(slot.audience) - 1); + slot.audience[sizeof(slot.audience) - 1] = '\0'; + slot.enabled = (slot.host[0] != '\0'); + if (_initialized && slot.enabled && customEndpointComplete(slot.host, slot.port)) { + // Same cap startup applies. teardownSlot() above already released this slot's own + // position, so reconfiguring a live slot still passes. + if (!canActivateSlot(slot_index)) { + MQTT_DEBUG_PRINTLN("MQTT%d skipped: max active slots (%d) reached", slot_index + 1, _max_active_slots); + slot.enabled = false; + return; + } + setupSlot(slot_index); + } + return; + } + + const MQTTPresetDef* preset = findMQTTPreset(preset_name); + if (preset) { + slot.enabled = true; + slot.preset = preset; + if (mqttPresetNeedsSlotCredentials(preset)) { + strncpy(slot.username, _obs->mqtt_slot_username[slot_index], sizeof(slot.username) - 1); + slot.username[sizeof(slot.username) - 1] = '\0'; + strncpy(slot.password, _obs->mqtt_slot_password[slot_index], sizeof(slot.password) - 1); + slot.password[sizeof(slot.password) - 1] = '\0'; + } + if (_initialized) { + char reason[80]; + if (!isSlotReady(slot_index, reason, sizeof(reason))) { + MQTT_DEBUG_PRINTLN("MQTT%d (%s) not ready - run '%s' to connect", slot_index + 1, preset_name, reason); + return; + } + // Same cap startup applies. Without this a live reconfigure could raise a + // non-PSRAM board to three concurrent TLS sessions against a cap of two. + if (!canActivateSlot(slot_index)) { + MQTT_DEBUG_PRINTLN("MQTT%d skipped: max active slots (%d) reached", slot_index + 1, _max_active_slots); + slot.enabled = false; + return; + } + setupSlot(slot_index); + } + } +} + +void MQTTBridge::setSlotCustomBroker(int slot_index, const char* host, uint16_t port, + const char* username, const char* password) { + if (slot_index < 0 || slot_index >= RUNTIME_MQTT_SLOTS) return; + MQTTSlot& slot = _slots[slot_index]; + + strncpy(slot.host, host ? host : "", sizeof(slot.host) - 1); + slot.host[sizeof(slot.host) - 1] = '\0'; + slot.port = port; + strncpy(slot.username, username ? username : "", sizeof(slot.username) - 1); + slot.username[sizeof(slot.username) - 1] = '\0'; + strncpy(slot.password, password ? password : "", sizeof(slot.password) - 1); + slot.password[sizeof(slot.password) - 1] = '\0'; +} + +// --------------------------------------------------------------------------- +// WiFi connection handling +// --------------------------------------------------------------------------- + +void MQTTBridge::checkConfigurationMismatch() { + // Warn if packets are enabled but both rx and tx are off — nothing will be published + if (_obs->mqtt_packets_enabled && !_obs->mqtt_rx_enabled && _obs->mqtt_tx_enabled == 0) { + unsigned long now = millis(); + if (_last_config_warning == 0 || (now - _last_config_warning > CONFIG_WARNING_INTERVAL)) { + MQTT_DEBUG_PRINTLN("MQTT: Both mqtt.rx and mqtt.tx are off - no packets will be published. Run 'set mqtt.rx on' or 'set mqtt.tx on' to fix."); + _last_config_warning = now; + } + } else { + _last_config_warning = 0; + } +} + +bool MQTTBridge::handleWiFiConnection(unsigned long now) { + wl_status_t current_wifi_status = WiFi.status(); + bool transitioned_to_connected = false; + + if (current_wifi_status == WL_CONNECTED && s_wifi_connected_at == 0) { + s_wifi_connected_at = now; + } + if (!_wifi_status_initialized) { + _last_wifi_status = current_wifi_status; + _wifi_status_initialized = true; + if (current_wifi_status != WL_CONNECTED) { + _wifi_disconnected_time = now; + } + } + if (now - _last_wifi_check <= 10000) { + return false; + } + _last_wifi_check = now; + + if (current_wifi_status == WL_CONNECTED) { + if (_last_wifi_status != WL_CONNECTED) { + transitioned_to_connected = true; + _wifi_disconnected_time = 0; + s_wifi_connected_at = now; + _wifi_reconnect_backoff_attempt = 0; + #ifdef ESP_PLATFORM + wifi_ps_type_t ps_mode; + uint8_t ps_pref = _obs->wifi_power_save; + if (ps_pref == 1) { + ps_mode = WIFI_PS_NONE; + } else if (ps_pref == 2) { + ps_mode = WIFI_PS_MAX_MODEM; + } else { + ps_mode = WIFI_PS_NONE; // default: no power save; eliminates DTIM wake latency on mains-powered bridges + } + esp_wifi_set_ps(ps_mode); + #ifdef MQTT_WIFI_TX_POWER + WiFi.setTxPower(MQTT_WIFI_TX_POWER); + #else + WiFi.setTxPower(WIFI_POWER_11dBm); + #endif + #endif + } + if (s_wifi_connected_at == 0) { + s_wifi_connected_at = now; + } + _last_wifi_status = WL_CONNECTED; + } else { + if (_last_wifi_status == WL_CONNECTED) { + _wifi_disconnected_time = now; + s_wifi_connected_at = 0; + // Disconnect all slot clients when WiFi drops + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (_slots[i].client && _slots[i].connected) { + _slots[i].client->disconnect(); + } + } + } else if (_wifi_disconnected_time > 0) { + // Backoff ladder + wrap-safe timing live in MQTTConnectionPolicy (Phase 6), + // exercised by host tests. Behavior is unchanged: both the link-down + // duration and the since-last-attempt interval must clear the current rung + // (elapsedMs is the wrap-safe form of the old ULONG_MAX branch). + if (MQTTConnectionPolicy::wifiReconnectDue( + (uint32_t)now, (uint32_t)_wifi_disconnected_time, + (uint32_t)_last_wifi_reconnect_attempt, + _wifi_reconnect_backoff_attempt)) { + _last_wifi_reconnect_attempt = now; + _wifi_reconnect_backoff_attempt = + MQTTConnectionPolicy::nextWifiBackoffAttempt(_wifi_reconnect_backoff_attempt); + WiFi.disconnect(); + WiFi.begin(_obs->wifi_ssid, _obs->wifi_password); + } + } + _last_wifi_status = current_wifi_status; + } + return transitioned_to_connected; +} + +bool MQTTBridge::isReady() const { + return _initialized && isWiFiConfigValid(_obs); +} + +bool MQTTBridge::isIATAValid() const { + if (strlen(_iata) == 0 || strcmp(_iata, "XXX") == 0) { + return false; + } + return true; +} + +bool MQTTBridge::isSlotReady(int index, char* reason_buf, size_t reason_size) const { + if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false; + const MQTTSlot& slot = _slots[index]; + + if (!slot.enabled) return true; // disabled slots are "ready" (nothing to do) + + if (slot.preset) { + if (slot.preset->topic_style == MQTT_TOPIC_MESHRANK) { + if (_obs->mqtt_slot_token[index][0] == '\0') { + if (reason_buf) snprintf(reason_buf, reason_size, "set mqtt%d.token ", index + 1); + return false; + } + } else if (slot.preset->topic_style == MQTT_TOPIC_MESHCORE) { + if (!isIATAValid()) { + if (reason_buf) snprintf(reason_buf, reason_size, "set mqtt.iata "); + return false; + } + } + if (mqttPresetNeedsSlotUsername(slot.preset) && + _obs->mqtt_slot_username[index][0] == '\0') { + if (reason_buf) snprintf(reason_buf, reason_size, "set mqtt%d.username ", index + 1); + return false; + } + if (mqttPresetNeedsSlotPassword(slot.preset) && + _obs->mqtt_slot_password[index][0] == '\0') { + if (reason_buf) snprintf(reason_buf, reason_size, "set mqtt%d.password ", index + 1); + return false; + } + } else { + // Custom slot without a topic template uses meshcore format, needs IATA + if (_obs->mqtt_slot_topic[index][0] == '\0' && !isIATAValid()) { + if (reason_buf) snprintf(reason_buf, reason_size, "set mqtt.iata or set mqtt%d.topic