Fix/noise floor ratchet, adapt noise to the real floor with recovery in both directions

src/Dispatcher.cpp

@@ -82,6 +82,11 @@ void Dispatcher::loop() {
   if (!is_recv && _ms->getMillis() - radio_nonrx_start > 8000) {   // radio has not been in Rx mode for 8 seconds!
     _err_flags |= ERR_EVENT_STARTRX_TIMEOUT;
   }
+  if (_radio->isRxDamaged()) {
+    // chip left RX behind the firmware's back and resisted the wrapper's recovery —
+    // the node is (still) deaf; a reboot restores it
+    _err_flags |= ERR_EVENT_RX_DESYNC;
+  }
 
   if (outbound) {  // waiting for outbound send to be completed
     if (_radio->isSendComplete()) {

src/Dispatcher.h

@@ -63,6 +63,11 @@ class Radio {
 
   virtual int getNoiseFloor() const { return 0; }
 
+  // RX-desync watchdog: true when the driver detected the chip left RX behind the
+  // firmware's back and its own recovery attempts failed (reboot needed). Radio
+  // implementations without an authoritative status register stay false.
+  virtual bool isRxDamaged() const { return false; }
+
   virtual void triggerNoiseFloorCalibrate(int threshold) { }
 
   virtual void setCADEnabled(bool enable) { }
@@ -110,6 +115,7 @@ typedef uint32_t  DispatcherAction;
 #define ERR_EVENT_FULL              (1 << 0)
 #define ERR_EVENT_CAD_TIMEOUT       (1 << 1)
 #define ERR_EVENT_STARTRX_TIMEOUT   (1 << 2)
+#define ERR_EVENT_RX_DESYNC         (1 << 3)   // radio wedged out of RX and resisted the wrapper's recovery
 
 /**
  * \brief  The low-level task that manages detecting incoming Packets, and the queueing

src/helpers/StatsFormatHelper.h

@@ -24,13 +24,14 @@ class StatsFormatHelper {
                               RadioDriverType& driver,
                               uint32_t total_air_time_ms,
                               uint32_t total_rx_air_time_ms) {
-    sprintf(reply, 
-      "{\"noise_floor\":%d,\"last_rssi\":%d,\"last_snr\":%.2f,\"tx_air_secs\":%u,\"rx_air_secs\":%u}",
+    sprintf(reply,
+      "{\"noise_floor\":%d,\"last_rssi\":%d,\"last_snr\":%.2f,\"tx_air_secs\":%u,\"rx_air_secs\":%u,\"rx_desync\":%u}",
       (int16_t)radio->getNoiseFloor(),
       (int16_t)driver.getLastRSSI(),
       driver.getLastSNR(),
       total_air_time_ms / 1000,
-      total_rx_air_time_ms / 1000
+      total_rx_air_time_ms / 1000,
+      driver.getRxDesyncEvents()
     );
   }
 

src/helpers/radiolib/CustomLLCC68Wrapper.h

@@ -37,6 +37,11 @@ class CustomLLCC68Wrapper : public RadioLibWrapper {
 
   void doResetAGC() override { sx126xResetAGC((SX126x *)_radio); }
 
+  // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
+  bool verifyRxChipMode() override {
+    return ((((CustomLLCC68 *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
+  }
+
   bool setRxBoostedGainMode(bool en) override {
     return ((CustomLLCC68 *)_radio)->setRxBoostedGainMode(en) == RADIOLIB_ERR_NONE;
   }

src/helpers/radiolib/CustomSTM32WLxWrapper.h

@@ -36,4 +36,9 @@ class CustomSTM32WLxWrapper : public RadioLibWrapper {
   uint8_t getSpreadingFactor() const override { return ((CustomSTM32WLx *)_radio)->spreadingFactor; }
 
   void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }
+
+  // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
+  bool verifyRxChipMode() override {
+    return ((((CustomSTM32WLx *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
+  }
 };

src/helpers/radiolib/CustomSX1262Wrapper.h

@@ -49,4 +49,10 @@ class CustomSX1262Wrapper : public RadioLibWrapper {
   }
 
   void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }
+
+  // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
+  // Plain SPI read, does not disturb reception.
+  bool verifyRxChipMode() override {
+    return ((((CustomSX1262 *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
+  }
 };

src/helpers/radiolib/CustomSX1268Wrapper.h

@@ -46,4 +46,9 @@ class CustomSX1268Wrapper : public RadioLibWrapper {
   }
 
   void doResetAGC() override { sx126xResetAGC((SX126x *)_radio, getRxBoostedGainMode()); }
+
+  // RX-desync watchdog probe: GetStatus byte, bits 6:4 = chip mode (0x5 = RX).
+  bool verifyRxChipMode() override {
+    return ((((CustomSX1268 *)_radio)->getStatus() >> 4) & 0x07) == 0x05;
+  }
 };

src/helpers/radiolib/RadioLibWrappers.cpp

@@ -8,11 +8,22 @@
 #define STATE_TX_DONE    4
 #define STATE_INT_READY 16
 
-#define NUM_NOISE_FLOOR_SAMPLES  64
-#define SAMPLING_THRESHOLD  14
-
 static volatile uint8_t state = STATE_IDLE;
 
+// In-place insertion sort of int16_t samples for the noise-floor median. Runs once per
+// calibration block (64 elements, ~every 2 s of idle), so O(n^2) is irrelevant here.
+static void sortInt16(int16_t* a, int n) {
+  for (int i = 1; i < n; i++) {
+    int16_t key = a[i];
+    int j = i - 1;
+    while (j >= 0 && a[j] > key) {
+      a[j + 1] = a[j];
+      j--;
+    }
+    a[j + 1] = key;
+  }
+}
+
 // this function is called when a complete packet
 // is transmitted by the module
 static
@@ -40,7 +51,13 @@ void RadioLibWrapper::begin() {
 
   // start average out some samples
   _num_floor_samples = 0;
-  _floor_sample_sum = 0;
+  _floor_block_ready = false;
+  _last_floor_sample_at = 0;
+  _held_block_count = 0;
+
+  _last_rx_sync_check = millis();
+  _rx_desync_streak = 0;
+  n_rx_desync_events = n_rx_desync_fatals = 0;
 }
 
 uint32_t RadioLibWrapper::getRngSeed() {
@@ -61,9 +78,9 @@ void RadioLibWrapper::idle() {
 
 void RadioLibWrapper::triggerNoiseFloorCalibrate(int threshold) {
   _threshold = threshold;
-  if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES) {  // ignore trigger if currently sampling
+  if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES) {  // restart only once the current block is complete
     _num_floor_samples = 0;
-    _floor_sample_sum = 0;
+    _floor_block_ready = false;
   }
 }
 
@@ -78,34 +95,116 @@ void RadioLibWrapper::resetAGC() {
   doResetAGC();
   state = STATE_IDLE;   // trigger a startReceive()
 
-  // Reset noise floor sampling so it reconverges from scratch.
-  // Without this, a stuck _noise_floor of -120 makes the sampling threshold
-  // too low (-106) to accept normal samples (~-105), self-reinforcing the
-  // stuck value even after the receiver has recovered.
-  _noise_floor = 0;
+  // Discard any in-progress noise-floor block: the analog frontend was just reset, so
+  // queued RSSI samples are stale. _noise_floor itself is left in place — the median
+  // estimator no longer drifts to -120 (the reason the old ratchet needed a hard
+  // _noise_floor = 0 reset), and forcing 0 here would create a brief permissive LBT
+  // window (margin = RSSI - 0) until the next block completes.
   _num_floor_samples = 0;
-  _floor_sample_sum = 0;
+  _floor_block_ready = false;
+  _held_block_count = 0;   // contamination context is stale after an AFE reset
 }
 
 void RadioLibWrapper::loop() {
   if (state == STATE_RX && _num_floor_samples < NUM_NOISE_FLOOR_SAMPLES) {
-    if (!isReceivingPacket()) {
-      int rssi = getCurrentRSSI();
-      if (rssi < _noise_floor + SAMPLING_THRESHOLD) {  // only consider samples below current floor + sampling THRESHOLD
-        _num_floor_samples++;
-        _floor_sample_sum += rssi;
-      }
+    uint32_t now = millis();
+    if (!isReceivingPacket() && now - _last_floor_sample_at >= NOISE_FLOOR_SAMPLE_INTERVAL_MS) {
+      // Accept every idle sample, spaced NOISE_FLOOR_SAMPLE_INTERVAL_MS apart so the block spans a real
+      // ~3.2 s window and the median rejects transient transmissions (not a few-ms snapshot). The old
+      // "rssi < floor + threshold" filter was a one-way ratchet: it only accepted samples below the
+      // current floor, so the block average drifted to the -120 clamp and never recovered — leaving
+      // _noise_floor stuck low and the RSSI-margin LBT permanently over-sensitive.
+      _floor_samples[_num_floor_samples++] = (int16_t)getCurrentRSSI();
+      _last_floor_sample_at = now;
     }
-  } else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && _floor_sample_sum != 0) {
-    _noise_floor = _floor_sample_sum / NUM_NOISE_FLOOR_SAMPLES;
-    if (_noise_floor < -120) {
-      _noise_floor = -120;    // clamp to lower bound of -120dBi
+  } else if (_num_floor_samples >= NUM_NOISE_FLOOR_SAMPLES && !_floor_block_ready) {
+    // Block complete: reduce to the median. The median rejects transient interference
+    // spikes (high and low outliers) and recovers in BOTH directions, unlike the ratcheted
+    // mean. _noise_floor is written only here, so the previous value stays valid while the
+    // next block is sampled — no reset-to-0, no permissive LBT window during reconvergence.
+    sortInt16(_floor_samples, NUM_NOISE_FLOOR_SAMPLES);
+    int16_t median = (int16_t)(((int32_t)_floor_samples[NUM_NOISE_FLOOR_SAMPLES / 2 - 1]
+                              + (int32_t)_floor_samples[NUM_NOISE_FLOOR_SAMPLES / 2]) / 2);
+    // One-sided hold: a median jumping far ABOVE the published floor is activity-contaminated
+    // (inter-packet energy slips past the !isReceivingPacket() idle guard). Hold the old value so
+    // the RSSI-margin LBT stays meaningful under load; near-stable/quieter blocks publish at once.
+    // First block always publishes (_noise_floor=0 from begin()), so the hold binds only post-boot.
+    //
+    // Bounded: after NOISE_FLOOR_MAX_HELD_BLOCKS consecutive held blocks accept the median, else a real
+    // permanent rise is held forever (stuck-floor bug from the other direction). Count-based so the hold
+    // rides out load bursts (slow blocks) while a quiet rise releases in a few blocks.
+    if (median > _noise_floor + NOISE_FLOOR_MAX_RISE_DB) {
+      _held_block_count++;
+      if (_held_block_count >= NOISE_FLOOR_MAX_HELD_BLOCKS) {
+        _noise_floor = median;
+        if (_noise_floor < -120) {
+          _noise_floor = -120;    // clamp to lower bound of -120dBi
+        }
+        _held_block_count = 0;
+        #ifdef MESH_DEBUG_NOISE_FLOOR
+        MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (accepted after %d held blocks, persistent rise)",
+                           (int)_noise_floor, NOISE_FLOOR_MAX_HELD_BLOCKS);
+        #endif
+      } else {
+        MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor held at %d (block median %d contaminated, held %d/%d)",
+                           (int)_noise_floor, (int)median, _held_block_count, NOISE_FLOOR_MAX_HELD_BLOCKS);
+      }
+    } else {
+      _held_block_count = 0;
+      _noise_floor = median;
+      if (_noise_floor < -120) {
+        _noise_floor = -120;    // clamp to lower bound of -120dBi
+      }
+      #ifdef MESH_DEBUG_NOISE_FLOOR
+      MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d (median)", (int)_noise_floor);
+      #endif
     }
-    _floor_sample_sum = 0;
+    _floor_block_ready = true;
+  }
 
-    #ifdef MESH_DEBUG_NOISE_FLOOR
-    MESH_DEBUG_PRINTLN("RadioLibWrapper: noise_floor = %d", (int)_noise_floor);
-    #endif
+  // --- RX-desync watchdog ---
+  // `state` is firmware-side truth. If the chip silently leaves RX (supply dip during
+  // TX, SPI glitch, front-end upset), the RAM copy still says STATE_RX, so recvRaw()
+  // never re-arms and the Dispatcher-side 8 s check — reading the same variable —
+  // stays quiet: the node goes deaf until reboot. (Visible symptom: the Current-RSSI
+  // register freezes at the last energy seen, pinning the noise floor high.)
+  // Here we ask the CHIP instead: verifyRxChipMode() reads its real operating mode.
+  // Radio types without an authoritative status register report true (watchdog off).
+  // Recovery: after RX_DESYNC_CONFIRM_TICKS bad polls, re-arm the receiver; if that
+  // doesn't take, escalate to a warm sleep (resets the modem state machine/AFE).
+  // A streak that survives both is counted fatal and surfaces as ERR_EVENT_RX_DESYNC.
+  if (state == STATE_RX) {
+    uint32_t now = millis();
+    if (now - _last_rx_sync_check >= RX_DESYNC_CHECK_INTERVAL_MS) {
+      _last_rx_sync_check = now;
+      if (!isReceivingPacket() && !verifyRxChipMode()) {
+        if (_rx_desync_streak == 0) {
+          n_rx_desync_events++;
+          MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync - chip reports mode != RX");
+        }
+        _rx_desync_streak++;
+        // samples taken while wedged read a frozen RSSI register: discard the block
+        _num_floor_samples = 0;
+        _floor_block_ready = false;
+        _held_block_count = 0;
+        if (_rx_desync_streak == RX_DESYNC_CONFIRM_TICKS) {
+          MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync confirmed - re-arming receiver");
+          idle();          // standby, then a fresh startReceive()
+          startRecv();
+        } else if (_rx_desync_streak > RX_DESYNC_CONFIRM_TICKS) {
+          if (_rx_desync_streak == RX_DESYNC_FATAL_STREAK) {
+            n_rx_desync_fatals++;
+            MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync survived AFE reset - radio damaged, reboot suggested");
+          }
+          // warm sleep resets the analog frontend and modem state machine;
+          // resetAGC() re-arms from STATE_IDLE via recvRaw() on the next pass
+          resetAGC();
+        }
+      } else if (_rx_desync_streak > 0) {
+        MESH_DEBUG_PRINTLN("RadioLibWrapper: RX desync resolved after %u ticks", _rx_desync_streak);
+        _rx_desync_streak = 0;
+      }
+    }
   }
 }
 

src/helpers/radiolib/RadioLibWrappers.h

@@ -3,6 +3,26 @@
 #include <Mesh.h>
 #include <RadioLib.h>
 
+#define NUM_NOISE_FLOOR_SAMPLES  64   // RSSI samples reduced to a median per noise-floor calibration block
+#define NOISE_FLOOR_MAX_RISE_DB  15   // block median jumping this far ABOVE the published floor is treated as
+                                      // activity-contaminated and held, so the RSSI-margin LBT keeps a meaningful
+                                      // (idle) reference while the channel is occupied
+#define NOISE_FLOOR_SAMPLE_INTERVAL_MS  50   // min spacing between RSSI samples so a 64-sample block spans a real
+                                             // ~3.2 s window, giving the median temporal interference rejection
+                                             // instead of collapsing to a few ms of near-simultaneous readings
+#define NOISE_FLOOR_MAX_HELD_BLOCKS  3   // after this many consecutive held blocks the median is accepted, so a
+                                         // permanent floor rise can't keep it stuck low. Count-based: rides out load
+                                         // bursts, a true rise releases in a few blocks.
+
+// RX-desync watchdog: the `state` variable is firmware-side truth. If the chip silently
+// leaves RX (supply dip during TX, SPI glitch, front-end upset) the RAM copy still says
+// STATE_RX, so recvRaw() never re-arms and the Dispatcher-side check (reading the same
+// variable) stays quiet — the node goes deaf until reboot, and the Current-RSSI register
+// freezes at the last energy seen (the "noise floor pinned high" symptom).
+#define RX_DESYNC_CHECK_INTERVAL_MS  10000   // cadence of the chip-mode verification poll
+#define RX_DESYNC_CONFIRM_TICKS      2       // consecutive bad polls before recovery starts (debounces one misread,
+                                             // e.g. the first GetStatus that wakes the chip from warm sleep)
+#define RX_DESYNC_FATAL_STREAK       5       // streak that survived sleep-level recovery — flag via ERR_EVENT_RX_DESYNC
 #ifdef USE_CC310_HW_CRYPTO
 #include <Adafruit_nRFCrypto.h>
 #endif
@@ -16,11 +36,18 @@ class RadioLibWrapper : public mesh::Radio {
   PhysicalLayer* _radio;
   mesh::MainBoard* _board;
   uint32_t n_recv, n_sent, n_recv_errors;
+  uint16_t n_rx_desync_events;    // desync episodes detected (recovery was attempted for each)
+  uint16_t n_rx_desync_fatals;    // episodes that survived sleep-level recovery (reboot needed)
   int16_t _noise_floor, _threshold;
   bool _cad_enabled;
   uint16_t _num_floor_samples;
-  int32_t _floor_sample_sum;
+  int16_t _floor_samples[NUM_NOISE_FLOOR_SAMPLES];
+  bool _floor_block_ready;   // true once a full block has been reduced to a median (waits for trigger to restart)
+  uint32_t _last_floor_sample_at;   // millis() of the last accepted RSSI sample (rate-limits block sampling)
+  uint8_t _held_block_count;   // consecutive held blocks since the last published noise-floor value
   uint8_t _preamble_sf;
+  uint32_t _last_rx_sync_check;   // millis() of the last chip-mode verification (RX-desync watchdog)
+  uint8_t _rx_desync_streak;      // consecutive verifications that found the chip out of RX (0 = healthy)
 
   void idle();
   void startRecv();
@@ -70,6 +97,14 @@ class RadioLibWrapper : public mesh::Radio {
   uint32_t getPacketsSent() const { return n_sent; }
   void resetStats() { n_recv = n_sent = n_recv_errors = 0; }
 
+  // RX-desync watchdog. verifyRxChipMode() reads the chip's real operating mode;
+  // base assumes RX (no authoritative status register on every radio type).
+  // Override in radio-specific wrappers that can check (SX126x GetStatus).
+  virtual bool verifyRxChipMode() { return true; }
+  bool isRxDamaged() const override { return _rx_desync_streak >= RX_DESYNC_FATAL_STREAK; }
+  uint16_t getRxDesyncEvents() const { return n_rx_desync_events; }
+  uint16_t getRxDesyncFatals() const { return n_rx_desync_fatals; }
+
   virtual float getLastRSSI() const override;
   virtual float getLastSNR() const override;