feat: add msgpack_strip_nulls() to shrink blobs by dropping nil map keys

README.md

@@ -427,6 +427,28 @@ SELECT msgpack_to_json(
 -- {"a":1,"c":3}
 ```
 
+#### `msgpack_strip_nulls(mp)`
+
+Recursively removes every map key whose value is `nil`, shrinking the
+encoded size. Recurses into nested maps, including maps nested inside
+arrays. Array **elements** are always preserved (a literal `null` inside
+an array stays), so array length/order never changes — only map keys are
+dropped.
+
+```sql
+SELECT msgpack_to_json(
+  msgpack_strip_nulls(msgpack_from_json('{"a":1,"b":null,"c":{"x":null,"y":2}}'))
+);
+-- {"a":1,"c":{"y":2}}
+
+SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_array(1, null, 3)));
+-- [1,null,3]   (array elements untouched)
+
+-- Shrinks storage for sparse wide records:
+SELECT length(mp), length(msgpack_strip_nulls(mp))
+FROM (SELECT msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL) AS mp);
+```
+
 ---
 
 ### JSON conversion

src/msgpack.c

@@ -29,6 +29,7 @@
 **     msgpack_remove(mp, path, ...)      -- remove element(s)
 **     msgpack_array_insert(mp, path, v)  -- insert before index
 **     msgpack_patch(mp, patch)           -- RFC 7386 merge-patch
+**     msgpack_strip_nulls(mp)            -- recursively drop nil-valued map keys
 **
 **   JSON conversion
 **     msgpack_from_json(json_text)       -- JSON → msgpack
@@ -1880,6 +1881,101 @@ static int mpMergePatch(
   return SQLITE_OK;
 }
 
+/*
+** mpStripNulls — recursively rebuild a msgpack value, dropping any map key
+** whose value is MP_NIL.  Recurses into nested maps and into maps/arrays
+** nested inside arrays; array elements themselves are never dropped (only
+** map keys), so array length/order is always preserved.  Scalars pass
+** through unchanged.
+*/
+static int mpStripNulls(
+  MpBuf *out,
+  const u8 *a, u32 n, u32 i,
+  int depth
+){
+  if(i>=n) return SQLITE_ERROR;
+  if(depth>MP_MAX_DEPTH) return SQLITE_ERROR;
+  u8 b = a[i];
+  int isMap   = (b>=0x80&&b<=0x8f)||b==MP_MAP16||b==MP_MAP32;
+  int isArray = (b>=0x90&&b<=0x9f)||b==MP_ARRAY16||b==MP_ARRAY32;
+
+  if(isMap){
+    u32 count, dataOff;
+    if(b>=0x80&&b<=0x8f)  { count=b&0x0f;          dataOff=i+1; }
+    else if(b==MP_MAP16)  {
+      if(i+3>n) return SQLITE_ERROR;
+      count=mpRead16(a+i+1);  dataOff=i+3;
+    } else {
+      if(i+5>n) return SQLITE_ERROR;
+      count=mpRead32(a+i+1);  dataOff=i+5;
+    }
+    /* Every pair needs >=2 bytes (>=1-byte key + >=1-byte value); reject a
+    ** declared count that cannot fit in the remaining buffer before doing
+    ** any work (guards against malformed/adversarial input). */
+    if(count > (n-dataOff)/2) return SQLITE_ERROR;
+
+    MpBuf tmp; mpBufInit(&tmp, out->pCtx);
+    u32 newCount=0, c=dataOff;
+    for(u32 k=0; k<count; k++){
+      if(c>=n){ mpBufReset(&tmp); return SQLITE_ERROR; }
+      u32 valOff = mpSkipOne(a,n,c);
+      if(!valOff){ mpBufReset(&tmp); return SQLITE_ERROR; }
+      u32 pairEnd = mpSkipOne(a,n,valOff);
+      if(!pairEnd){ mpBufReset(&tmp); return SQLITE_ERROR; }
+
+      if(a[valOff]!=MP_NIL){
+        MpBuf mb; mpBufInit(&mb,out->pCtx);
+        int rc = mpStripNulls(&mb, a,n,valOff, depth+1);
+        if(rc!=SQLITE_OK){ mpBufReset(&mb); mpBufReset(&tmp); return rc; }
+        mpBufAppend(&tmp, a+c, valOff-c);      /* key, verbatim */
+        mpBufAppend(&tmp, mb.aBuf, mb.nUsed);  /* stripped value */
+        mpBufReset(&mb);
+        newCount++;
+      }
+      c = pairEnd;
+    }
+    if( tmp.bErr ){ mpBufReset(&tmp); return SQLITE_NOMEM; }
+    mpEncodeMapHeader(out, newCount);
+    mpBufAppend(out, tmp.aBuf, tmp.nUsed);
+    mpBufReset(&tmp);
+    return out->bErr ? SQLITE_NOMEM : SQLITE_OK;
+
+  } else if(isArray){
+    u32 count, dataOff;
+    if(b>=0x90&&b<=0x9f)   { count=b&0x0f;          dataOff=i+1; }
+    else if(b==MP_ARRAY16) {
+      if(i+3>n) return SQLITE_ERROR;
+      count=mpRead16(a+i+1);  dataOff=i+3;
+    } else {
+      if(i+5>n) return SQLITE_ERROR;
+      count=mpRead32(a+i+1);  dataOff=i+5;
+    }
+    /* Every element needs >=1 byte. */
+    if(count > n-dataOff) return SQLITE_ERROR;
+
+    /* Element count is unchanged (array elements are never dropped), so the
+    ** header can be written immediately and every element recursed straight
+    ** into out; on error the caller discards the whole buffer. */
+    mpEncodeArrayHeader(out, count);
+    u32 c=dataOff;
+    for(u32 k=0; k<count; k++){
+      if(c>=n) return SQLITE_ERROR;
+      int rc = mpStripNulls(out, a,n,c, depth+1);
+      if(rc!=SQLITE_OK) return rc;
+      u32 next = mpSkipOne(a,n,c);
+      if(!next) return SQLITE_ERROR;
+      c = next;
+    }
+    return out->bErr ? SQLITE_NOMEM : SQLITE_OK;
+
+  } else {
+    u32 end = mpSkipOne(a,n,i);
+    if(!end) return SQLITE_ERROR;
+    mpBufAppend(out, a+i, end-i);
+    return out->bErr ? SQLITE_NOMEM : SQLITE_OK;
+  }
+}
+
 /* ---- SQL wrapper functions ---- */
 static void msgpackSetFunc(
   sqlite3_context *ctx, int argc, sqlite3_value **argv
@@ -1919,6 +2015,39 @@ static void msgpackPatchFunc(
   }
 }
 
+/*
+** msgpack_strip_nulls(blob) → BLOB
+** Recursively remove map keys whose value is nil, shrinking the encoded
+** size.  Array elements (including nil ones) are always preserved so
+** ordering/indices never change.  NULL → NULL; malformed msgpack → error.
+*/
+static void msgpackStripNullsFunc(
+  sqlite3_context *ctx, int argc, sqlite3_value **argv
+){
+  const u8 *a; u32 n;
+  (void)argc;
+  if( sqlite3_value_type(argv[0])==SQLITE_NULL ){
+    sqlite3_result_null(ctx); return;
+  }
+  if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ){
+    sqlite3_result_error(ctx, "msgpack_strip_nulls() requires a BLOB argument", -1); return;
+  }
+  a=(const u8*)sqlite3_value_blob(argv[0]); n=(u32)sqlite3_value_bytes(argv[0]);
+  if(n==0){ sqlite3_result_error(ctx, "invalid msgpack data", -1); return; }
+  MpBuf out; mpBufInit(&out,ctx);
+  int rc=mpStripNulls(&out, a,n,0, 0);
+  if(rc==SQLITE_OK){
+    u32 nOut; u8 *res=mpBufFinish(&out,&nOut);
+    if(res) sqlite3_result_blob(ctx,res,(int)nOut,sqlite3_free);
+  } else if(rc==SQLITE_NOMEM){
+    mpBufReset(&out);
+    sqlite3_result_error_nomem(ctx);
+  } else {
+    mpBufReset(&out);
+    sqlite3_result_error(ctx,"invalid msgpack data",-1);
+  }
+}
+
 
 
 /*
@@ -4454,6 +4583,10 @@ int sqlite3_msgpack_init(
          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
          0, msgpackPatchFunc, 0, 0, 0);
   if( rc ) return rc;
+  rc = sqlite3_create_function_v2(db, "msgpack_strip_nulls", 1,
+         SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
+         0, msgpackStripNullsFunc, 0, 0, 0);
+  if( rc ) return rc;
 
   /* Phase 5: Conversion */
   rc = sqlite3_create_function_v2(db, "msgpack_to_json", 1,

tests/fuzz_msgpack.c

@@ -38,6 +38,7 @@ static sqlite3_stmt *g_arrlen       = NULL;
 static sqlite3_stmt *g_errpos       = NULL;
 static sqlite3_stmt *g_quote        = NULL;
 static sqlite3_stmt *g_msgpack      = NULL;
+static sqlite3_stmt *g_strip_nulls  = NULL;
 
 /* 2-arg path variants: (?1 = blob, ?2 = path text) */
 static sqlite3_stmt *g_valid_path   = NULL;
@@ -142,6 +143,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) {
   prep(&g_errpos,   "SELECT msgpack_error_position(?1)");
   prep(&g_quote,    "SELECT msgpack_quote(?1)");
   prep(&g_msgpack,  "SELECT hex(msgpack(?1))");
+  prep(&g_strip_nulls, "SELECT hex(msgpack_strip_nulls(?1))");
 
   /* ── 2-arg path variants ────────────────────────────────────────── */
   prep(&g_valid_path,  "SELECT msgpack_valid(?1, ?2)");
@@ -263,6 +265,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   run_blob(g_errpos,  data, size);
   run_blob(g_quote,   data, size);
   run_blob(g_msgpack, data, size);
+  run_blob(g_strip_nulls, data, size);
 
   if (size <= 256) {
     run_blob(g_pretty, data, size);

tests/test_spec_p5_mutation.c

@@ -548,6 +548,89 @@ static void test_patch(sqlite3 *db){
   }
 }
 
+/* ── msgpack_strip_nulls ─────────────────────────────────────────── */
+
+static void test_strip_nulls(sqlite3 *db){
+  /* Top-level: nil-valued keys dropped, others preserved. */
+  { char *r = exec1(db,
+      "SELECT msgpack_to_json(msgpack_strip_nulls("
+      "  msgpack_object('a',1,'b',NULL,'c',3)))");
+    CHECK("9.1 strip_nulls: top-level nil key dropped",
+      r && strcmp(r,"{\"a\":1,\"c\":3}")==0); sqlite3_free(r);
+  }
+
+  /* Recursive: nil dropped inside a nested map too. */
+  { char *r = exec1(db,
+      "SELECT msgpack_to_json(msgpack_strip_nulls("
+      "  msgpack_object('a', msgpack_object('x',1,'y',NULL), 'b', NULL)))");
+    CHECK("9.2 strip_nulls: recursive nested map",
+      r && strcmp(r,"{\"a\":{\"x\":1}}")==0); sqlite3_free(r);
+  }
+
+  /* Array elements (including literal nulls) are preserved verbatim, but a
+  ** map nested inside an array still has its nil keys stripped. */
+  { char *r = exec1(db,
+      "SELECT msgpack_to_json(msgpack_strip_nulls("
+      "  msgpack_array(1, NULL, msgpack_object('p',1,'q',NULL))))");
+    CHECK("9.3 strip_nulls: array elements untouched, nested map stripped",
+      r && strcmp(r,"[1,null,{\"p\":1}]")==0); sqlite3_free(r);
+  }
+
+  /* All keys nil -> empty map, not removed entirely. */
+  { char *r = exec1(db,
+      "SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_object('a',NULL,'b',NULL)))");
+    CHECK("9.4 strip_nulls: all-nil map becomes {}",
+      r && strcmp(r,"{}")==0); sqlite3_free(r);
+  }
+
+  /* Top-level scalar (non-map) passes through unchanged. */
+  { char *r = exec1(db, "SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_quote(5)))");
+    CHECK("9.5 strip_nulls: scalar passthrough", r && strcmp(r,"5")==0); sqlite3_free(r);
+  }
+
+  /* SQL NULL input -> NULL. */
+  { sqlite3_stmt *s = NULL;
+    sqlite3_prepare_v2(db, "SELECT msgpack_strip_nulls(NULL) IS NULL", -1, &s, NULL);
+    sqlite3_step(s);
+    CHECK("9.6 strip_nulls: NULL input -> NULL", sqlite3_column_int(s,0)==1);
+    sqlite3_finalize(s);
+  }
+
+  /* Deeply/widely nested: nils stripped at every level, non-nil values and
+  ** array elements survive untouched. */
+  { char *r = exec1(db,
+      "SELECT msgpack_to_json(msgpack_strip_nulls("
+      "  msgpack_object("
+      "    'keep', 1,"
+      "    'drop', NULL,"
+      "    'nested', msgpack_object('a',NULL,'b',msgpack_array(NULL,2,msgpack_object('c',NULL,'d',4))))))");
+    CHECK("9.7 strip_nulls: deep nesting through map+array+map",
+      r && strcmp(r,"{\"keep\":1,\"nested\":{\"b\":[null,2,{\"d\":4}]}}")==0);
+    sqlite3_free(r);
+  }
+
+  /* Result is strictly smaller (or equal) in byte size after stripping. */
+  { sqlite3_stmt *s = NULL; sqlite3_int64 before=-1, after=-1;
+    sqlite3_prepare_v2(db,
+      "SELECT length(msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL,'e',NULL)),"
+      "       length(msgpack_strip_nulls(msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL,'e',NULL)))",
+      -1, &s, NULL);
+    if(sqlite3_step(s)==SQLITE_ROW){ before=sqlite3_column_int64(s,0); after=sqlite3_column_int64(s,1); }
+    CHECK("9.8 strip_nulls: shrinks encoded size", before>0 && after>0 && after<before);
+    sqlite3_finalize(s);
+  }
+
+  /* Malformed input (bogus MAP32 count) rejected, not crash. */
+  { sqlite3_stmt *s = NULL;
+    unsigned char bad32[] = { 0xdf, 0x80, 0x00, 0x00, 0x01, 0x31, 0x01, 0x02 };
+    sqlite3_prepare_v2(db, "SELECT msgpack_strip_nulls(?)", -1, &s, NULL);
+    sqlite3_bind_blob(s, 1, bad32, (int)sizeof bad32, SQLITE_STATIC);
+    int rc = sqlite3_step(s);
+    CHECK("9.9 strip_nulls: bogus MAP32 count rejected (no overflow)", rc == SQLITE_ERROR);
+    sqlite3_finalize(s);
+  }
+}
+
 /* ── immutability / original unmodified ─────────────────────────── */
 
 static void test_immutability(sqlite3 *db){
@@ -658,6 +741,7 @@ int main(void){
   test_remove(db);
   test_array_insert(db);
   test_patch(db);
+  test_strip_nulls(db);
   test_immutability(db);
   test_batch_fastpath(db);