From 0e167edf4cac86f2fc3d2e7579618bf84aaf72d9 Mon Sep 17 00:00:00 2001 From: Affan Khan Date: Thu, 2 Jul 2026 13:16:17 -0400 Subject: [PATCH] feat: add msgpack_strip_nulls() to shrink blobs by dropping nil map keys New SQL function msgpack_strip_nulls(mp) recursively rebuilds a msgpack value, dropping any map key whose value is nil. Recurses into nested maps, including maps nested inside array elements. Array elements themselves (even literal nulls) are always preserved, so array length/order never changes -- only map keys are dropped. Scalars and non-map/array roots pass through unchanged. NULL input -> NULL; malformed msgpack -> error (never a crash). Useful for shrinking sparse wide records (e.g. many optional columns encoded as msgpack_object(...) with NULL for absent fields) before storing them. Implementation mirrors the mpMergePatch rebuild style: a recursive mpStripNulls() walks the source buffer, buffering filtered map pairs in a temp MpBuf before writing the final header (since the pair count isn't known until nil keys are excluded), and appending array elements directly since their count never changes. Map/array header counts are bounds-checked against the remaining buffer before use, consistent with the mpMergePatch fix in the previous release. Testing: - 9 new regression checks (9.1-9.9) in test_spec_p5_mutation.c: top-level and recursive nil-stripping, array-element preservation (including a nested map inside an array), all-nil map -> {}, scalar passthrough, NULL -> NULL, deep multi-level nesting, size-shrink assertion, and a malformed MAP32-count rejection check. - Added msgpack_strip_nulls to the fuzz_msgpack.c harness so it's exercised by both fuzz_corpus (ctest) and ad-hoc libFuzzer runs. - Full ctest suite: 16/16 passing. - 45s libFuzzer+ASan run across the whole harness (now including msgpack_strip_nulls): 102,261 execs, 0 crashes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 22 ++++++ src/msgpack.c | 133 ++++++++++++++++++++++++++++++++++ tests/fuzz_msgpack.c | 3 + tests/test_spec_p5_mutation.c | 84 +++++++++++++++++++++ 4 files changed, 242 insertions(+) diff --git a/README.md b/README.md index 7d818e6..6d3c26f 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,28 @@ SELECT msgpack_to_json( -- {"a":1,"c":3} ``` +#### `msgpack_strip_nulls(mp)` + +Recursively removes every map key whose value is `nil`, shrinking the +encoded size. Recurses into nested maps, including maps nested inside +arrays. Array **elements** are always preserved (a literal `null` inside +an array stays), so array length/order never changes — only map keys are +dropped. + +```sql +SELECT msgpack_to_json( + msgpack_strip_nulls(msgpack_from_json('{"a":1,"b":null,"c":{"x":null,"y":2}}')) +); +-- {"a":1,"c":{"y":2}} + +SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_array(1, null, 3))); +-- [1,null,3] (array elements untouched) + +-- Shrinks storage for sparse wide records: +SELECT length(mp), length(msgpack_strip_nulls(mp)) +FROM (SELECT msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL) AS mp); +``` + --- ### JSON conversion diff --git a/src/msgpack.c b/src/msgpack.c index 9c845e6..4ee6a5f 100644 --- a/src/msgpack.c +++ b/src/msgpack.c @@ -29,6 +29,7 @@ ** msgpack_remove(mp, path, ...) -- remove element(s) ** msgpack_array_insert(mp, path, v) -- insert before index ** msgpack_patch(mp, patch) -- RFC 7386 merge-patch +** msgpack_strip_nulls(mp) -- recursively drop nil-valued map keys ** ** JSON conversion ** msgpack_from_json(json_text) -- JSON → msgpack @@ -1880,6 +1881,101 @@ static int mpMergePatch( return SQLITE_OK; } +/* +** mpStripNulls — recursively rebuild a msgpack value, dropping any map key +** whose value is MP_NIL. Recurses into nested maps and into maps/arrays +** nested inside arrays; array elements themselves are never dropped (only +** map keys), so array length/order is always preserved. Scalars pass +** through unchanged. +*/ +static int mpStripNulls( + MpBuf *out, + const u8 *a, u32 n, u32 i, + int depth +){ + if(i>=n) return SQLITE_ERROR; + if(depth>MP_MAX_DEPTH) return SQLITE_ERROR; + u8 b = a[i]; + int isMap = (b>=0x80&&b<=0x8f)||b==MP_MAP16||b==MP_MAP32; + int isArray = (b>=0x90&&b<=0x9f)||b==MP_ARRAY16||b==MP_ARRAY32; + + if(isMap){ + u32 count, dataOff; + if(b>=0x80&&b<=0x8f) { count=b&0x0f; dataOff=i+1; } + else if(b==MP_MAP16) { + if(i+3>n) return SQLITE_ERROR; + count=mpRead16(a+i+1); dataOff=i+3; + } else { + if(i+5>n) return SQLITE_ERROR; + count=mpRead32(a+i+1); dataOff=i+5; + } + /* Every pair needs >=2 bytes (>=1-byte key + >=1-byte value); reject a + ** declared count that cannot fit in the remaining buffer before doing + ** any work (guards against malformed/adversarial input). */ + if(count > (n-dataOff)/2) return SQLITE_ERROR; + + MpBuf tmp; mpBufInit(&tmp, out->pCtx); + u32 newCount=0, c=dataOff; + for(u32 k=0; k=n){ mpBufReset(&tmp); return SQLITE_ERROR; } + u32 valOff = mpSkipOne(a,n,c); + if(!valOff){ mpBufReset(&tmp); return SQLITE_ERROR; } + u32 pairEnd = mpSkipOne(a,n,valOff); + if(!pairEnd){ mpBufReset(&tmp); return SQLITE_ERROR; } + + if(a[valOff]!=MP_NIL){ + MpBuf mb; mpBufInit(&mb,out->pCtx); + int rc = mpStripNulls(&mb, a,n,valOff, depth+1); + if(rc!=SQLITE_OK){ mpBufReset(&mb); mpBufReset(&tmp); return rc; } + mpBufAppend(&tmp, a+c, valOff-c); /* key, verbatim */ + mpBufAppend(&tmp, mb.aBuf, mb.nUsed); /* stripped value */ + mpBufReset(&mb); + newCount++; + } + c = pairEnd; + } + if( tmp.bErr ){ mpBufReset(&tmp); return SQLITE_NOMEM; } + mpEncodeMapHeader(out, newCount); + mpBufAppend(out, tmp.aBuf, tmp.nUsed); + mpBufReset(&tmp); + return out->bErr ? SQLITE_NOMEM : SQLITE_OK; + + } else if(isArray){ + u32 count, dataOff; + if(b>=0x90&&b<=0x9f) { count=b&0x0f; dataOff=i+1; } + else if(b==MP_ARRAY16) { + if(i+3>n) return SQLITE_ERROR; + count=mpRead16(a+i+1); dataOff=i+3; + } else { + if(i+5>n) return SQLITE_ERROR; + count=mpRead32(a+i+1); dataOff=i+5; + } + /* Every element needs >=1 byte. */ + if(count > n-dataOff) return SQLITE_ERROR; + + /* Element count is unchanged (array elements are never dropped), so the + ** header can be written immediately and every element recursed straight + ** into out; on error the caller discards the whole buffer. */ + mpEncodeArrayHeader(out, count); + u32 c=dataOff; + for(u32 k=0; k=n) return SQLITE_ERROR; + int rc = mpStripNulls(out, a,n,c, depth+1); + if(rc!=SQLITE_OK) return rc; + u32 next = mpSkipOne(a,n,c); + if(!next) return SQLITE_ERROR; + c = next; + } + return out->bErr ? SQLITE_NOMEM : SQLITE_OK; + + } else { + u32 end = mpSkipOne(a,n,i); + if(!end) return SQLITE_ERROR; + mpBufAppend(out, a+i, end-i); + return out->bErr ? SQLITE_NOMEM : SQLITE_OK; + } +} + /* ---- SQL wrapper functions ---- */ static void msgpackSetFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv @@ -1919,6 +2015,39 @@ static void msgpackPatchFunc( } } +/* +** msgpack_strip_nulls(blob) → BLOB +** Recursively remove map keys whose value is nil, shrinking the encoded +** size. Array elements (including nil ones) are always preserved so +** ordering/indices never change. NULL → NULL; malformed msgpack → error. +*/ +static void msgpackStripNullsFunc( + sqlite3_context *ctx, int argc, sqlite3_value **argv +){ + const u8 *a; u32 n; + (void)argc; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + sqlite3_result_null(ctx); return; + } + if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ){ + sqlite3_result_error(ctx, "msgpack_strip_nulls() requires a BLOB argument", -1); return; + } + a=(const u8*)sqlite3_value_blob(argv[0]); n=(u32)sqlite3_value_bytes(argv[0]); + if(n==0){ sqlite3_result_error(ctx, "invalid msgpack data", -1); return; } + MpBuf out; mpBufInit(&out,ctx); + int rc=mpStripNulls(&out, a,n,0, 0); + if(rc==SQLITE_OK){ + u32 nOut; u8 *res=mpBufFinish(&out,&nOut); + if(res) sqlite3_result_blob(ctx,res,(int)nOut,sqlite3_free); + } else if(rc==SQLITE_NOMEM){ + mpBufReset(&out); + sqlite3_result_error_nomem(ctx); + } else { + mpBufReset(&out); + sqlite3_result_error(ctx,"invalid msgpack data",-1); + } +} + /* @@ -4454,6 +4583,10 @@ int sqlite3_msgpack_init( SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, msgpackPatchFunc, 0, 0, 0); if( rc ) return rc; + rc = sqlite3_create_function_v2(db, "msgpack_strip_nulls", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, msgpackStripNullsFunc, 0, 0, 0); + if( rc ) return rc; /* Phase 5: Conversion */ rc = sqlite3_create_function_v2(db, "msgpack_to_json", 1, diff --git a/tests/fuzz_msgpack.c b/tests/fuzz_msgpack.c index 3d3b883..937f0f9 100644 --- a/tests/fuzz_msgpack.c +++ b/tests/fuzz_msgpack.c @@ -38,6 +38,7 @@ static sqlite3_stmt *g_arrlen = NULL; static sqlite3_stmt *g_errpos = NULL; static sqlite3_stmt *g_quote = NULL; static sqlite3_stmt *g_msgpack = NULL; +static sqlite3_stmt *g_strip_nulls = NULL; /* 2-arg path variants: (?1 = blob, ?2 = path text) */ static sqlite3_stmt *g_valid_path = NULL; @@ -142,6 +143,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) { prep(&g_errpos, "SELECT msgpack_error_position(?1)"); prep(&g_quote, "SELECT msgpack_quote(?1)"); prep(&g_msgpack, "SELECT hex(msgpack(?1))"); + prep(&g_strip_nulls, "SELECT hex(msgpack_strip_nulls(?1))"); /* ── 2-arg path variants ────────────────────────────────────────── */ prep(&g_valid_path, "SELECT msgpack_valid(?1, ?2)"); @@ -263,6 +265,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { run_blob(g_errpos, data, size); run_blob(g_quote, data, size); run_blob(g_msgpack, data, size); + run_blob(g_strip_nulls, data, size); if (size <= 256) { run_blob(g_pretty, data, size); diff --git a/tests/test_spec_p5_mutation.c b/tests/test_spec_p5_mutation.c index c12b4f3..3ab5377 100644 --- a/tests/test_spec_p5_mutation.c +++ b/tests/test_spec_p5_mutation.c @@ -548,6 +548,89 @@ static void test_patch(sqlite3 *db){ } } +/* ── msgpack_strip_nulls ─────────────────────────────────────────── */ + +static void test_strip_nulls(sqlite3 *db){ + /* Top-level: nil-valued keys dropped, others preserved. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_object('a',1,'b',NULL,'c',3)))"); + CHECK("9.1 strip_nulls: top-level nil key dropped", + r && strcmp(r,"{\"a\":1,\"c\":3}")==0); sqlite3_free(r); + } + + /* Recursive: nil dropped inside a nested map too. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_object('a', msgpack_object('x',1,'y',NULL), 'b', NULL)))"); + CHECK("9.2 strip_nulls: recursive nested map", + r && strcmp(r,"{\"a\":{\"x\":1}}")==0); sqlite3_free(r); + } + + /* Array elements (including literal nulls) are preserved verbatim, but a + ** map nested inside an array still has its nil keys stripped. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_array(1, NULL, msgpack_object('p',1,'q',NULL))))"); + CHECK("9.3 strip_nulls: array elements untouched, nested map stripped", + r && strcmp(r,"[1,null,{\"p\":1}]")==0); sqlite3_free(r); + } + + /* All keys nil -> empty map, not removed entirely. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_object('a',NULL,'b',NULL)))"); + CHECK("9.4 strip_nulls: all-nil map becomes {}", + r && strcmp(r,"{}")==0); sqlite3_free(r); + } + + /* Top-level scalar (non-map) passes through unchanged. */ + { char *r = exec1(db, "SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_quote(5)))"); + CHECK("9.5 strip_nulls: scalar passthrough", r && strcmp(r,"5")==0); sqlite3_free(r); + } + + /* SQL NULL input -> NULL. */ + { sqlite3_stmt *s = NULL; + sqlite3_prepare_v2(db, "SELECT msgpack_strip_nulls(NULL) IS NULL", -1, &s, NULL); + sqlite3_step(s); + CHECK("9.6 strip_nulls: NULL input -> NULL", sqlite3_column_int(s,0)==1); + sqlite3_finalize(s); + } + + /* Deeply/widely nested: nils stripped at every level, non-nil values and + ** array elements survive untouched. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_object(" + " 'keep', 1," + " 'drop', NULL," + " 'nested', msgpack_object('a',NULL,'b',msgpack_array(NULL,2,msgpack_object('c',NULL,'d',4))))))"); + CHECK("9.7 strip_nulls: deep nesting through map+array+map", + r && strcmp(r,"{\"keep\":1,\"nested\":{\"b\":[null,2,{\"d\":4}]}}")==0); + sqlite3_free(r); + } + + /* Result is strictly smaller (or equal) in byte size after stripping. */ + { sqlite3_stmt *s = NULL; sqlite3_int64 before=-1, after=-1; + sqlite3_prepare_v2(db, + "SELECT length(msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL,'e',NULL))," + " length(msgpack_strip_nulls(msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL,'e',NULL)))", + -1, &s, NULL); + if(sqlite3_step(s)==SQLITE_ROW){ before=sqlite3_column_int64(s,0); after=sqlite3_column_int64(s,1); } + CHECK("9.8 strip_nulls: shrinks encoded size", before>0 && after>0 && after