diff --git a/README.md b/README.md index 7d818e6..6d3c26f 100644 --- a/README.md +++ b/README.md @@ -427,6 +427,28 @@ SELECT msgpack_to_json( -- {"a":1,"c":3} ``` +#### `msgpack_strip_nulls(mp)` + +Recursively removes every map key whose value is `nil`, shrinking the +encoded size. Recurses into nested maps, including maps nested inside +arrays. Array **elements** are always preserved (a literal `null` inside +an array stays), so array length/order never changes — only map keys are +dropped. + +```sql +SELECT msgpack_to_json( + msgpack_strip_nulls(msgpack_from_json('{"a":1,"b":null,"c":{"x":null,"y":2}}')) +); +-- {"a":1,"c":{"y":2}} + +SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_array(1, null, 3))); +-- [1,null,3] (array elements untouched) + +-- Shrinks storage for sparse wide records: +SELECT length(mp), length(msgpack_strip_nulls(mp)) +FROM (SELECT msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL) AS mp); +``` + --- ### JSON conversion diff --git a/src/msgpack.c b/src/msgpack.c index 9c845e6..4ee6a5f 100644 --- a/src/msgpack.c +++ b/src/msgpack.c @@ -29,6 +29,7 @@ ** msgpack_remove(mp, path, ...) -- remove element(s) ** msgpack_array_insert(mp, path, v) -- insert before index ** msgpack_patch(mp, patch) -- RFC 7386 merge-patch +** msgpack_strip_nulls(mp) -- recursively drop nil-valued map keys ** ** JSON conversion ** msgpack_from_json(json_text) -- JSON → msgpack @@ -1880,6 +1881,101 @@ static int mpMergePatch( return SQLITE_OK; } +/* +** mpStripNulls — recursively rebuild a msgpack value, dropping any map key +** whose value is MP_NIL. Recurses into nested maps and into maps/arrays +** nested inside arrays; array elements themselves are never dropped (only +** map keys), so array length/order is always preserved. Scalars pass +** through unchanged. +*/ +static int mpStripNulls( + MpBuf *out, + const u8 *a, u32 n, u32 i, + int depth +){ + if(i>=n) return SQLITE_ERROR; + if(depth>MP_MAX_DEPTH) return SQLITE_ERROR; + u8 b = a[i]; + int isMap = (b>=0x80&&b<=0x8f)||b==MP_MAP16||b==MP_MAP32; + int isArray = (b>=0x90&&b<=0x9f)||b==MP_ARRAY16||b==MP_ARRAY32; + + if(isMap){ + u32 count, dataOff; + if(b>=0x80&&b<=0x8f) { count=b&0x0f; dataOff=i+1; } + else if(b==MP_MAP16) { + if(i+3>n) return SQLITE_ERROR; + count=mpRead16(a+i+1); dataOff=i+3; + } else { + if(i+5>n) return SQLITE_ERROR; + count=mpRead32(a+i+1); dataOff=i+5; + } + /* Every pair needs >=2 bytes (>=1-byte key + >=1-byte value); reject a + ** declared count that cannot fit in the remaining buffer before doing + ** any work (guards against malformed/adversarial input). */ + if(count > (n-dataOff)/2) return SQLITE_ERROR; + + MpBuf tmp; mpBufInit(&tmp, out->pCtx); + u32 newCount=0, c=dataOff; + for(u32 k=0; k=n){ mpBufReset(&tmp); return SQLITE_ERROR; } + u32 valOff = mpSkipOne(a,n,c); + if(!valOff){ mpBufReset(&tmp); return SQLITE_ERROR; } + u32 pairEnd = mpSkipOne(a,n,valOff); + if(!pairEnd){ mpBufReset(&tmp); return SQLITE_ERROR; } + + if(a[valOff]!=MP_NIL){ + MpBuf mb; mpBufInit(&mb,out->pCtx); + int rc = mpStripNulls(&mb, a,n,valOff, depth+1); + if(rc!=SQLITE_OK){ mpBufReset(&mb); mpBufReset(&tmp); return rc; } + mpBufAppend(&tmp, a+c, valOff-c); /* key, verbatim */ + mpBufAppend(&tmp, mb.aBuf, mb.nUsed); /* stripped value */ + mpBufReset(&mb); + newCount++; + } + c = pairEnd; + } + if( tmp.bErr ){ mpBufReset(&tmp); return SQLITE_NOMEM; } + mpEncodeMapHeader(out, newCount); + mpBufAppend(out, tmp.aBuf, tmp.nUsed); + mpBufReset(&tmp); + return out->bErr ? SQLITE_NOMEM : SQLITE_OK; + + } else if(isArray){ + u32 count, dataOff; + if(b>=0x90&&b<=0x9f) { count=b&0x0f; dataOff=i+1; } + else if(b==MP_ARRAY16) { + if(i+3>n) return SQLITE_ERROR; + count=mpRead16(a+i+1); dataOff=i+3; + } else { + if(i+5>n) return SQLITE_ERROR; + count=mpRead32(a+i+1); dataOff=i+5; + } + /* Every element needs >=1 byte. */ + if(count > n-dataOff) return SQLITE_ERROR; + + /* Element count is unchanged (array elements are never dropped), so the + ** header can be written immediately and every element recursed straight + ** into out; on error the caller discards the whole buffer. */ + mpEncodeArrayHeader(out, count); + u32 c=dataOff; + for(u32 k=0; k=n) return SQLITE_ERROR; + int rc = mpStripNulls(out, a,n,c, depth+1); + if(rc!=SQLITE_OK) return rc; + u32 next = mpSkipOne(a,n,c); + if(!next) return SQLITE_ERROR; + c = next; + } + return out->bErr ? SQLITE_NOMEM : SQLITE_OK; + + } else { + u32 end = mpSkipOne(a,n,i); + if(!end) return SQLITE_ERROR; + mpBufAppend(out, a+i, end-i); + return out->bErr ? SQLITE_NOMEM : SQLITE_OK; + } +} + /* ---- SQL wrapper functions ---- */ static void msgpackSetFunc( sqlite3_context *ctx, int argc, sqlite3_value **argv @@ -1919,6 +2015,39 @@ static void msgpackPatchFunc( } } +/* +** msgpack_strip_nulls(blob) → BLOB +** Recursively remove map keys whose value is nil, shrinking the encoded +** size. Array elements (including nil ones) are always preserved so +** ordering/indices never change. NULL → NULL; malformed msgpack → error. +*/ +static void msgpackStripNullsFunc( + sqlite3_context *ctx, int argc, sqlite3_value **argv +){ + const u8 *a; u32 n; + (void)argc; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + sqlite3_result_null(ctx); return; + } + if( sqlite3_value_type(argv[0])!=SQLITE_BLOB ){ + sqlite3_result_error(ctx, "msgpack_strip_nulls() requires a BLOB argument", -1); return; + } + a=(const u8*)sqlite3_value_blob(argv[0]); n=(u32)sqlite3_value_bytes(argv[0]); + if(n==0){ sqlite3_result_error(ctx, "invalid msgpack data", -1); return; } + MpBuf out; mpBufInit(&out,ctx); + int rc=mpStripNulls(&out, a,n,0, 0); + if(rc==SQLITE_OK){ + u32 nOut; u8 *res=mpBufFinish(&out,&nOut); + if(res) sqlite3_result_blob(ctx,res,(int)nOut,sqlite3_free); + } else if(rc==SQLITE_NOMEM){ + mpBufReset(&out); + sqlite3_result_error_nomem(ctx); + } else { + mpBufReset(&out); + sqlite3_result_error(ctx,"invalid msgpack data",-1); + } +} + /* @@ -4454,6 +4583,10 @@ int sqlite3_msgpack_init( SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, msgpackPatchFunc, 0, 0, 0); if( rc ) return rc; + rc = sqlite3_create_function_v2(db, "msgpack_strip_nulls", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, msgpackStripNullsFunc, 0, 0, 0); + if( rc ) return rc; /* Phase 5: Conversion */ rc = sqlite3_create_function_v2(db, "msgpack_to_json", 1, diff --git a/tests/fuzz_msgpack.c b/tests/fuzz_msgpack.c index 3d3b883..937f0f9 100644 --- a/tests/fuzz_msgpack.c +++ b/tests/fuzz_msgpack.c @@ -38,6 +38,7 @@ static sqlite3_stmt *g_arrlen = NULL; static sqlite3_stmt *g_errpos = NULL; static sqlite3_stmt *g_quote = NULL; static sqlite3_stmt *g_msgpack = NULL; +static sqlite3_stmt *g_strip_nulls = NULL; /* 2-arg path variants: (?1 = blob, ?2 = path text) */ static sqlite3_stmt *g_valid_path = NULL; @@ -142,6 +143,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) { prep(&g_errpos, "SELECT msgpack_error_position(?1)"); prep(&g_quote, "SELECT msgpack_quote(?1)"); prep(&g_msgpack, "SELECT hex(msgpack(?1))"); + prep(&g_strip_nulls, "SELECT hex(msgpack_strip_nulls(?1))"); /* ── 2-arg path variants ────────────────────────────────────────── */ prep(&g_valid_path, "SELECT msgpack_valid(?1, ?2)"); @@ -263,6 +265,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { run_blob(g_errpos, data, size); run_blob(g_quote, data, size); run_blob(g_msgpack, data, size); + run_blob(g_strip_nulls, data, size); if (size <= 256) { run_blob(g_pretty, data, size); diff --git a/tests/test_spec_p5_mutation.c b/tests/test_spec_p5_mutation.c index c12b4f3..3ab5377 100644 --- a/tests/test_spec_p5_mutation.c +++ b/tests/test_spec_p5_mutation.c @@ -548,6 +548,89 @@ static void test_patch(sqlite3 *db){ } } +/* ── msgpack_strip_nulls ─────────────────────────────────────────── */ + +static void test_strip_nulls(sqlite3 *db){ + /* Top-level: nil-valued keys dropped, others preserved. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_object('a',1,'b',NULL,'c',3)))"); + CHECK("9.1 strip_nulls: top-level nil key dropped", + r && strcmp(r,"{\"a\":1,\"c\":3}")==0); sqlite3_free(r); + } + + /* Recursive: nil dropped inside a nested map too. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_object('a', msgpack_object('x',1,'y',NULL), 'b', NULL)))"); + CHECK("9.2 strip_nulls: recursive nested map", + r && strcmp(r,"{\"a\":{\"x\":1}}")==0); sqlite3_free(r); + } + + /* Array elements (including literal nulls) are preserved verbatim, but a + ** map nested inside an array still has its nil keys stripped. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_array(1, NULL, msgpack_object('p',1,'q',NULL))))"); + CHECK("9.3 strip_nulls: array elements untouched, nested map stripped", + r && strcmp(r,"[1,null,{\"p\":1}]")==0); sqlite3_free(r); + } + + /* All keys nil -> empty map, not removed entirely. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_object('a',NULL,'b',NULL)))"); + CHECK("9.4 strip_nulls: all-nil map becomes {}", + r && strcmp(r,"{}")==0); sqlite3_free(r); + } + + /* Top-level scalar (non-map) passes through unchanged. */ + { char *r = exec1(db, "SELECT msgpack_to_json(msgpack_strip_nulls(msgpack_quote(5)))"); + CHECK("9.5 strip_nulls: scalar passthrough", r && strcmp(r,"5")==0); sqlite3_free(r); + } + + /* SQL NULL input -> NULL. */ + { sqlite3_stmt *s = NULL; + sqlite3_prepare_v2(db, "SELECT msgpack_strip_nulls(NULL) IS NULL", -1, &s, NULL); + sqlite3_step(s); + CHECK("9.6 strip_nulls: NULL input -> NULL", sqlite3_column_int(s,0)==1); + sqlite3_finalize(s); + } + + /* Deeply/widely nested: nils stripped at every level, non-nil values and + ** array elements survive untouched. */ + { char *r = exec1(db, + "SELECT msgpack_to_json(msgpack_strip_nulls(" + " msgpack_object(" + " 'keep', 1," + " 'drop', NULL," + " 'nested', msgpack_object('a',NULL,'b',msgpack_array(NULL,2,msgpack_object('c',NULL,'d',4))))))"); + CHECK("9.7 strip_nulls: deep nesting through map+array+map", + r && strcmp(r,"{\"keep\":1,\"nested\":{\"b\":[null,2,{\"d\":4}]}}")==0); + sqlite3_free(r); + } + + /* Result is strictly smaller (or equal) in byte size after stripping. */ + { sqlite3_stmt *s = NULL; sqlite3_int64 before=-1, after=-1; + sqlite3_prepare_v2(db, + "SELECT length(msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL,'e',NULL))," + " length(msgpack_strip_nulls(msgpack_object('a',1,'b',NULL,'c',NULL,'d',NULL,'e',NULL)))", + -1, &s, NULL); + if(sqlite3_step(s)==SQLITE_ROW){ before=sqlite3_column_int64(s,0); after=sqlite3_column_int64(s,1); } + CHECK("9.8 strip_nulls: shrinks encoded size", before>0 && after>0 && after