From e0ebb7771ec4c058c42d5602f1a982e1ac2b539e Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Fri, 11 Sep 2026 18:04:57 +0200 Subject: [PATCH 01/28] fix: harden sync error handling and bump version to 1.1.4 --- Makefile | 23 ++- docs/internal/audit-regressions.md | 74 +++++++ modules/fractional-indexing | 2 +- packages/node/src/platform.test.ts | 34 +++ packages/node/src/platform.ts | 8 +- src/block.c | 38 +++- src/cloudsync.c | 251 +++++++++++++++-------- src/cloudsync.h | 7 +- src/database.h | 3 +- src/network/network.c | 133 +++++++++--- src/network/network_private.h | 6 + src/pk.c | 11 +- src/postgresql/cloudsync_postgresql.c | 134 +----------- src/postgresql/database_postgresql.c | 22 +- src/sqlite/cloudsync_changes_sqlite.c | 14 +- src/sqlite/cloudsync_sqlite.c | 137 +------------ test/network_unit.c | 64 +++++- test/postgresql/57_audit_regressions.sql | 78 +++++++ test/postgresql/full_test.sql | 1 + test/review_regressions.c | 183 +++++++++++++++++ test/unit.c | 98 ++++++--- 21 files changed, 859 insertions(+), 462 deletions(-) create mode 100644 docs/internal/audit-regressions.md create mode 100644 packages/node/src/platform.test.ts create mode 100644 test/postgresql/57_audit_regressions.sql create mode 100644 test/review_regressions.c diff --git a/Makefile b/Makefile index f9d1acf1..d22a2314 100644 --- a/Makefile +++ b/Makefile @@ -93,10 +93,9 @@ TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC))) # tested directly on in-memory buffers. NT_LDFLAGS reuses the platform LDFLAGS # (which carries -lcurl) minus the shared-library-only flags (-shared on Linux, # -dynamiclib on macOS) so it links as an executable, plus the test link libs. -# -undefined dynamic_lookup is kept: the test never opens a connection, so curl's -# transport symbols are linked but never invoked. +# The deadline regression uses a loopback socket; no external service is needed. BUILD_NETTEST = build/nettest -NT_CFLAGS = $(filter-out -DCLOUDSYNC_OMIT_NETWORK,$(T_CFLAGS)) +NT_CFLAGS = $(filter-out -DCLOUDSYNC_OMIT_NETWORK,$(T_CFLAGS)) -DCLOUDSYNC_REQUEST_TIMEOUT_SECONDS=1L -DCLOUDSYNC_CONNECT_TIMEOUT_SECONDS=1L NT_LDFLAGS = $(filter-out -shared -dynamiclib -headerpad_max_install_names,$(LDFLAGS)) $(T_LDFLAGS) NT_SRC = $(SRC_FILES) $(SQLITE_DIR)/sqlite3.c $(TEST_DIR)/network_unit.c NT_OBJ = $(patsubst %.c,$(BUILD_NETTEST)/%.o,$(notdir $(NT_SRC))) @@ -298,8 +297,22 @@ ifneq ($(COVERAGE),false) endif # Run only unit tests -unittest: $(TARGET) $(DIST_DIR)/unit$(EXE) +unittest: $(TARGET) $(DIST_DIR)/unit$(EXE) $(DIST_DIR)/review_regressions$(EXE) @./$(DIST_DIR)/unit$(EXE) + @./$(DIST_DIR)/review_regressions$(EXE) + +# Force pk.c's endian-conversion branch while preserving the real host ABI. +# This catches double conversion regressions even on a little-endian CI host. +$(BUILD_TEST)/pk_forced_big_endian.o: $(SRC_DIR)/pk.c $(SRC_DIR)/cloudsync_endian.h + @mkdir -p $(BUILD_TEST) + $(CC) $(T_CFLAGS) -U__BYTE_ORDER__ -D__BYTE_ORDER__=__ORDER_BIG_ENDIAN__ -c $< -o $@ + +$(DIST_DIR)/review_regressions_big_endian$(EXE): $(TEST_OBJ) $(BUILD_TEST)/pk_forced_big_endian.o + $(CC) $(filter-out $(BUILD_TEST)/pk.o $(patsubst %.c,$(BUILD_TEST)/%.o,$(notdir $(TEST_SRC))),$(TEST_OBJ)) $(BUILD_TEST)/review_regressions.o $(BUILD_TEST)/pk_forced_big_endian.o -o $@ $(T_LDFLAGS) + +.PHONY: endian-unittest +endian-unittest: $(DIST_DIR)/review_regressions_big_endian$(EXE) + @./$(DIST_DIR)/review_regressions_big_endian$(EXE) # Network-enabled unit test binary. Link it via a file rule (like dist/unit), not in # the run recipe below: on Android `make test` runs binaries on the emulator from a @@ -308,7 +321,7 @@ unittest: $(TARGET) $(DIST_DIR)/unit$(EXE) $(DIST_DIR)/network_unit$(EXE): $(CURL_LIB) $(NT_OBJ) $(CC) $(NT_OBJ) -o $@ $(NT_LDFLAGS) -# Run the network-layer unit tests (networking compiled in, no server) +# Run the network-layer unit tests (networking compiled in, loopback only) network-unittest: $(DIST_DIR)/network_unit$(EXE) @./$(DIST_DIR)/network_unit$(EXE) diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md new file mode 100644 index 00000000..f90af4f1 --- /dev/null +++ b/docs/internal/audit-regressions.md @@ -0,0 +1,74 @@ +# Repository audit fixes + +The previously reported `MAX_PARAMS` issue is outside this change. + +## Compatibility and limits + +- PK doubles retain their deployed little-endian IEEE754 bytes. The old code + combined host conversion with manual big-endian serialization; unconditional + byte swapping now makes the historical format explicit on every architecture. + Integers keep their existing encoding. No migration is needed for supported + little-endian deployments. +- Decompressed payloads are limited to 256 MiB before allocation. Builds may + override `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE`; LZ4's `INT_MAX` bound still + applies. Existing default chunk sizes are below this limit. Oversized legacy + monolithic payloads must be rechunked or used with an explicitly raised limit. +- Curl requests now have a 30-second connection deadline and a 300-second total + deadline, including reused handles. Build overrides are + `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. +- Failed payload writes report an error and do not advance the receive cursor. + PostgreSQL's explicit RLS WITH CHECK rejection remains a skippable policy + outcome, distinct from generic SQL/permission errors; the call still reports + processed rows, but does not advance the cursor when a policy denied rows. + SQLite retains its existing per-group partial-application behavior. +- Test database files live in a private per-run temporary directory, not HOME. + The harness deletes only its own flat test files at shutdown. + +## Regression coverage + +| Area | Focused coverage | +| --- | --- | +| 64-bit clocks | Incoming column/database versions, causal length and sequence above UINT32_MAX | +| Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; forced big-endian conversion build | +| Virtual-table planner | Unusable/unsupported constraints before an accepted constraint; no accepted constraints | +| Payload failures | First, middle and final PK errors; checkpoint unchanged; allocation limits | +| Metadata refill | Trigger rejects insertion of a missing column clock | +| Block LWW | Insert/update rollback on block write failure; allocation failure at each split/list/diff allocation | +| PostgreSQL ownership | Block failure while another SPI cursor is active; no invalid tuple-table cleanup | +| JSON | Root-only member lookup, string values resembling keys, nested keys, Unicode and invalid surrogates | +| Curl | Stalled loopback HTTP server, unpooled handle and pooled handle before/after reset | +| Node | ia32 rejection on all OS families and preservation of x64/arm64-musl selection | +| Fractional indexing | 4096-byte common prefix plus the existing module suite | +| Test harness | Temporary directory cleanup, memory accounting and sanitizer-safe RowID generation | + +Run `make unittest`, `make endian-unittest`, `make network-unittest`, and +`make -C modules/fractional-indexing/test run`. The network test needs permission +to bind a loopback socket; it does not contact an external service. Run Node +checks from `packages/node` with `npm test -- --run`, `npm run typecheck`, and +`npm run build`. + +PostgreSQL's `test/postgresql/full_test.sql` includes the focused audit cases in +`57_audit_regressions.sql`. Run it only against a disposable PostgreSQL instance: +the existing suite creates and drops its test databases. + +The fractional-indexing changes and its new test are inside a Git submodule; +they must be recorded there before updating the parent repository's submodule +pointer when preparing a commit. + +## Validation performed (2026-09-11) + +- macOS arm64: all 150 existing unit checks and the new audit regressions passed. +- The same SQLite suites passed AddressSanitizer and UndefinedBehaviorSanitizer + with `UBSAN_OPTIONS=halt_on_error=1`, including zero outstanding SQLite memory. +- The forced big-endian conversion regression build passed. +- All 7 network tests and all 340 fractional-indexing module tests passed. +- PostgreSQL 17, rebuilt in an isolated Linux container: 479 reported checks + passed across 57 test groups, with zero failures and no SPI cleanup warnings. +- Node: 14 tests, TypeScript checking, and CJS/ESM/declaration builds passed. +- PostgreSQL migration compatibility check and Git whitespace checks passed. + +No live cloud-service integration was run. Windows, Android and WebAssembly +runtime suites were not executed. The x86_64 runtime attempt was unavailable on +this host (`Bad CPU type in executable`, Rosetta not available). The endian +test exercises conversion logic; it is not a substitute for real big-endian +hardware testing. diff --git a/modules/fractional-indexing b/modules/fractional-indexing index b9af0ec5..ddaf4147 160000 --- a/modules/fractional-indexing +++ b/modules/fractional-indexing @@ -1 +1 @@ -Subproject commit b9af0ec5b818bca29919e1a8d42b142feb71f269 +Subproject commit ddaf4147101462b7062c549f3fad82fe9775e645 diff --git a/packages/node/src/platform.test.ts b/packages/node/src/platform.test.ts new file mode 100644 index 00000000..f5949737 --- /dev/null +++ b/packages/node/src/platform.test.ts @@ -0,0 +1,34 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { arch, platform } from 'node:os'; +import { existsSync } from 'node:fs'; +import { getCurrentPlatform, getPlatformPackageName } from './platform'; + +vi.mock('node:os', () => ({ arch: vi.fn(), platform: vi.fn() })); +vi.mock('node:fs', () => ({ existsSync: vi.fn(), readFileSync: vi.fn() })); +vi.mock('node:child_process', () => ({ execSync: vi.fn(() => 'glibc') })); + +describe('native binary architecture selection', () => { + beforeEach(() => { + vi.mocked(existsSync).mockReturnValue(false); + vi.spyOn(process.report, 'getReport').mockReturnValue({ + header: { glibcVersionRuntime: '2.38' }, + } as ReturnType); + }); + afterEach(() => vi.restoreAllMocks()); + it.each(['darwin', 'linux', 'win32'] as const)('rejects ia32 on %s', (os) => { + vi.mocked(platform).mockReturnValue(os); + vi.mocked(arch).mockReturnValue('ia32'); + expect(() => getCurrentPlatform()).toThrow(`Unsupported platform: ${os}-ia32`); + }); + it.each(['darwin', 'linux', 'win32'] as const)('keeps x64 support on %s', (os) => { + vi.mocked(platform).mockReturnValue(os); + vi.mocked(arch).mockReturnValue('x64'); + expect(getPlatformPackageName()).toBe(`@sqliteai/sqlite-sync-${os}-x86_64`); + }); + it('keeps Linux arm64 musl support', () => { + vi.mocked(platform).mockReturnValue('linux'); + vi.mocked(arch).mockReturnValue('arm64'); + vi.mocked(existsSync).mockReturnValue(true); + expect(getCurrentPlatform()).toBe('linux-arm64-musl'); + }); +}); diff --git a/packages/node/src/platform.ts b/packages/node/src/platform.ts index f40146f0..dee4fb53 100644 --- a/packages/node/src/platform.ts +++ b/packages/node/src/platform.ts @@ -97,24 +97,24 @@ export function getCurrentPlatform(): Platform { // macOS if (platformName === 'darwin') { if (archName === 'arm64') return 'darwin-arm64'; - if (archName === 'x64' || archName === 'ia32') return 'darwin-x86_64'; + if (archName === 'x64') return 'darwin-x86_64'; } // Linux (with musl detection) - if (platformName === 'linux') { + if (platformName === 'linux' && (archName === 'arm64' || archName === 'x64')) { const muslSuffix = isMusl() ? '-musl' : ''; if (archName === 'arm64') { return `linux-arm64${muslSuffix}` as Platform; } - if (archName === 'x64' || archName === 'ia32') { + if (archName === 'x64') { return `linux-x86_64${muslSuffix}` as Platform; } } // Windows if (platformName === 'win32') { - if (archName === 'x64' || archName === 'ia32') return 'win32-x86_64'; + if (archName === 'x64') return 'win32-x86_64'; } // Unsupported platform diff --git a/src/block.c b/src/block.c index dd99b266..7087b46d 100644 --- a/src/block.c +++ b/src/block.c @@ -65,7 +65,11 @@ static bool block_list_append(block_list_t *list, const char *content, size_t co block_entry_t *e = &list->entries[list->count]; e->content = cloudsync_string_ndup(content, content_len); e->position_id = position_id ? cloudsync_string_dup(position_id) : NULL; - if (!e->content) return false; + if (!e->content || (position_id && !e->position_id)) { + cloudsync_memory_free(e->content); + cloudsync_memory_free(e->position_id); + return false; + } list->count++; return true; } @@ -95,14 +99,21 @@ block_list_t *block_split(const char *text, const char *delimiter) { if (!text || !*text) { // Empty text produces a single empty block - block_list_append(list, "", 0, NULL); + if (!block_list_append(list, "", 0, NULL)) { + block_list_free(list); + return NULL; + } return list; } + if (!delimiter) delimiter = BLOCK_DEFAULT_DELIMITER; size_t dlen = strlen(delimiter); if (dlen == 0) { // No delimiter: entire text is one block - block_list_append(list, text, strlen(text), NULL); + if (!block_list_append(list, text, strlen(text), NULL)) { + block_list_free(list); + return NULL; + } return list; } @@ -176,6 +187,11 @@ static bool block_diff_append(block_diff_t *diff, block_diff_type type, const ch e->type = type; e->position_id = cloudsync_string_dup(position_id); e->content = content ? cloudsync_string_dup(content) : NULL; + if (!e->position_id || (content && !e->content)) { + cloudsync_memory_free(e->position_id); + cloudsync_memory_free(e->content); + return false; + } diff->count++; return true; } @@ -228,7 +244,7 @@ block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, // Exact match — mark any skipped old blocks as REMOVED for (int si = old_scan; si < oi; si++) { if (!old_consumed[si]) { - block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[si].position_id, NULL); + if (!block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[si].position_id, NULL)) goto fail; old_consumed[si] = true; } } @@ -252,10 +268,12 @@ block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, } char *new_pos = block_position_between(last_position, next_pos); - if (new_pos) { - block_diff_append(diff, BLOCK_DIFF_ADDED, new_pos, new_parts[ni]); - last_position = diff->entries[diff->count - 1].position_id; + if (!new_pos) goto fail; + { + bool appended = block_diff_append(diff, BLOCK_DIFF_ADDED, new_pos, new_parts[ni]); cloudsync_memory_free(new_pos); + if (!appended) goto fail; + last_position = diff->entries[diff->count - 1].position_id; } } } @@ -263,12 +281,16 @@ block_diff_t *block_diff(block_entry_t *old_blocks, int old_count, // Mark remaining unconsumed old blocks as REMOVED for (int oi = old_scan; oi < old_count; oi++) { if (!old_consumed[oi]) { - block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[oi].position_id, NULL); + if (!block_diff_append(diff, BLOCK_DIFF_REMOVED, old_blocks[oi].position_id, NULL)) goto fail; } } if (old_consumed) cloudsync_memory_free(old_consumed); return diff; +fail: + cloudsync_memory_free(old_consumed); + block_diff_free(diff); + return NULL; } // MARK: - Materialization - diff --git a/src/cloudsync.c b/src/cloudsync.c index 6cc1f01e..78468254 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -1325,6 +1325,7 @@ static int merge_pending_add (cloudsync_context *data, cloudsync_table_context * merge_pending_entry *e = &batch->entries[batch->count]; e->col_name = stable_col_name; e->col_value = col_value ? (dbvalue_t *)database_value_dup(col_value) : NULL; + if (col_value && !e->col_value) return DBRES_NOMEM; e->col_version = col_version; e->db_version = db_version; e->site_id_len = (site_len <= (int)sizeof(e->site_id)) ? site_len : (int)sizeof(e->site_id); @@ -1362,6 +1363,7 @@ static int merge_flush_pending (cloudsync_context *data) { int rc = DBRES_OK; bool flush_savepoint = false; + char error_message[1024] = {0}; // Nothing to write — handle sentinel-only case or skip if (batch->count == 0 && !(batch->sentinel_pending && batch->table)) { @@ -1371,7 +1373,9 @@ static int merge_flush_pending (cloudsync_context *data) { // Wrap database operations in a savepoint so that on failure (e.g. RLS // denial) the rollback properly releases all executor resources (open // relations, snapshots, plan cache) acquired during the failed statement. - flush_savepoint = (database_begin_savepoint(data, "merge_flush") == DBRES_OK); + rc = database_begin_savepoint(data, "merge_flush"); + if (rc != DBRES_OK) goto cleanup; + flush_savepoint = true; if (batch->count == 0) { // Sentinel with no winning columns (PK-only row) @@ -1456,12 +1460,15 @@ static int merge_flush_pending (cloudsync_context *data) { if (batch->cached_col_count > 0) { const char **new_names = (const char **)cloudsync_memory_realloc( batch->cached_col_names, batch->count * sizeof(const char *)); - if (new_names) { - for (int i = 0; i < batch->count; i++) { - new_names[i] = batch->entries[i].col_name; - } - batch->cached_col_names = new_names; + if (!new_names) { + batch->cached_col_count = 0; + rc = DBRES_NOMEM; + goto cleanup; } + for (int i = 0; i < batch->count; i++) { + new_names[i] = batch->entries[i].col_name; + } + batch->cached_col_names = new_names; } } @@ -1519,11 +1526,13 @@ static int merge_flush_pending (cloudsync_context *data) { } cleanup: + if (rc != DBRES_OK) snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); merge_pending_free_entries(batch); if (flush_savepoint) { - if (rc == DBRES_OK) database_commit_savepoint(data, "merge_flush"); - else database_rollback_savepoint(data, "merge_flush"); + if (rc == DBRES_OK) rc = database_commit_savepoint(data, "merge_flush"); + if (rc != DBRES_OK) database_rollback_savepoint(data, "merge_flush"); } + if (rc != DBRES_OK) cloudsync_set_error(data, error_message[0] ? error_message : "Unable to flush pending changes", rc); return rc; } @@ -1883,6 +1892,7 @@ int block_materialize_column (cloudsync_context *data, cloudsync_table_context * block_cap = new_cap; } block_values[block_count] = value ? cloudsync_string_dup(value) : cloudsync_string_dup(""); + if (!block_values[block_count]) { rc = DBRES_NOMEM; break; } block_count++; } databasevm_reset(vm); @@ -1892,7 +1902,7 @@ int block_materialize_column (cloudsync_context *data, cloudsync_table_context * // Free collected values for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); if (block_values) cloudsync_memory_free((void *)block_values); - return cloudsync_set_dberror(data); + return cloudsync_set_error(data, "Unable to read block values", rc); } // Materialize text (NULL when no alive blocks) @@ -2103,7 +2113,6 @@ static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table const char *col_name = table->col_name[col_idx]; if (!col_name || !table->meta_ref || !table->blocks_ref) return DBRES_OK; - const char *delim = table->col_delimiter[col_idx] ? table->col_delimiter[col_idx] : BLOCK_DEFAULT_DELIMITER; int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET); // Phase 1: collect all existing PKs that have an alive regular col_name entry @@ -2145,9 +2154,11 @@ static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table void **new_pks = (void **)cloudsync_memory_realloc(pks, (uint64_t)(new_cap * sizeof(void *))); size_t *new_pklens = (size_t *)cloudsync_memory_realloc(pklens, (uint64_t)(new_cap * sizeof(size_t))); if (!new_pks || !new_pklens) { + for (int i = 0; i < pk_count; i++) cloudsync_memory_free((new_pks ? new_pks : pks)[i]); cloudsync_memory_free(new_pks ? new_pks : pks); cloudsync_memory_free(new_pklens ? new_pklens : pklens); databasevm_finalize(scan_vm); + cloudsync_memory_free(like_pattern); return DBRES_NOMEM; } pks = new_pks; @@ -2177,83 +2188,24 @@ static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table return DBRES_OK; } - // Phase 2: for each collected PK, read the column value, split into blocks, - // and insert into the blocks table + metadata using INSERT OR IGNORE. - - char *meta_sql = cloudsync_memory_mprintf(SQL_META_INSERT_BLOCK_IGNORE, table->meta_ref); - if (!meta_sql) { rc = DBRES_NOMEM; goto cleanup_pks; } - dbvm_t *meta_vm = NULL; - rc = databasevm_prepare(data, meta_sql, &meta_vm, 0); - cloudsync_memory_free(meta_sql); - if (rc != DBRES_OK) goto cleanup_pks; - - char *blocks_sql = cloudsync_memory_mprintf(SQL_BLOCKS_INSERT_IGNORE, table->blocks_ref); - if (!blocks_sql) { databasevm_finalize(meta_vm); rc = DBRES_NOMEM; goto cleanup_pks; } - dbvm_t *blocks_vm = NULL; - rc = databasevm_prepare(data, blocks_sql, &blocks_vm, 0); - cloudsync_memory_free(blocks_sql); - if (rc != DBRES_OK) { databasevm_finalize(meta_vm); goto cleanup_pks; } - - dbvm_t *val_vm = (dbvm_t *)table_column_lookup(table, col_name, false, NULL); - - for (int p = 0; p < pk_count; p++) { - const void *pk = pks[p]; - size_t pklen = pklens[p]; - - if (!val_vm) continue; - - // Read current column value from the base table - int bind_rc = pk_decode_prikey((char *)pk, pklen, pk_decode_bind_callback, (void *)val_vm); - if (bind_rc < 0) { databasevm_reset(val_vm); continue; } - - int step_rc = databasevm_step(val_vm); - const char *text = (step_rc == DBRES_ROW) ? database_column_text(val_vm, 0) : NULL; - // Make a copy of text before resetting val_vm, as the pointer is only valid until reset - char *text_copy = text ? cloudsync_string_dup(text) : NULL; + // Reuse the checked block writer; the scan above excludes migrated rows. + dbvm_t *val_vm = table_column_lookup(table, col_name, false, NULL); + rc = val_vm ? DBRES_OK : DBRES_MISUSE; + for (int p = 0; p < pk_count && rc == DBRES_OK; p++) { + rc = pk_decode_prikey(pks[p], pklens[p], pk_decode_bind_callback, val_vm); + if (rc >= 0) rc = databasevm_step(val_vm); + if (rc == DBRES_ROW) { + const char *text = database_column_text(val_vm, 0); + bool has_text = text != NULL; + char *copy = text ? cloudsync_string_dup(text) : NULL; + databasevm_reset(val_vm); + rc = has_text && !copy ? DBRES_NOMEM : DBRES_OK; + if (rc == DBRES_OK && copy) + rc = local_block_update(data, table, pks[p], pklens[p], col_idx, copy, db_version, true); + cloudsync_memory_free(copy); + } else if (rc == DBRES_DONE) rc = DBRES_ERROR; databasevm_reset(val_vm); - - if (!text_copy) continue; // NULL column value: nothing to migrate - - // Split text into blocks and store each one - block_list_t *blocks = block_split(text_copy, delim); - cloudsync_memory_free(text_copy); - if (!blocks) continue; - - char **positions = block_initial_positions(blocks->count); - if (positions) { - for (int b = 0; b < blocks->count; b++) { - char *block_cn = block_build_colname(col_name, positions[b]); - if (block_cn) { - // Metadata entry (skip if this block position already exists) - databasevm_bind_blob(meta_vm, 1, pk, (int)pklen); - databasevm_bind_text(meta_vm, 2, block_cn, -1); - databasevm_bind_int(meta_vm, 3, 1); // col_version = 1 (alive) - databasevm_bind_int(meta_vm, 4, db_version); - databasevm_bind_int(meta_vm, 5, cloudsync_bumpseq(data)); - databasevm_step(meta_vm); - databasevm_reset(meta_vm); - - // Block value (skip if this block position already exists) - databasevm_bind_blob(blocks_vm, 1, pk, (int)pklen); - databasevm_bind_text(blocks_vm, 2, block_cn, -1); - databasevm_bind_text(blocks_vm, 3, blocks->entries[b].content, -1); - databasevm_step(blocks_vm); - databasevm_reset(blocks_vm); - - cloudsync_memory_free(block_cn); - } - cloudsync_memory_free(positions[b]); - } - cloudsync_memory_free(positions); - } - block_list_free(blocks); } - - databasevm_finalize(meta_vm); - databasevm_finalize(blocks_vm); - rc = DBRES_OK; - -cleanup_pks: for (int i = 0; i < pk_count; i++) cloudsync_memory_free(pks[i]); cloudsync_memory_free(pks); cloudsync_memory_free(pklens); @@ -2806,6 +2758,7 @@ int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name) const void *pk = (const char *)database_column_blob(vm, 0, &pklen); if (!pk) { rc = DBRES_ERROR; break; } rc = local_mark_insert_or_update_meta(table, pk, pklen, col_name, db_version, cloudsync_bumpseq(data)); + if (rc != DBRES_OK) break; } else if (rc == DBRES_DONE) { rc = DBRES_OK; break; @@ -4099,6 +4052,10 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // check if payload is compressed char *clone = NULL; if (header.expanded_size != 0) { + // Bound untrusted allocation sizes before passing them to LZ4's int API. + if (header.expanded_size > CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE || header.expanded_size > INT_MAX) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: expanded payload exceeds limit", DBRES_MISUSE); + } clone = (char *)cloudsync_memory_alloc(header.expanded_size); if (!clone) return cloudsync_set_error(data, "Unable to allocate memory to uncompress payload", DBRES_NOMEM); @@ -4156,6 +4113,9 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b uint16_t ncols = header.ncols; uint32_t nrows = header.nrows; int64_t last_payload_db_version = -1; + int first_error = DBRES_OK; + bool policy_denied = false; + char first_error_message[1024] = {0}; cloudsync_pk_decode_bind_context decoded_context = {.vm = vm}; // Initialize deferred column-batch merge @@ -4193,9 +4153,10 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // Flush pending batch before any boundary change if (pk_changed || tbl_changed || db_version_changed) { int flush_rc = merge_flush_pending(data); - if (flush_rc != DBRES_OK) { - rc = flush_rc; - // continue processing remaining rows + if (flush_rc == DBRES_POLICY_DENIED) policy_denied = true; + else if (flush_rc != DBRES_OK && first_error == DBRES_OK) { + first_error = flush_rc; + snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); } } @@ -4239,7 +4200,12 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b last_tbl_len = decoded_context.tbl_len; rc = databasevm_step(vm); + if (rc == DBRES_POLICY_DENIED) { policy_denied = true; rc = DBRES_DONE; } if (rc != DBRES_DONE) { + if (first_error == DBRES_OK) { + first_error = rc; + snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); + } // don't "break;", the error can be due to a RLS policy. // in case of error we try to apply the following changes } @@ -4252,7 +4218,11 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // Final flush after loop { int flush_rc = merge_flush_pending(data); - if (flush_rc != DBRES_OK && rc == DBRES_OK) rc = flush_rc; + if (flush_rc == DBRES_POLICY_DENIED) policy_denied = true; + else if (flush_rc != DBRES_OK && first_error == DBRES_OK) { + first_error = flush_rc; + snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); + } } data->pending_batch = NULL; @@ -4260,10 +4230,11 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b int rc1 = database_commit_savepoint(data, "cloudsync_payload_apply"); if (rc1 != DBRES_OK) rc = rc1; } + if (first_error != DBRES_OK) rc = first_error; // save last error (unused if function returns OK) if (rc != DBRES_OK && rc != DBRES_DONE) { - cloudsync_set_dberror(data); + cloudsync_set_error(data, first_error_message[0] ? first_error_message : "Unable to apply payload changes", rc); } if (rc == DBRES_DONE) rc = DBRES_OK; @@ -4276,7 +4247,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b data->apply_last_db_version = decoded_context.db_version; data->apply_last_seq = decoded_context.seq; } - cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); + if (!policy_denied) cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); } cleanup: @@ -4299,6 +4270,104 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b return DBRES_OK; } +/* Shared by both backends: failures must abort the caller's statement. */ +int local_block_update(cloudsync_context *data, cloudsync_table_context *table, + const void *pk, size_t pklen, int column, const char *text, + int64_t version, bool initial) { + int rc = DBRES_NOMEM; + const char *col = table_colname(table, column); + block_list_t *old = block_list_create_empty(); + block_list_t *next = (text || initial) ? block_split(text ? text : "", table_col_delimiter(table, column)) : block_list_create_empty(); + block_diff_t *diff = NULL; + const char **parts = NULL; + dbvm_t *vm = NULL; + char *sql = NULL; + if (!old || !next) goto done; + if (!initial) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + sql = cloudsync_memory_mprintf("SELECT col_name, col_value FROM %s WHERE pk=$1 ORDER BY col_name COLLATE \"C\"", table_blocks_ref(table)); +#else + sql = cloudsync_memory_mprintf("SELECT col_name, col_value FROM %s WHERE pk=?1 ORDER BY col_name COLLATE BINARY", table_blocks_ref(table)); +#endif + if (!sql) goto done; + rc = databasevm_prepare(data, sql, &vm, 0); + if (rc != DBRES_OK) goto done; + rc = databasevm_bind_blob(vm, 1, pk, (int)pklen); + if (rc != DBRES_OK) goto done; + while ((rc = databasevm_step(vm)) == DBRES_ROW) { + const char *name = database_column_text(vm, 0); + const char *value = database_column_text(vm, 1); + const char *pos = block_extract_position_id(name); + /* Literal prefix comparison: SQL LIKE would mix columns containing % or _. */ + if (pos && (size_t)(pos - name - 1) == strlen(col) && memcmp(name, col, strlen(col)) == 0) { + if (!block_list_add(old, value ? value : "", pos)) { rc = DBRES_NOMEM; goto done; } + } + } + if (rc != DBRES_DONE) goto done; + databasevm_finalize(vm); + vm = NULL; + } + rc = DBRES_NOMEM; + if (next->count) { + parts = cloudsync_memory_alloc((uint64_t)next->count * sizeof(*parts)); + if (!parts) goto done; + for (int i = 0; i < next->count; i++) parts[i] = next->entries[i].content; + } + diff = block_diff(old->entries, old->count, parts, next->count); + if (!diff) goto done; + rc = DBRES_OK; + for (int i = 0; i < diff->count; i++) { + block_diff_entry_t *entry = &diff->entries[i]; + char *name = block_build_colname(col, entry->position_id); + if (!name) { rc = DBRES_NOMEM; break; } + if (entry->type == BLOCK_DIFF_REMOVED) { + rc = local_mark_delete_block_meta(table, pk, pklen, name, version, cloudsync_bumpseq(data)); + if (rc == DBRES_OK) rc = block_delete_value_external(data, table, pk, pklen, name); + } else { + rc = local_mark_insert_or_update_meta(table, pk, pklen, name, version, cloudsync_bumpseq(data)); + dbvm_t *write = table_block_value_write_stmt(table); + if (rc == DBRES_OK && !write) rc = DBRES_MISUSE; + if (rc == DBRES_OK) rc = databasevm_bind_blob(write, 1, pk, (int)pklen); + if (rc == DBRES_OK) rc = databasevm_bind_text(write, 2, name, -1); + if (rc == DBRES_OK) rc = databasevm_bind_text(write, 3, entry->content, -1); + if (rc == DBRES_OK) rc = databasevm_step(write); + if (write) databasevm_reset(write); + if (rc == DBRES_DONE) rc = DBRES_OK; + } + cloudsync_memory_free(name); + if (rc != DBRES_OK) break; + } +done: + if (vm) databasevm_finalize(vm); + cloudsync_memory_free(sql); + cloudsync_memory_free((void *)parts); + block_diff_free(diff); + block_list_free(old); + block_list_free(next); + if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to update block metadata or values", rc); + return rc; +} + +int local_block_insert(cloudsync_context *data, cloudsync_table_context *table, + const void *pk, size_t pklen, int column, int64_t version) { + dbvm_t *vm = table_column_lookup(table, table_colname(table, column), false, NULL); + if (!vm) return cloudsync_set_error(data, "Missing block column statement", DBRES_MISUSE); + int rc = pk_decode_prikey((char *)pk, pklen, pk_decode_bind_callback, vm); + if (rc >= 0) rc = databasevm_step(vm); + char *copy = NULL; + if (rc == DBRES_ROW) { + const char *text = database_column_text(vm, 0); + copy = cloudsync_string_dup(text ? text : ""); + rc = copy ? DBRES_OK : DBRES_NOMEM; + } + else if (rc == DBRES_DONE) rc = DBRES_ERROR; + // End the read cursor before writes that can invoke nested triggers/SPI errors. + databasevm_reset(vm); + if (rc == DBRES_OK) rc = local_block_update(data, table, pk, pklen, column, copy, version, true); + cloudsync_memory_free(copy); + return rc; +} + // MARK: - Payload load/store - int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size, int *db_version, int64_t *new_db_version) { diff --git a/src/cloudsync.h b/src/cloudsync.h index 7e72f86e..56077d8c 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -18,7 +18,10 @@ extern "C" { #endif -#define CLOUDSYNC_VERSION "1.1.3" +#define CLOUDSYNC_VERSION "1.1.4" +#ifndef CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE +#define CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE (256U * 1024U * 1024U) +#endif #define CLOUDSYNC_MAX_TABLENAME_LEN 512 #define CLOUDSYNC_VALUE_NOTSET -1 @@ -207,6 +210,8 @@ int local_mark_insert_or_update_meta (cloudsync_table_context *table, const void int local_mark_delete_meta (cloudsync_table_context *table, const void *pk, size_t pklen, int64_t db_version, int seq); int local_mark_delete_block_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname, int64_t db_version, int seq); int block_delete_value_external (cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, const char *block_colname); +int local_block_update(cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, int column, const char *text, int64_t version, bool initial); +int local_block_insert(cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, int column, int64_t version); int local_drop_meta (cloudsync_table_context *table, const void *pk, size_t pklen); int local_update_move_meta (cloudsync_table_context *table, const void *pk, size_t pklen, const void *pk2, size_t pklen2, int64_t db_version); diff --git a/src/database.h b/src/database.h index 56bb2d66..9ccdc6fe 100644 --- a/src/database.h +++ b/src/database.h @@ -25,7 +25,8 @@ typedef enum { DBRES_CONSTRAINT = 19, DBRES_MISUSE = 21, DBRES_ROW = 100, - DBRES_DONE = 101 + DBRES_DONE = 101, + DBRES_POLICY_DENIED = 1001 // PostgreSQL RLS WITH CHECK denial, not a generic SQL error } DBRES; typedef enum { diff --git a/src/network/network.c b/src/network/network.c index 038adffd..512af7d5 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -374,7 +374,12 @@ static bool network_curl_pool_enabled(network_data *data) { static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, bool *pooled) { if (pooled) *pooled = false; if (!network_curl_pool_enabled(data)) { - return curl_easy_init(); + CURL *handle = curl_easy_init(); + if (!handle) return NULL; + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); + return handle; } CURL **slot = network_endpoint_is_api(data, endpoint) ? &data->api_curl : &data->artifact_curl; @@ -384,6 +389,9 @@ static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, curl_easy_reset(*slot); } if (!*slot) return NULL; + curl_easy_setopt(*slot, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); + curl_easy_setopt(*slot, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); + curl_easy_setopt(*slot, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(*slot, CURLOPT_MAXCONNECTS, CLOUDSYNC_CURL_MAXCONNECTS); curl_easy_setopt(*slot, CURLOPT_MAXAGE_CONN, CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS); @@ -392,6 +400,30 @@ static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, return *slot; } +#if defined(CLOUDSYNC_UNITTEST) && !defined(CLOUDSYNC_OMIT_CURL) +bool network_test_curl_timeout(const char *url, bool use_pool) { + network_data data = {0}; + data.curl_pool_enabled = use_pool ? 1 : -1; + bool ok = true; + // The second pooled call exercises curl_easy_reset as well as initialization. + for (int i = 0; i < 2; i++) { + bool pooled = false; + CURL *handle = network_curl_for_endpoint(&data, url, &pooled); + if (!handle) { ok = false; break; } + curl_easy_setopt(handle, CURLOPT_URL, url); + curl_easy_setopt(handle, CURLOPT_PROXY, ""); + CURLcode rc = curl_easy_perform(handle); + double seconds = 0; + curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &seconds); + ok = ok && rc == CURLE_OPERATION_TIMEDOUT && seconds < CLOUDSYNC_REQUEST_TIMEOUT_SECONDS + 2; + if (!pooled) curl_easy_cleanup(handle); + } + if (data.api_curl) curl_easy_cleanup(data.api_curl); + if (data.artifact_curl) curl_easy_cleanup(data.artifact_curl); + return ok; +} +#endif + static bool network_buffer_check (network_buffer *data, size_t needed) { // alloc/resize buffer if (data->bused + needed > data->balloc) { @@ -813,13 +845,6 @@ static bool jsmn_token_eq(const char *json, const jsmntok_t *tok, const char *s) strncmp(json + tok->start, s, tok->end - tok->start) == 0); } -static int jsmn_find_key(const char *json, const jsmntok_t *tokens, int ntokens, const char *key) { - for (int i = 1; i + 1 < ntokens; i++) { - if (jsmn_token_eq(json, &tokens[i], key)) return i; - } - return -1; -} - static int jsmn_token_span(const jsmntok_t *tokens, int ntokens, int index) { if (!tokens || index < 0 || index >= ntokens) return 0; int start = tokens[index].start; @@ -871,42 +896,84 @@ static jsmntok_t *json_parse_tokens_alloc(const char *json, size_t json_len, int return tokens; } +static int jsmn_find_key(const char *json, const jsmntok_t *tokens, int ntokens, const char *key) { + int value_index; + return jsmn_find_object_value(json, tokens, ntokens, 0, key, &value_index) ? value_index - 1 : -1; +} + +static int json_hex4(const char *src) { + int value = 0; + for (int i = 0; i < 4; i++) { + unsigned char c = (unsigned char)src[i]; + int digit = c >= '0' && c <= '9' ? c - '0' : + c >= 'a' && c <= 'f' ? c - 'a' + 10 : + c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; + if (digit < 0) return -1; + value = (value << 4) | digit; + } + return value; +} + static char *json_unescape_string(const char *src, int len) { - char *out = cloudsync_memory_zeroalloc(len + 1); + char *out = cloudsync_memory_zeroalloc((uint64_t)len + 1); if (!out) return NULL; - int j = 0; - for (int i = 0; i < len; ) { - if (src[i] == '\\' && i + 1 < len) { - char c = src[i + 1]; - if (c == '"' || c == '\\' || c == '/') { out[j++] = c; i += 2; } - else if (c == 'n') { out[j++] = '\n'; i += 2; } - else if (c == 'r') { out[j++] = '\r'; i += 2; } - else if (c == 't') { out[j++] = '\t'; i += 2; } - else if (c == 'b') { out[j++] = '\b'; i += 2; } - else if (c == 'f') { out[j++] = '\f'; i += 2; } - else if (c == 'u' && i + 5 < len) { - unsigned int cp = 0; - for (int k = 0; k < 4; k++) { - char h = src[i + 2 + k]; - cp <<= 4; - if (h >= '0' && h <= '9') cp |= h - '0'; - else if (h >= 'a' && h <= 'f') cp |= 10 + h - 'a'; - else if (h >= 'A' && h <= 'F') cp |= 10 + h - 'A'; + for (int i = 0; i < len;) { + unsigned char c = (unsigned char)src[i++]; + if (c != '\\') { out[j++] = (char)c; continue; } + if (i == len) goto invalid; + c = (unsigned char)src[i++]; + switch (c) { + case '"': case '\\': case '/': out[j++] = (char)c; break; + case 'n': out[j++] = '\n'; break; + case 'r': out[j++] = '\r'; break; + case 't': out[j++] = '\t'; break; + case 'b': out[j++] = '\b'; break; + case 'f': out[j++] = '\f'; break; + case 'u': { + if (len - i < 4) goto invalid; + int cp = json_hex4(src + i); + if (cp < 0) goto invalid; + i += 4; + if (cp >= 0xD800 && cp <= 0xDBFF) { + if (len - i < 6 || src[i] != '\\' || src[i + 1] != 'u') goto invalid; + int low = json_hex4(src + i + 2); + if (low < 0xDC00 || low > 0xDFFF) goto invalid; + cp = 0x10000 + ((cp - 0xD800) << 10) + low - 0xDC00; + i += 6; + } else if (cp >= 0xDC00 && cp <= 0xDFFF) goto invalid; + // Network consumers use C strings: reject embedded NUL truncation. + if (cp == 0) goto invalid; + if (cp < 0x80) out[j++] = (char)cp; + else if (cp < 0x800) { + out[j++] = (char)(0xC0 | (cp >> 6)); + out[j++] = (char)(0x80 | (cp & 0x3F)); + } else { + if (cp >= 0x10000) { + out[j++] = (char)(0xF0 | (cp >> 18)); + out[j++] = (char)(0x80 | ((cp >> 12) & 0x3F)); + } else out[j++] = (char)(0xE0 | (cp >> 12)); + out[j++] = (char)(0x80 | ((cp >> 6) & 0x3F)); + out[j++] = (char)(0x80 | (cp & 0x3F)); } - if (cp < 0x80) { out[j++] = (char)cp; } - else { out[j++] = '?'; } // non-ASCII: replace - i += 6; + break; } - else { out[j++] = src[i]; i++; } - } else { - out[j++] = src[i]; i++; + default: goto invalid; } } out[j] = '\0'; return out; +invalid: + cloudsync_memory_free(out); + return NULL; } +#ifdef CLOUDSYNC_UNITTEST +char *network_test_unescape(const char *src) { + return json_unescape_string(src, (int)strlen(src)); +} +#endif + static char *json_extract_string(const char *json, size_t json_len, const char *key) { if (!json || json_len == 0 || !key) return NULL; diff --git a/src/network/network_private.h b/src/network/network_private.h index 21a46fd4..113aab86 100644 --- a/src/network/network_private.h +++ b/src/network/network_private.h @@ -12,6 +12,12 @@ #include #define CLOUDSYNC_DEFAULT_ADDRESS "https://cloudsync.sqlite.ai" +#ifndef CLOUDSYNC_CONNECT_TIMEOUT_SECONDS +#define CLOUDSYNC_CONNECT_TIMEOUT_SECONDS 30L +#endif +#ifndef CLOUDSYNC_REQUEST_TIMEOUT_SECONDS +#define CLOUDSYNC_REQUEST_TIMEOUT_SECONDS 300L +#endif #define CLOUDSYNC_ENDPOINT_PREFIX "v2/cloudsync/databases" #define CLOUDSYNC_ENDPOINT_UPLOAD "upload" #define CLOUDSYNC_ENDPOINT_CHECK "check" diff --git a/src/pk.c b/src/pk.c index dcc8ca67..40e88e9d 100644 --- a/src/pk.c +++ b/src/pk.c @@ -195,13 +195,13 @@ static int pk_decode_data (const uint8_t *buffer, size_t blen, size_t *bseek, si } int pk_decode_double (const uint8_t *buffer, size_t blen, size_t *bseek, double *out) { - // Doubles are encoded as IEEE754 64-bit, big-endian. - // Convert back to host order before memcpy into double. + // Historical wire format is little-endian IEEE754, unlike integer fields. + // pk_decode_uint64 already constructs a host integer from big-endian bytes. uint64_t bits_be = 0; if (!pk_decode_uint64(buffer, blen, bseek, sizeof(uint64_t), &bits_be)) return 0; - uint64_t bits = be64_to_host(bits_be); + uint64_t bits = bswap64_u64(bits_be); double value = 0.0; memcpy(&value, &bits, sizeof(bits)); *out = value; @@ -527,12 +527,13 @@ char *pk_encode (dbvalue_t **argv, int argc, char *b, bool is_prikey, size_t *bs } break; case DBTYPE_FLOAT: { - // Encode doubles as IEEE754 64-bit, big-endian + // Encode doubles as IEEE754 64-bit, little-endian. double value = database_value_double(argv[i]); if (value < 0) {value = -value; type = DATABASE_TYPE_NEGATIVE_FLOAT;} uint64_t bits; memcpy(&bits, &value, sizeof(bits)); - bits = host_to_be64(bits); + // Preserve the deployed little-endian double wire format on all hosts. + bits = bswap64_u64(bits); bseek = pk_encode_u8(buffer, bseek, (uint8_t)type); bseek = pk_encode_uint64(buffer, bseek, bits, sizeof(bits)); } diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c index 3f62d2f6..a310f96c 100644 --- a/src/postgresql/cloudsync_postgresql.c +++ b/src/postgresql/cloudsync_postgresql.c @@ -2125,50 +2125,7 @@ Datum cloudsync_insert (PG_FUNCTION_ARGS) { // Process each non-primary key column for insert or update for (int i = 0; i < table_count_cols(table); i++) { if (table_col_algo(table, i) == col_algo_block) { - // Block column: read value from base table, split into blocks, store each block - dbvm_t *val_vm = table_column_lookup(table, table_colname(table, i), false, NULL); - if (!val_vm) { rc = DBRES_ERROR; break; } - - int bind_rc = pk_decode_prikey(cleanup.pk, pklen, pk_decode_bind_callback, (void *)val_vm); - if (bind_rc < 0) { databasevm_reset(val_vm); rc = DBRES_ERROR; break; } - - int step_rc = databasevm_step(val_vm); - if (step_rc == DBRES_ROW) { - const char *text = database_column_text(val_vm, 0); - const char *delim = table_col_delimiter(table, i); - const char *col = table_colname(table, i); - - block_list_t *blocks = block_split(text ? text : "", delim); - if (blocks) { - char **positions = block_initial_positions(blocks->count); - if (positions) { - for (int b = 0; b < blocks->count; b++) { - char *block_cn = block_build_colname(col, positions[b]); - if (block_cn) { - rc = local_mark_insert_or_update_meta(table, cleanup.pk, pklen, block_cn, db_version, cloudsync_bumpseq(data)); - - // Store block value in blocks table - dbvm_t *wvm = table_block_value_write_stmt(table); - if (wvm && rc == DBRES_OK) { - databasevm_bind_blob(wvm, 1, cleanup.pk, (int)pklen); - databasevm_bind_text(wvm, 2, block_cn, -1); - databasevm_bind_text(wvm, 3, blocks->entries[b].content, -1); - databasevm_step(wvm); - databasevm_reset(wvm); - } - - cloudsync_memory_free(block_cn); - } - cloudsync_memory_free(positions[b]); - if (rc != DBRES_OK) break; - } - cloudsync_memory_free(positions); - } - block_list_free(blocks); - } - } - databasevm_reset(val_vm); - if (step_rc == DBRES_ROW || step_rc == DBRES_DONE) { if (rc == DBRES_OK) continue; } + rc = local_block_insert(data, table, cleanup.pk, pklen, i, db_version); if (rc != DBRES_OK) break; } else { rc = local_mark_insert_or_update_meta(table, cleanup.pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); @@ -2476,93 +2433,8 @@ Datum cloudsync_update_finalfn (PG_FUNCTION_ARGS) { if (dbutils_value_compare((dbvalue_t *)payload->old_values[col_index], (dbvalue_t *)payload->new_values[col_index]) != 0) { if (table_col_algo(table, i) == col_algo_block) { - // Block column: diff old and new text, emit per-block metadata changes - const char *new_text = (const char *)database_value_text(payload->new_values[col_index]); - const char *delim = table_col_delimiter(table, i); - const char *col = table_colname(table, i); - - // Read existing blocks from blocks table - block_list_t *old_blocks = block_list_create_empty(); - char *like_pattern = block_build_colname(col, "%"); - if (like_pattern && old_blocks) { - char *list_sql = cloudsync_memory_mprintf( - "SELECT col_name, col_value FROM %s WHERE pk = $1 AND col_name LIKE $2 ORDER BY col_name COLLATE \"C\"", - table_blocks_ref(table)); - if (list_sql) { - dbvm_t *list_vm = NULL; - if (databasevm_prepare(data, list_sql, &list_vm, 0) == DBRES_OK) { - databasevm_bind_blob(list_vm, 1, pk, (int)pklen); - databasevm_bind_text(list_vm, 2, like_pattern, -1); - while (databasevm_step(list_vm) == DBRES_ROW) { - const char *bcn = database_column_text(list_vm, 0); - const char *bval = database_column_text(list_vm, 1); - const char *pos = block_extract_position_id(bcn); - if (pos && old_blocks) { - block_list_add(old_blocks, bval ? bval : "", pos); - } - } - databasevm_finalize(list_vm); - } - cloudsync_memory_free(list_sql); - } - } - - // Split new text into parts (NULL text = all blocks removed) - block_list_t *new_blocks = new_text ? block_split(new_text, delim) : block_list_create_empty(); - if (new_blocks && old_blocks) { - // Build array of new content strings (NULL when count is 0) - const char **new_parts = NULL; - if (new_blocks->count > 0) { - new_parts = (const char **)cloudsync_memory_alloc( - (uint64_t)(new_blocks->count * sizeof(char *))); - if (new_parts) { - for (int b = 0; b < new_blocks->count; b++) { - new_parts[b] = new_blocks->entries[b].content; - } - } - } - - if (new_parts || new_blocks->count == 0) { - block_diff_t *diff = block_diff(old_blocks->entries, old_blocks->count, - new_parts, new_blocks->count); - if (diff) { - for (int d = 0; d < diff->count; d++) { - block_diff_entry_t *de = &diff->entries[d]; - char *block_cn = block_build_colname(col, de->position_id); - if (!block_cn) continue; - - if (de->type == BLOCK_DIFF_ADDED || de->type == BLOCK_DIFF_MODIFIED) { - rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, - db_version, cloudsync_bumpseq(data)); - // Store block value - if (rc == DBRES_OK && table_block_value_write_stmt(table)) { - dbvm_t *wvm = table_block_value_write_stmt(table); - databasevm_bind_blob(wvm, 1, pk, (int)pklen); - databasevm_bind_text(wvm, 2, block_cn, -1); - databasevm_bind_text(wvm, 3, de->content, -1); - databasevm_step(wvm); - databasevm_reset(wvm); - } - } else if (de->type == BLOCK_DIFF_REMOVED) { - // Mark block as deleted in metadata (even col_version) - rc = local_mark_delete_block_meta(table, pk, pklen, block_cn, - db_version, cloudsync_bumpseq(data)); - // Remove from blocks table - if (rc == DBRES_OK) { - block_delete_value_external(data, table, pk, pklen, block_cn); - } - } - cloudsync_memory_free(block_cn); - if (rc != DBRES_OK) break; - } - block_diff_free(diff); - } - if (new_parts) cloudsync_memory_free((void *)new_parts); - } - } - if (new_blocks) block_list_free(new_blocks); - if (old_blocks) block_list_free(old_blocks); - if (like_pattern) cloudsync_memory_free(like_pattern); + rc = local_block_update(data, table, pk, pklen, i, + (const char *)database_value_text(payload->new_values[col_index]), db_version, false); if (rc != DBRES_OK) goto cleanup; } else { rc = local_mark_insert_or_update_meta(table, pk, pklen, table_colname(table, i), db_version, cloudsync_bumpseq(data)); diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c index 0f9a50b5..d81fa3bf 100644 --- a/src/postgresql/database_postgresql.c +++ b/src/postgresql/database_postgresql.c @@ -581,6 +581,7 @@ static int map_spi_result (int rc) { static void clear_fetch_batch (pg_stmt_t *stmt) { if (!stmt) return; if (stmt->last_tuptable) { + if (SPI_tuptable == stmt->last_tuptable) SPI_tuptable = NULL; SPI_freetuptable(stmt->last_tuptable); stmt->last_tuptable = NULL; } @@ -2199,6 +2200,7 @@ int databasevm_step (dbvm_t *vm) { clear_fetch_batch(stmt); SPI_cursor_fetch(stmt->portal, true, 1); + stmt->last_tuptable = SPI_tuptable; if (SPI_processed == 0) { clear_fetch_batch(stmt); @@ -2218,6 +2220,7 @@ int databasevm_step (dbvm_t *vm) { MemoryContextReset(stmt->row_mcxt); stmt->last_tuptable = SPI_tuptable; + SPI_tuptable = NULL; stmt->current_tupdesc = stmt->last_tuptable->tupdesc; stmt->current_tuple = stmt->last_tuptable->vals[0]; rc = DBRES_ROW; @@ -2242,6 +2245,7 @@ int databasevm_step (dbvm_t *vm) { // fetch first row clear_fetch_batch(stmt); SPI_cursor_fetch(stmt->portal, true, 1); + stmt->last_tuptable = SPI_tuptable; if (SPI_processed == 0) { // No rows - close portal, don't set portal_open @@ -2262,6 +2266,7 @@ int databasevm_step (dbvm_t *vm) { MemoryContextReset(stmt->row_mcxt); stmt->last_tuptable = SPI_tuptable; + SPI_tuptable = NULL; stmt->current_tupdesc = stmt->last_tuptable->tupdesc; stmt->current_tuple = stmt->last_tuptable->vals[0]; @@ -2298,7 +2303,11 @@ int databasevm_step (dbvm_t *vm) { { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - int err = cloudsync_set_error(data, edata->message, DBRES_ERROR); + // PostgreSQL uses 42501 for both missing privileges and RLS. Only the + // executor's WITH CHECK policy rejection is safe to skip during merge. + bool policy_denied = edata->sqlerrcode == ERRCODE_INSUFFICIENT_PRIVILEGE && + edata->funcname && strcmp(edata->funcname, "ExecWithCheckOptions") == 0; + int err = cloudsync_set_error(data, edata->message, policy_denied ? DBRES_POLICY_DENIED : DBRES_ERROR); FreeErrorData(edata); FlushErrorState(); @@ -2320,10 +2329,7 @@ void databasevm_finalize (dbvm_t *vm) { { clear_fetch_batch(stmt); close_portal(stmt); - if (SPI_tuptable) { - SPI_freetuptable(SPI_tuptable); - SPI_tuptable = NULL; - } + // Only free this statement's tuple table, never another active cursor's. if (stmt->plan_is_prepared && stmt->plan) { SPI_freeplan(stmt->plan); @@ -2350,11 +2356,7 @@ void databasevm_reset (dbvm_t *vm) { clear_fetch_batch(stmt); close_portal(stmt); - // Clear global SPI tuple table if any - if (SPI_tuptable) { - SPI_freetuptable(SPI_tuptable); - SPI_tuptable = NULL; - } + // Non-row results are freed by step(); cursor results belong to last_tuptable. // Reset execution state stmt->executed_nonselect = false; diff --git a/src/sqlite/cloudsync_changes_sqlite.c b/src/sqlite/cloudsync_changes_sqlite.c index 5cd8a145..1cb831f1 100644 --- a/src/sqlite/cloudsync_changes_sqlite.c +++ b/src/sqlite/cloudsync_changes_sqlite.c @@ -275,7 +275,7 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id // +512 for the extra space and for the WHERE and ORDER BY literals // memory internally manager by SQLite, so I cannot use memory_alloc here - size_t slen = (count1 * (11 + 1 + 11 + 1 + 5)) + (count2 * 11 + 1 + 5) + 512; + size_t slen = ((size_t)count1 * 32) + ((size_t)count2 * 20) + 512; char *s = (char *)sqlite3_malloc64((sqlite3_uint64)slen); if (!s) return SQLITE_NOMEM; size_t sindex= 0; @@ -285,7 +285,7 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id int orderconsumed = 1; // is there a WHERE clause ? - if (count1 > 0) sindex += snprintf(s+sindex, slen-sindex, "WHERE "); + int accepted = 0; // check constraints for (int i=0; i < count1; ++i) { @@ -301,7 +301,7 @@ int cloudsync_changesvtab_best_index (sqlite3_vtab *vtab, sqlite3_index_info *id if (!opname) continue; // build next constraint - if (i > 0) sindex += snprintf(s+sindex, slen-sindex, " AND "); + sindex += snprintf(s+sindex, slen-sindex, accepted++ ? " AND " : "WHERE "); // handle special case where value is not needed if ((op == SQLITE_INDEX_CONSTRAINT_ISNULL) || (op == SQLITE_INDEX_CONSTRAINT_ISNOTNULL)) { @@ -566,12 +566,12 @@ int cloudsync_changesvtab_insert (sqlite3_vtab *vtab, int argc, sqlite3_value ** int insert_pk_len = sqlite3_value_bytes(argv[1]); const char *insert_name = (sqlite3_value_type(argv[2]) == SQLITE_NULL) ? CLOUDSYNC_TOMBSTONE_VALUE : (const char *)sqlite3_value_text(argv[2]); sqlite3_value *insert_value = argv[3]; - int64_t insert_col_version = (int64_t)sqlite3_value_int(argv[4]); - int64_t insert_db_version = (int64_t)sqlite3_value_int(argv[5]); + int64_t insert_col_version = sqlite3_value_int64(argv[4]); + int64_t insert_db_version = sqlite3_value_int64(argv[5]); const char *insert_site_id = (const char *)sqlite3_value_blob(argv[6]); int insert_site_id_len = sqlite3_value_bytes(argv[6]); - int64_t insert_cl = (int64_t)sqlite3_value_int(argv[7]); - int64_t insert_seq = (int64_t)sqlite3_value_int(argv[8]); + int64_t insert_cl = sqlite3_value_int64(argv[7]); + int64_t insert_seq = sqlite3_value_int64(argv[8]); // perform different logic for each different table algorithm if (table_algo_isgos(table)) return cloudsync_changesvtab_insert_gos(vtab, data, table, insert_pk, insert_pk_len, insert_name, insert_value, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, (int64_t *)rowid); diff --git a/src/sqlite/cloudsync_sqlite.c b/src/sqlite/cloudsync_sqlite.c index c416b015..aa31761e 100644 --- a/src/sqlite/cloudsync_sqlite.c +++ b/src/sqlite/cloudsync_sqlite.c @@ -471,50 +471,7 @@ void dbsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { // process each non-primary key column for insert or update for (int i=0; icount); - if (positions) { - for (int b = 0; b < blocks->count; b++) { - char *block_cn = block_build_colname(col, positions[b]); - if (block_cn) { - rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, db_version, cloudsync_bumpseq(data)); - - // Store block value in blocks table - dbvm_t *wvm = table_block_value_write_stmt(table); - if (wvm && rc == SQLITE_OK) { - databasevm_bind_blob(wvm, 1, pk, (int)pklen); - databasevm_bind_text(wvm, 2, block_cn, -1); - databasevm_bind_text(wvm, 3, blocks->entries[b].content, -1); - databasevm_step(wvm); - databasevm_reset(wvm); - } - - cloudsync_memory_free(block_cn); - } - cloudsync_memory_free(positions[b]); - if (rc != SQLITE_OK) break; - } - cloudsync_memory_free(positions); - } - block_list_free(blocks); - } - } - databasevm_reset((dbvm_t *)val_vm); - if (rc == DBRES_ROW || rc == DBRES_DONE) rc = SQLITE_OK; + rc = local_block_insert(data, table, pk, pklen, i, db_version); if (rc != SQLITE_OK) goto cleanup; } else { // Regular column: mark as inserted or updated in the metadata @@ -743,96 +700,8 @@ void dbsync_update_final (sqlite3_context *context) { if (dbutils_value_compare(payload->old_values[col_index], payload->new_values[col_index]) != 0) { if (table_col_algo(table, i) == col_algo_block) { - // Block column: diff old and new text, emit per-block metadata changes - const char *new_text = (const char *)database_value_text(payload->new_values[col_index]); - const char *delim = table_col_delimiter(table, i); - const char *col = table_colname(table, i); - - // Read existing blocks from blocks table - block_list_t *old_blocks = block_list_create_empty(); - if (table_block_list_stmt(table)) { - char *like_pattern = block_build_colname(col, "%"); - if (like_pattern) { - // Query blocks table directly for existing block names and values - char *list_sql = cloudsync_memory_mprintf( - "SELECT col_name, col_value FROM %s WHERE pk = ?1 AND col_name LIKE ?2 ORDER BY col_name", - table_blocks_ref(table)); - if (list_sql) { - dbvm_t *list_vm = NULL; - if (databasevm_prepare(data, list_sql, &list_vm, 0) == DBRES_OK) { - databasevm_bind_blob(list_vm, 1, pk, (int)pklen); - databasevm_bind_text(list_vm, 2, like_pattern, -1); - while (databasevm_step(list_vm) == DBRES_ROW) { - const char *bcn = database_column_text(list_vm, 0); - const char *bval = database_column_text(list_vm, 1); - const char *pos = block_extract_position_id(bcn); - if (pos && old_blocks) { - block_list_add(old_blocks, bval ? bval : "", pos); - } - } - databasevm_finalize(list_vm); - } - cloudsync_memory_free(list_sql); - } - cloudsync_memory_free(like_pattern); - } - } - - // Split new text into parts (NULL text = all blocks removed) - block_list_t *new_blocks = new_text ? block_split(new_text, delim) : block_list_create_empty(); - if (new_blocks && old_blocks) { - // Build array of new content strings (NULL when count is 0) - const char **new_parts = NULL; - if (new_blocks->count > 0) { - new_parts = (const char **)cloudsync_memory_alloc( - (uint64_t)(new_blocks->count * sizeof(char *))); - if (new_parts) { - for (int b = 0; b < new_blocks->count; b++) { - new_parts[b] = new_blocks->entries[b].content; - } - } - } - - if (new_parts || new_blocks->count == 0) { - block_diff_t *diff = block_diff(old_blocks->entries, old_blocks->count, - new_parts, new_blocks->count); - if (diff) { - for (int d = 0; d < diff->count; d++) { - block_diff_entry_t *de = &diff->entries[d]; - char *block_cn = block_build_colname(col, de->position_id); - if (!block_cn) continue; - - if (de->type == BLOCK_DIFF_ADDED || de->type == BLOCK_DIFF_MODIFIED) { - rc = local_mark_insert_or_update_meta(table, pk, pklen, block_cn, - db_version, cloudsync_bumpseq(data)); - // Store block value - if (rc == SQLITE_OK && table_block_value_write_stmt(table)) { - dbvm_t *wvm = table_block_value_write_stmt(table); - databasevm_bind_blob(wvm, 1, pk, (int)pklen); - databasevm_bind_text(wvm, 2, block_cn, -1); - databasevm_bind_text(wvm, 3, de->content, -1); - databasevm_step(wvm); - databasevm_reset(wvm); - } - } else if (de->type == BLOCK_DIFF_REMOVED) { - // Mark block as deleted in metadata (even col_version) - rc = local_mark_delete_block_meta(table, pk, pklen, block_cn, - db_version, cloudsync_bumpseq(data)); - // Remove from blocks table - if (rc == SQLITE_OK) { - block_delete_value_external(data, table, pk, pklen, block_cn); - } - } - cloudsync_memory_free(block_cn); - if (rc != SQLITE_OK) break; - } - block_diff_free(diff); - } - if (new_parts) cloudsync_memory_free((void *)new_parts); - } - } - if (new_blocks) block_list_free(new_blocks); - if (old_blocks) block_list_free(old_blocks); + rc = local_block_update(data, table, pk, pklen, i, + (const char *)database_value_text(payload->new_values[col_index]), db_version, false); if (rc != SQLITE_OK) goto cleanup; } else { // Regular column: mark as updated in the metadata (columns are in cid order) diff --git a/test/network_unit.c b/test/network_unit.c index f4ccb27c..f8fd2391 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -5,7 +5,8 @@ // Unit tests for the network layer's pure response-handling logic. Built with // networking ENABLED (unlike dist/unit, which is -DCLOUDSYNC_OMIT_NETWORK), so it // can call the internal functions directly on crafted in-memory NETWORK_RESULT -// buffers — no server, no sockets. +// buffers. The deadline regression also uses a stalled loopback HTTP socket; +// no external server is contacted. // #include @@ -127,7 +128,68 @@ static bool test_compute_status(void) { return ok; } +extern char *network_test_unescape(const char *); +static bool test_json_scope(void) { + char json[] = "{\"noise\":\"lastOptimisticVersion\",\"nested\":{\"lastConfirmedVersion\":999},\"lastOptimisticVersion\":42,\"lastConfirmedVersion\":7}"; + NETWORK_RESULT r = json_buffer(json); + int64_t optimistic = -1, confirmed = -1; + int gaps = -1; + char *apply = NULL, *check_failure = NULL; + network_sync_state_update_from_response(&r, &optimistic, &confirmed, &gaps, &apply, &check_failure); + bool ok = optimistic == 42 && confirmed == 7; + cloudsync_memory_free(apply); + cloudsync_memory_free(check_failure); + char nested[] = "{\"nested\":{\"lastOptimisticVersion\":999,\"lastConfirmedVersion\":999}}"; + r = json_buffer(nested); + apply = check_failure = NULL; + network_sync_state_update_from_response(&r, &optimistic, &confirmed, &gaps, &apply, &check_failure); + ok = ok && optimistic == 42 && confirmed == 7; + cloudsync_memory_free(apply); + cloudsync_memory_free(check_failure); + return ok; +} +static bool test_unicode(void) { + char *s = network_test_unescape("caf\\u00e9 \\u20ac \\ud83d\\ude80 \\/\\n"); + bool ok = s && strcmp(s, "caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x9a\x80 /\n") == 0; + cloudsync_memory_free(s); + const char *invalid[] = {"\\ud800", "\\udc00", "\\ud800\\u0041", "\\u0000", "\\uZZZZ", "\\u123", "\\"}; + for (size_t i = 0; i < sizeof(invalid) / sizeof(*invalid); i++) { + s = network_test_unescape(invalid[i]); + ok = ok && s == NULL; + cloudsync_memory_free(s); + } + return ok; +} + +#if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) +#include +#include +#include +extern bool network_test_curl_timeout(const char *, bool); +static bool test_stalled_http_timeout(void) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return false; + struct sockaddr_in address = {0}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + bool ok = bind(fd, (struct sockaddr *)&address, sizeof(address)) == 0 && listen(fd, 8) == 0; + socklen_t len = sizeof(address); + ok = ok && getsockname(fd, (struct sockaddr *)&address, &len) == 0; + char url[80]; + snprintf(url, sizeof(url), "http://127.0.0.1:%u/", ntohs(address.sin_port)); + // A listening socket that never sends HTTP simulates a stalled server. + if (ok) ok = network_test_curl_timeout(url, false) && network_test_curl_timeout(url, true); + close(fd); + return ok; +} +#endif + int main(void) { +#if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) + check("HTTP deadlines for new and reset pooled handles:", test_stalled_http_timeout()); +#endif + check("JSON keys only match root object members:", test_json_scope()); + check("JSON Unicode, surrogate pairs and malformed escapes:", test_unicode()); printf("\nNetwork unit tests\n"); check("optimistic/confirmed version folds latest-valid (allows rollback):", test_optimistic_version_rollback()); check("non-buffer response is a no-op:", test_non_buffer_is_noop()); diff --git a/test/postgresql/57_audit_regressions.sql b/test/postgresql/57_audit_regressions.sql new file mode 100644 index 00000000..865f6ea8 --- /dev/null +++ b/test/postgresql/57_audit_regressions.sql @@ -0,0 +1,78 @@ +-- Audit: database errors must not be mistaken for skippable RLS denials. +\set ON_ERROR_STOP on +\connect postgres +DROP DATABASE IF EXISTS cloudsync_audit_source; +DROP DATABASE IF EXISTS cloudsync_audit_target; +CREATE DATABASE cloudsync_audit_source; +CREATE DATABASE cloudsync_audit_target; +\connect cloudsync_audit_source +CREATE EXTENSION cloudsync; +CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('t'); +INSERT INTO t VALUES ('1','a'),('2','b'),('3','c'); +SELECT encode(cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq),'hex') AS audit_payload FROM cloudsync_changes \gset + +\connect cloudsync_audit_target +CREATE EXTENSION cloudsync; +CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('t'); +CREATE TEMP TABLE audit_payload(data BYTEA); +INSERT INTO audit_payload VALUES (decode(:'audit_payload','hex')); +CREATE FUNCTION deny_audit_row() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.id = current_setting('audit.denied_id') THEN + RAISE EXCEPTION 'audit write denied'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER deny_audit BEFORE INSERT ON t FOR EACH ROW EXECUTE FUNCTION deny_audit_row(); +DO $$ +DECLARE denied INTEGER; failed BOOLEAN; +BEGIN + FOR denied IN 1..3 LOOP + PERFORM set_config('audit.denied_id', denied::TEXT, false); + failed := false; + BEGIN + PERFORM cloudsync_payload_apply(data) FROM audit_payload; + EXCEPTION WHEN OTHERS THEN + IF SQLERRM NOT LIKE '%audit write denied%' THEN RAISE; END IF; + failed := true; + END; + IF NOT failed THEN RAISE EXCEPTION 'Payload silently ignored error at row %', denied; END IF; + IF EXISTS (SELECT FROM t) THEN RAISE EXCEPTION 'Failed payload committed partial data'; END IF; + IF coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) <> 0 THEN + RAISE EXCEPTION 'Failed payload advanced its checkpoint'; + END IF; + END LOOP; +END $$; +\echo [PASS] (57-audit) first, middle and final merge errors are propagated without checkpoint advancement + +DROP TRIGGER deny_audit ON t; +SELECT cloudsync_set_column('t','value','algo','block'); +CREATE FUNCTION deny_audit_block() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN RAISE EXCEPTION 'audit block write denied'; END $$; +CREATE TRIGGER deny_block BEFORE INSERT ON t_cloudsync_blocks FOR EACH ROW EXECUTE FUNCTION deny_audit_block(); +DO $$ +DECLARE failed BOOLEAN := false; +BEGIN + BEGIN INSERT INTO t VALUES('block','new'); + EXCEPTION WHEN OTHERS THEN failed := true; END; + IF NOT failed OR EXISTS(SELECT FROM t) THEN RAISE EXCEPTION 'Block insert failed to roll back'; END IF; +END $$; +DROP TRIGGER deny_block ON t_cloudsync_blocks; +INSERT INTO t VALUES('block','old'); +CREATE TRIGGER deny_block BEFORE INSERT ON t_cloudsync_blocks FOR EACH ROW EXECUTE FUNCTION deny_audit_block(); +DO $$ +DECLARE failed BOOLEAN := false; +BEGIN + BEGIN UPDATE t SET value='new' WHERE id='block'; + EXCEPTION WHEN OTHERS THEN failed := true; END; + IF NOT failed OR (SELECT value FROM t WHERE id='block') <> 'old' THEN + RAISE EXCEPTION 'Block update failed to roll back'; + END IF; +END $$; +\echo [PASS] (57-audit) block insert/update failures roll back base rows and metadata +\connect postgres +DROP DATABASE cloudsync_audit_source; +DROP DATABASE cloudsync_audit_target; +\set ON_ERROR_STOP off diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index fc760828..da0d11e0 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -64,6 +64,7 @@ \ir 54_payload_chunks_fragment_state.sql \ir 55_payload_chunks_positional_resume.sql \ir 56_many_columns.sql +\ir 57_audit_regressions.sql -- 'Test summary' \echo '\nTest summary:' diff --git a/test/review_regressions.c b/test/review_regressions.c new file mode 100644 index 00000000..af40466e --- /dev/null +++ b/test/review_regressions.c @@ -0,0 +1,183 @@ +// Focused audit regressions. No server or on-disk database required. +#include +#include +#include +#include +#include "sqlite3.h" +#include "cloudsync.h" +#include "cloudsync_sqlite.h" +#include "utils.h" +#include "pk.h" +extern int cloudsync_changesvtab_best_index(sqlite3_vtab *, sqlite3_index_info *); +static int failures; +#define CHECK(x) do { if (!(x)) { fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #x); failures++; } } while (0) +static sqlite3 *open_db(void) { + sqlite3 *db = NULL; + CHECK(sqlite3_open(":memory:", &db) == SQLITE_OK); + CHECK(sqlite3_cloudsync_init(db, NULL, NULL) == SQLITE_OK); + return db; +} +static int sql(sqlite3 *db, const char *query) { return sqlite3_exec(db, query, NULL, NULL, NULL); } +static int close_db(sqlite3 *db) { + CHECK(sql(db, "SELECT cloudsync_terminate()") == SQLITE_OK); + return sqlite3_close(db); +} +static int64_t scalar(sqlite3 *db, const char *query) { + sqlite3_stmt *vm = NULL; + int64_t value = INT64_MIN; + CHECK(sqlite3_prepare_v2(db, query, -1, &vm, NULL) == SQLITE_OK); + if (vm && sqlite3_step(vm) == SQLITE_ROW) value = sqlite3_column_int64(vm, 0); + else CHECK(false); + sqlite3_finalize(vm); + return value; +} +static void test_clocks_and_double(void) { + sqlite3 *db = open_db(); + CHECK(sql(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); SELECT cloudsync_init('t');") == SQLITE_OK); + CHECK(sql(db, "INSERT INTO cloudsync_changes VALUES ('t', cloudsync_pk_encode('key'), 'value', 'ok', 4294967297, 4294967300, randomblob(16), 4294967297, 4294967310);") == SQLITE_OK); + CHECK(scalar(db, "SELECT col_version FROM cloudsync_changes WHERE col_name='value'") == INT64_C(4294967297)); + CHECK(scalar(db, "SELECT db_version FROM cloudsync_changes WHERE col_name='value'") >= INT64_C(4294967300)); + CHECK(scalar(db, "SELECT seq FROM cloudsync_changes WHERE col_name='value'") == INT64_C(4294967310)); + CHECK(scalar(db, "SELECT cl FROM cloudsync_changes WHERE col_name='value'") == INT64_C(4294967297)); + CHECK(scalar(db, "SELECT count(*) FROM t WHERE value='ok'") == 1); + CHECK(scalar(db, "SELECT hex(cloudsync_pk_encode(1.0))='0102000000000000F03F'") == 1); + CHECK(scalar(db, "SELECT cloudsync_pk_decode(x'0102000000000000F03F',1)=1.0") == 1); + CHECK(scalar(db, "SELECT cloudsync_pk_decode(cloudsync_pk_encode(-1.5),1)=-1.5") == 1); + CHECK(close_db(db) == SQLITE_OK); +} +static void test_best_index(void) { + struct sqlite3_index_constraint constraints[3] = { + {.iColumn=0, .op=SQLITE_INDEX_CONSTRAINT_EQ, .usable=0}, + {.iColumn=1, .op=SQLITE_INDEX_CONSTRAINT_MATCH, .usable=1}, + {.iColumn=5, .op=SQLITE_INDEX_CONSTRAINT_GT, .usable=1} + }; + struct sqlite3_index_constraint_usage usage[3] = {{0}}; + sqlite3_index_info info = {.nConstraint=3, .aConstraint=constraints, .aConstraintUsage=usage}; + CHECK(cloudsync_changesvtab_best_index(NULL, &info) == SQLITE_OK); + CHECK(strcmp(info.idxStr, "WHERE db_version > ? ORDER BY db_version, seq ASC") == 0); + CHECK(usage[2].argvIndex == 1); + sqlite3_free(info.idxStr); + info.nConstraint = 2; + CHECK(cloudsync_changesvtab_best_index(NULL, &info) == SQLITE_OK); + CHECK(strcmp(info.idxStr, " ORDER BY db_version, seq ASC") == 0); + sqlite3_free(info.idxStr); +} +static void test_payload_errors(void) { + for (int denied = 1; denied <= 3; denied++) { + sqlite3 *source = open_db(), *target = open_db(); + const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL,value TEXT); SELECT cloudsync_init('t');"; + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('1','a'),('2','b'),('3','c');") == SQLITE_OK); + char trigger[256]; + snprintf(trigger, sizeof(trigger), "CREATE TRIGGER deny BEFORE INSERT ON t WHEN NEW.id='%d' BEGIN SELECT RAISE(ABORT,'denied'); END", denied); + CHECK(sql(target, trigger) == SQLITE_OK); + sqlite3_stmt *read = NULL, *write = NULL; + CHECK(sqlite3_prepare_v2(source, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes", -1, &read, NULL) == SQLITE_OK); + CHECK(sqlite3_step(read) == SQLITE_ROW); + CHECK(sqlite3_prepare_v2(target, "SELECT cloudsync_payload_decode(?1)", -1, &write, NULL) == SQLITE_OK); + CHECK(sqlite3_bind_value(write, 1, sqlite3_column_value(read, 0)) == SQLITE_OK); + CHECK(sqlite3_step(write) != SQLITE_ROW); + CHECK(strstr(sqlite3_errmsg(target), "denied") != NULL); + sqlite3_finalize(write); + sqlite3_finalize(read); + CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") == 0); + CHECK(sqlite3_get_autocommit(target)); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); + } + sqlite3 *db = open_db(); + CHECK(sql(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL); SELECT cloudsync_init('t');") == SQLITE_OK); + // v1 header: request a 4GB decompression without checksum/schema requirements. + CHECK(sql(db, "SELECT cloudsync_payload_decode(x'434C535901000000FFFFFFFF00090000000000000000000000000000000000000000')") != SQLITE_OK); + CHECK(strstr(sqlite3_errmsg(db), "exceeds limit") != NULL); + CHECK(sql(db, "SELECT cloudsync_payload_decode(x'434C5359010000001000000100090000000000000000000000000000000000000000')") != SQLITE_OK); + CHECK(strstr(sqlite3_errmsg(db), "exceeds limit") != NULL); + CHECK(close_db(db) == SQLITE_OK); +} +static void test_block_write_errors(void) { + for (int update = 0; update < 2; update++) { + sqlite3 *db = open_db(); + CHECK(sql(db, "CREATE TABLE docs(id TEXT PRIMARY KEY NOT NULL, body TEXT); SELECT cloudsync_init('docs'); SELECT cloudsync_set_column('docs','body','algo','block');") == SQLITE_OK); + if (update) CHECK(sql(db, "INSERT INTO docs VALUES('1','old')") == SQLITE_OK); + CHECK(sql(db, "CREATE TRIGGER deny_block BEFORE INSERT ON docs_cloudsync_blocks BEGIN SELECT RAISE(ABORT,'block write denied'); END") == SQLITE_OK); + CHECK(sql(db, update ? "UPDATE docs SET body='new' WHERE id='1'" : "INSERT INTO docs VALUES('1','new')") != SQLITE_OK); + CHECK(scalar(db, update ? "SELECT count(*) FROM docs WHERE body='old'" : "SELECT count(*) FROM docs") == update); + CHECK(close_db(db) == SQLITE_OK); + } +} +static void test_refill_error(void) { + sqlite3 *db = open_db(); + CHECK(sql(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL,a TEXT,b TEXT); SELECT cloudsync_init('t'); INSERT INTO t VALUES('1','a','b'),('2','a','b'); DELETE FROM t_cloudsync WHERE col_name='a';") == SQLITE_OK); + cloudsync_context *ctx = cloudsync_context_create(db); + CHECK(ctx && cloudsync_context_init(ctx)); + CHECK(sql(db, "CREATE TRIGGER deny_meta BEFORE INSERT ON t_cloudsync WHEN NEW.col_name='a' BEGIN SELECT RAISE(ABORT,'metadata denied'); END") == SQLITE_OK); + CHECK(cloudsync_refill_metatable(ctx, "t") != DBRES_OK); + CHECK(scalar(db, "SELECT count(*) FROM t_cloudsync WHERE col_name='a'") == 0); + cloudsync_context_free(ctx); + CHECK(close_db(db) == SQLITE_OK); +} + +static sqlite3_mem_methods memory; +static int fail_after = -1; +static bool fail_alloc(void) { + if (fail_after < 0) return false; + if (fail_after == 0) return true; + fail_after--; + return false; +} +static void *fault_malloc(int size) { return fail_alloc() ? NULL : memory.xMalloc(size); } +static void *fault_realloc(void *ptr, int size) { return fail_alloc() ? NULL : memory.xRealloc(ptr, size); } +static void test_block_oom(void) { + block_init_allocator(); + for (int kind = 0; kind < 5; kind++) { + bool succeeded = false; + for (int n = 0; n < 100 && !succeeded; n++) { + sqlite3_int64 before = sqlite3_memory_used(); + fail_after = n; + if (kind < 3) { + block_list_t *list = block_split(kind == 0 ? "" : "a\nb", kind == 1 ? "" : "\n"); + fail_after = -1; + if (list) { + CHECK(list->count == (kind == 2 ? 2 : 1)); + block_list_free(list); + succeeded = true; + } + } else if (kind == 3) { + block_entry_t old[] = {{.content="old", .position_id="a0"}, {.content="kept", .position_id="a1"}, {.content="removed", .position_id="a2"}}; + const char *parts[] = {"kept", "new"}; + block_diff_t *diff = block_diff(old, 3, parts, 2); + fail_after = -1; + if (diff) { CHECK(diff->count == 3); block_diff_free(diff); succeeded = true; } + } else { + block_list_t *list = block_list_create_empty(); + if (list) { + bool added = block_list_add(list, "content", "a0"); + fail_after = -1; + CHECK(list->count == (added ? 1 : 0)); + block_list_free(list); + succeeded = added; + } else fail_after = -1; + } + CHECK(sqlite3_memory_used() == before); + } + CHECK(succeeded); + } +} +int main(void) { + CHECK(sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memory) == SQLITE_OK); + sqlite3_mem_methods faults = memory; + faults.xMalloc = fault_malloc; + faults.xRealloc = fault_realloc; + CHECK(sqlite3_config(SQLITE_CONFIG_MALLOC, &faults) == SQLITE_OK); + CHECK(sqlite3_initialize() == SQLITE_OK); + test_clocks_and_double(); + test_best_index(); + test_payload_errors(); + test_block_write_errors(); + test_refill_error(); + test_block_oom(); + cloudsync_memory_finalize(); + CHECK(sqlite3_memory_used() == 0); + printf("Audit regressions: %d failures\n", failures); + return failures ? 1 : 0; +} diff --git a/test/unit.c b/test/unit.c index ca7d6fc6..97b2ea2d 100644 --- a/test/unit.c +++ b/test/unit.c @@ -18,6 +18,7 @@ #include #else #include +#include #endif #include "pk.h" @@ -30,6 +31,7 @@ extern char *OUT_OF_MEMORY_BUFFER; extern bool force_vtab_filter_abort; extern bool force_uncompressed_blob; +static char test_directory[192]; void dbvm_reset (dbvm_t *stmt); int dbvm_count (dbvm_t *stmt, const char *value, size_t len, int type); @@ -1130,7 +1132,7 @@ bool do_alter_tables (int table_mask, sqlite3 *db, int alter_version) { } if (table_mask & TEST_NOCOLS) { - const char *sql; + const char *sql = NULL; switch (alter_version) { case 1: sql = "SELECT cloudsync_begin_alter('" CUSTOMERS_NOCOLS_TABLE "'); " @@ -1168,7 +1170,7 @@ bool do_alter_tables (int table_mask, sqlite3 *db, int alter_version) { if (table_mask & TEST_NOPRIKEYS) { // TEST a table with implicit rowid primary key - const char *sql; + const char *sql = NULL; switch (alter_version) { case 1: sql = "SELECT cloudsync_begin_alter('customers_noprikey'); " @@ -1735,7 +1737,7 @@ bool do_test_rowid (int ntest, bool print_result) { // for an explanation see https://github.com/sqliteai/sqlite-sync/blob/main/docs/RowID.md int64_t db_version = random_int64_range(1, 17179869183); int64_t seq = random_int64_range(1, 1073741823); - int64_t rowid = (db_version << 30) | seq; + int64_t rowid = (int64_t)(((uint64_t)db_version << 30) | (uint64_t)seq); int64_t value1; int64_t value2; @@ -1747,7 +1749,7 @@ bool do_test_rowid (int ntest, bool print_result) { // special case that failed in an old version int64_t db_version = 14963874252; int64_t seq = 172784902; - int64_t rowid = (db_version << 30) | seq; + int64_t rowid = (int64_t)(((uint64_t)db_version << 30) | (uint64_t)seq); int64_t value1; int64_t value2; @@ -2346,11 +2348,7 @@ bool do_test_stale_table_settings(bool cleanup_databases) { char dbpath[256]; time_t timestamp = time(NULL); - #ifdef __ANDROID__ - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-stale-%ld.sqlite", ".", timestamp); - #else - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-stale-%ld.sqlite", getenv("HOME"), timestamp); - #endif + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-stale-%ld.sqlite", test_directory, timestamp); // Phase 1: create database, table, and init cloudsync sqlite3 *db = NULL; @@ -2417,11 +2415,7 @@ bool do_test_stale_table_settings_dropped_meta(bool cleanup_databases) { char dbpath[256]; time_t timestamp = time(NULL); - #ifdef __ANDROID__ - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-stale-meta-%ld.sqlite", ".", timestamp); - #else - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-stale-meta-%ld.sqlite", getenv("HOME"), timestamp); - #endif + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-stale-meta-%ld.sqlite", test_directory, timestamp); // Phase 1: create database, table, and init cloudsync sqlite3 *db = NULL; @@ -4130,18 +4124,64 @@ sqlite3 *do_create_database (void) { return db; } +static bool create_test_directory(void) { +#ifdef _WIN32 + char base[MAX_PATH]; + DWORD n = GetTempPathA(sizeof(base), base); + if (!n || n >= sizeof(base)) return false; + int len = snprintf(test_directory, sizeof(test_directory), "%scloudsync-%lu-%llu", base, + (unsigned long)GetCurrentProcessId(), (unsigned long long)GetTickCount64()); + return len > 0 && len < sizeof(test_directory) && CreateDirectoryA(test_directory, NULL); +#else + const char *base = getenv("TMPDIR"); + if (!base || !*base) { +#ifdef __ANDROID__ + base = "."; // Android test runners execute from /data/local/tmp. +#else + base = "/tmp"; +#endif + } + int len = snprintf(test_directory, sizeof(test_directory), "%s/cloudsync-test-XXXXXX", base); + return len > 0 && (size_t)len < sizeof(test_directory) && mkdtemp(test_directory) != NULL; +#endif +} + +static bool remove_test_directory(void) { + char path[512]; +#ifdef _WIN32 + WIN32_FIND_DATAA entry; + snprintf(path, sizeof(path), "%s\\*", test_directory); + HANDLE handle = FindFirstFileA(path, &entry); + if (handle == INVALID_HANDLE_VALUE) return false; + do { + if (strncmp(entry.cFileName, "cloudsync-test-", 15) != 0) continue; + snprintf(path, sizeof(path), "%s\\%s", test_directory, entry.cFileName); + DeleteFileA(path); + } while (FindNextFileA(handle, &entry)); + FindClose(handle); + return RemoveDirectoryA(test_directory) != 0; +#else + DIR *dir = opendir(test_directory); + if (!dir) return false; + struct dirent *entry; + while ((entry = readdir(dir)) != NULL) { + if (strncmp(entry->d_name, "cloudsync-test-", 15) != 0) continue; + snprintf(path, sizeof(path), "%s/%s", test_directory, entry->d_name); + unlink(path); + } + closedir(dir); + return rmdir(test_directory) == 0; +#endif +} + void do_build_database_path (char buf[256], int i, time_t timestamp, int ntest) { - #ifdef __ANDROID__ - snprintf(buf, 256, "%s/cloudsync-test-%ld-%d-%d.sqlite", ".", timestamp, ntest, i); - #else - snprintf(buf, 256, "%s/cloudsync-test-%ld-%d-%d.sqlite", getenv("HOME"), timestamp, ntest, i); - #endif + snprintf(buf, 256, "%s/cloudsync-test-%ld-%d-%d.sqlite", test_directory, timestamp, ntest, i); } sqlite3 *do_create_database_file_v2 (int i, time_t timestamp, int ntest) { sqlite3 *db = NULL; - // open database in home dir + // Open database in the private per-run temporary directory. char buf[256]; do_build_database_path(buf, i, timestamp, ntest); int rc = sqlite3_open(buf, &db); @@ -9241,11 +9281,7 @@ bool do_test_block_column_reload(bool cleanup_databases) { char dbpath[256]; time_t timestamp = time(NULL); - #ifdef __ANDROID__ - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockreload-%ld.sqlite", ".", timestamp); - #else - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockreload-%ld.sqlite", getenv("HOME"), timestamp); - #endif + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockreload-%ld.sqlite", test_directory, timestamp); // Phase 1: create database, table, init cloudsync, mark a column as block algo. // Use a custom delimiter so both "algo" and "delimiter" rows get persisted. @@ -9346,11 +9382,7 @@ bool do_test_block_lww_existing_data(bool cleanup_databases) { char dbpath[256]; time_t timestamp = time(NULL); - #ifdef __ANDROID__ - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", ".", timestamp); - #else - snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", getenv("HOME"), timestamp); - #endif + snprintf(dbpath, sizeof(dbpath), "%s/cloudsync-test-blockexist-%ld.sqlite", test_directory, timestamp); int rc = sqlite3_open(dbpath, &db); if (rc != SQLITE_OK) return false; @@ -13389,6 +13421,10 @@ int test_report(const char *description, bool result){ } int main (int argc, const char * argv[]) { + if (!create_test_directory()) { + fprintf(stderr, "Unable to create private test directory\n"); + return 1; + } sqlite3 *db = NULL; int result = 0; bool print_result = false; @@ -13590,6 +13626,8 @@ int main (int argc, const char * argv[]) { printf("\tleaked: %" PRId64 " B\n", memory_used); result++; } + + if (cleanup_databases) result += test_report("Temporary Directory Cleanup:", remove_test_directory()); return result; } From 9d89fbb3aa5d73b122b66f2762087291064d0352 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 11:13:27 -0600 Subject: [PATCH 02/28] test(postgres): expect the reported error when an apply is lock-blocked cloudsync_payload_apply now keeps the first error instead of letting a later successful row overwrite it, so a lock-blocked apply reports the failure rather than returning quietly. Test 39 encoded the old lenient behaviour and aborted the script under ON_ERROR_STOP; it now tolerates the error the way tests 41, 46 and 53 already do, and still asserts the row kept its old value. Also restores the changelog workflow's v-prefixed tag filter. Widening it made the workflow fire but it then failed: the called workflow derives the version with ${GITHUB_REF#refs/tags/v} and rejects our unprefixed tags. Fixing that needs a change in changelog-action, so the filter goes back and the note records why, keeping the manual run. Adds the 1.1.4 changelog entry. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- .github/workflows/changelog.yml | 8 ++++++-- CHANGELOG.md | 16 ++++++++++++++++ test/postgresql/39_concurrent_write_apply.sql | 4 ++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 107ae0c4..b958e391 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -3,8 +3,12 @@ name: Release and Update Website Changelog on: push: tags: - # main.yml creates release tags without a "v" prefix (e.g. 1.1.3) - - "*.*.*" + # NOTE: this never matches. main.yml creates release tags without a "v" + # prefix (e.g. 1.1.3), so the changelog is published by running this + # workflow manually. Widening the filter is not enough on its own: the + # called workflow derives the version with ${GITHUB_REF#refs/tags/v} and + # rejects an unprefixed tag. Fixing it needs a change in changelog-action. + - "v*.*.*" workflow_dispatch: inputs: test_version: diff --git a/CHANGELOG.md b/CHANGELOG.md index 44d16fc7..b0adcd10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [1.1.4] - 2026-09-11 + +### Added + +- **Network requests now have deadlines** — 30 seconds to connect and 300 seconds in total, applied to pooled handles as well as new ones. A stalled server previously left a sync call waiting indefinitely. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. +- **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. + +### Fixed + +- **A failed payload write now reports the error and leaves the receive cursor where it was.** An error on one row could previously be overwritten by a later successful row, so `cloudsync_payload_apply` could report success after dropping changes and still advance the checkpoint — losing them silently. PostgreSQL's RLS `WITH CHECK` rejection remains a skippable policy outcome, distinct from generic SQL and permission errors: the call still reports the rows it processed, but does not advance the cursor when a policy denied rows. SQLite keeps its existing per-group partial-application behaviour. +- **Primary-key doubles keep their deployed little-endian IEEE754 byte order on every architecture.** The previous code combined host conversion with manual big-endian serialization; the historical format is now explicit. Integer keys are unchanged, and no migration is needed for little-endian deployments. +- **PostgreSQL no longer frees a tuple table belonging to another open cursor.** A block write that failed while a second SPI cursor was active could release rows still in use; tuple tables are now owned per statement. +- **Block-level LWW text writes roll back cleanly when a block write fails**, instead of leaving the row partially written. +- **64-bit clock values above `UINT32_MAX` are handled correctly on incoming changes** — column and database versions, causal length, and sequence. +- **The Node package rejects `ia32` on every operating system** rather than selecting an incompatible binary. `x64` and `arm64-musl` selection is unchanged. + ## [1.1.3] - 2026-09-11 ### Added diff --git a/test/postgresql/39_concurrent_write_apply.sql b/test/postgresql/39_concurrent_write_apply.sql index d84397ed..a9c850ec 100644 --- a/test/postgresql/39_concurrent_write_apply.sql +++ b/test/postgresql/39_concurrent_write_apply.sql @@ -81,9 +81,13 @@ BEGIN; \set ON_ERROR_ROLLBACK on SET LOCAL lock_timeout = '500ms'; +-- Expected: the apply cannot take its lock and reports the failure — locally +-- disable ON_ERROR_STOP. ON_ERROR_ROLLBACK keeps the transaction usable. +\set ON_ERROR_STOP off \if :payload_upd_ok SELECT cloudsync_payload_apply(decode(substr(:'payload_upd', 3), 'hex')) AS _blocked_apply \gset \endif +\set ON_ERROR_STOP on COMMIT; \set ON_ERROR_ROLLBACK off From c6200a88aa2ffaf6bc912a7177fb79519f1a89e9 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 11:38:33 -0600 Subject: [PATCH 03/28] fix(network): unwrap the gateway data envelope before scoped key lookups Key lookups are now scoped to one object, but six reads on the send and status path take their key from a raw response body, where the gateway wraps every success payload in {"data": ...}: the upload URL, the three sync-state fields, and both failure stages. Root-scoped, they stopped resolving, so every send failed with "missing 'url' in upload response" while the local suites stayed green. Those readers now resolve the payload first, the way the /check path already does, keeping lookups scoped to a single object. Chunk objects sliced out of chunks[] and legacy unwrapped bodies fall through unchanged. The new test covers the documented shapes in both directions: an enveloped status payload with gaps and failures, a legacy unwrapped body, an enveloped url staying invisible to a root-scoped read, and a sliced chunk object resolving directly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- src/network/network.c | 48 ++++++++++++++++++++++++++++++++++------ test/network_unit.c | 51 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/network/network.c b/src/network/network.c index 512af7d5..bea15020 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -992,6 +992,12 @@ static char *json_extract_string(const char *json, size_t json_len, const char * return result; } +#ifdef CLOUDSYNC_UNITTEST +char *network_test_extract_string(const char *json, const char *key) { + return json_extract_string(json, strlen(json), key); +} +#endif + static int64_t json_extract_int(const char *json, size_t json_len, const char *key, int64_t default_value) { if (!json || json_len == 0 || !key) return default_value; @@ -1391,6 +1397,20 @@ typedef struct { int64_t send_bytes; // serialized payload bytes sent this call } sync_result; +// Gateway success responses wrap the payload in {"data": {...}}; legacy servers +// and chunk objects sliced out of a chunks array are not wrapped. Key lookups are +// scoped to one object, so a caller reading a raw response body resolves the +// payload first. Frees through *owned. Mirrors the /check unwrap below. +static const char *json_response_payload(const char *json, size_t json_len, char **owned, size_t *payload_len) { + *owned = json_extract_object_raw(json, json_len, "data"); + if (*owned) { + *payload_len = strlen(*owned); + return *owned; + } + *payload_len = json_len; + return json; +} + // Returns a malloc'd raw JSON copy of failures. ("apply" or "check"), // or NULL when the field is missing or is JSON null. Caller frees with cloudsync_memory_free. static char *json_extract_failure_stage(const char *json, size_t json_len, const char *stage_key) { @@ -1646,7 +1666,12 @@ static int network_send_payload_to_apply(sqlite3_context *context, network_data return SQLITE_ERROR; } - char *s3_url = json_extract_string(upload_res.buffer, upload_res.blen, "url"); + char *upload_payload_owned = NULL; + size_t upload_payload_len = 0; + const char *upload_payload = json_response_payload(upload_res.buffer, upload_res.blen, + &upload_payload_owned, &upload_payload_len); + char *s3_url = json_extract_string(upload_payload, upload_payload_len, "url"); + cloudsync_memory_free(upload_payload_owned); if (!s3_url) { sqlite3_result_error(context, "cloudsync_network_send_changes: missing 'url' in upload response.", -1); network_result_cleanup(&upload_res); @@ -1690,24 +1715,29 @@ void network_sync_state_update_from_response(NETWORK_RESULT *res, // BACKWARD on a rollback when a later send chunk fails, and lastOptimisticVersion // becomes the durable send checkpoint — masking a decrease would advance the // checkpoint past the rolled-back changes and silently drop them. - int64_t parsed_optimistic = json_extract_int(res->buffer, res->blen, "lastOptimisticVersion", -1); + char *state_owned = NULL; + size_t state_len = 0; + const char *state_json = json_response_payload(res->buffer, res->blen, &state_owned, &state_len); + + int64_t parsed_optimistic = json_extract_int(state_json, state_len, "lastOptimisticVersion", -1); if (parsed_optimistic >= 0) *last_optimistic_version = parsed_optimistic; - int64_t parsed_confirmed = json_extract_int(res->buffer, res->blen, "lastConfirmedVersion", -1); + int64_t parsed_confirmed = json_extract_int(state_json, state_len, "lastConfirmedVersion", -1); if (parsed_confirmed >= 0) *last_confirmed_version = parsed_confirmed; - int parsed_gaps_size = json_extract_array_size(res->buffer, res->blen, "gaps"); + int parsed_gaps_size = json_extract_array_size(state_json, state_len, "gaps"); if (parsed_gaps_size >= 0) *gaps_size = parsed_gaps_size; - char *apply_failure = json_extract_failure_stage(res->buffer, res->blen, "apply"); + char *apply_failure = json_extract_failure_stage(state_json, state_len, "apply"); if (apply_failure) { if (*apply_failure_json) cloudsync_memory_free(*apply_failure_json); *apply_failure_json = apply_failure; } - char *check_failure = json_extract_failure_stage(res->buffer, res->blen, "check"); + char *check_failure = json_extract_failure_stage(state_json, state_len, "check"); if (check_failure) { if (*check_failure_json) cloudsync_memory_free(*check_failure_json); *check_failure_json = check_failure; } + cloudsync_memory_free(state_owned); #ifdef CLOUDSYNC_NETWORK_TRACE // Full endpoint response body that the sync-state fields above were parsed from. @@ -1752,7 +1782,11 @@ void cloudsync_network_has_unsent_changes (sqlite3_context *context, int argc, s int64_t last_optimistic_version = -1; if (res.code == CLOUDSYNC_NETWORK_BUFFER && res.buffer) { - last_optimistic_version = json_extract_int(res.buffer, res.blen, "lastOptimisticVersion", -1); + char *ack_owned = NULL; + size_t ack_len = 0; + const char *ack_json = json_response_payload(res.buffer, res.blen, &ack_owned, &ack_len); + last_optimistic_version = json_extract_int(ack_json, ack_len, "lastOptimisticVersion", -1); + cloudsync_memory_free(ack_owned); } else if (res.code != CLOUDSYNC_NETWORK_OK) { network_result_to_sqlite_error(context, res, "unable to retrieve current status from remote host."); network_result_cleanup(&res); diff --git a/test/network_unit.c b/test/network_unit.c index f8fd2391..ce0911f4 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -129,6 +129,7 @@ static bool test_compute_status(void) { } extern char *network_test_unescape(const char *); +extern char *network_test_extract_string(const char *, const char *); static bool test_json_scope(void) { char json[] = "{\"noise\":\"lastOptimisticVersion\",\"nested\":{\"lastConfirmedVersion\":999},\"lastOptimisticVersion\":42,\"lastConfirmedVersion\":7}"; NETWORK_RESULT r = json_buffer(json); @@ -148,6 +149,55 @@ static bool test_json_scope(void) { cloudsync_memory_free(check_failure); return ok; } + +// Gateway success responses wrap the payload in {"data": ...} (API.md, "Success +// envelope"); legacy servers do not. Key lookups stay scoped to one object, so +// readers of a raw response body must unwrap first. +static bool test_json_envelope(void) { + char enveloped[] = "{\"data\":{\"nested\":{\"lastOptimisticVersion\":999}," + "\"lastOptimisticVersion\":15,\"lastConfirmedVersion\":12}}"; + NETWORK_RESULT r = json_buffer(enveloped); + int64_t optimistic = -1, confirmed = -1; + int gaps = -1; + char *apply = NULL, *check_failure = NULL; + network_sync_state_update_from_response(&r, &optimistic, &confirmed, &gaps, &apply, &check_failure); + bool ok = optimistic == 15 && confirmed == 12; + cloudsync_memory_free(apply); + cloudsync_memory_free(check_failure); + + // the enveloped 202 status payload also carries gaps and failures + char full[] = "{\"data\":{\"lastOptimisticVersion\":20,\"lastConfirmedVersion\":18," + "\"gaps\":[{\"dbVersionMin\":13,\"dbVersionMax\":15}]," + "\"failures\":{\"apply\":null,\"check\":{\"code\":\"boom\",\"retryable\":false}}}}"; + r = json_buffer(full); + apply = check_failure = NULL; + network_sync_state_update_from_response(&r, &optimistic, &confirmed, &gaps, &apply, &check_failure); + ok = ok && optimistic == 20 && confirmed == 18 && gaps == 1; + ok = ok && check_failure && strstr(check_failure, "boom"); + cloudsync_memory_free(apply); + cloudsync_memory_free(check_failure); + + // a legacy un-enveloped body still parses + char legacy[] = "{\"lastOptimisticVersion\":7,\"lastConfirmedVersion\":5}"; + r = json_buffer(legacy); + apply = check_failure = NULL; + network_sync_state_update_from_response(&r, &optimistic, &confirmed, &gaps, &apply, &check_failure); + ok = ok && optimistic == 7 && confirmed == 5; + cloudsync_memory_free(apply); + cloudsync_memory_free(check_failure); + + // key lookups remain scoped to one object: an enveloped url is not visible + // to a root-scoped read, which is why raw-response readers unwrap first + char *url = network_test_extract_string("{\"data\":{\"url\":\"https://s3/a\"}}", "url"); + ok = ok && url == NULL; + cloudsync_memory_free(url); + + // an un-enveloped chunk object sliced out of chunks[] resolves directly + url = network_test_extract_string("{\"cursor\":0,\"url\":\"https://s3/b\",\"watermark\":18}", "url"); + ok = ok && url && strcmp(url, "https://s3/b") == 0; + cloudsync_memory_free(url); + return ok; +} static bool test_unicode(void) { char *s = network_test_unescape("caf\\u00e9 \\u20ac \\ud83d\\ude80 \\/\\n"); bool ok = s && strcmp(s, "caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x9a\x80 /\n") == 0; @@ -189,6 +239,7 @@ int main(void) { check("HTTP deadlines for new and reset pooled handles:", test_stalled_http_timeout()); #endif check("JSON keys only match root object members:", test_json_scope()); + check("Gateway data envelope is unwrapped before scoped lookups:", test_json_envelope()); check("JSON Unicode, surrogate pairs and malformed escapes:", test_unicode()); printf("\nNetwork unit tests\n"); check("optimistic/confirmed version folds latest-valid (allows rollback):", test_optimistic_version_rollback()); From a1f3ea33e59d987e0dea93817bc56f2b7ce60f73 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 11:48:44 -0600 Subject: [PATCH 04/28] fix(network): unwrap the envelope for failures.check on the receive path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opportunistic failures.check read takes its key from the raw /check response body, which the gateway wraps in {"data": ...}. Root-scoped it returned NULL, so a server-reported check failure was never surfaced — silently, since the field is optional. This was the one site missed by the previous commit; every remaining lookup now reads either an unwrapped payload, a chunk object sliced out of chunks[], or an already-extracted sub-object. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- src/network/network.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/network/network.c b/src/network/network.c index bea15020..bed14dd8 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -2157,7 +2157,11 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync if (data_json) cloudsync_memory_free(data_json); // failures.check may appear in either shape; extract opportunistically. if (out) { - char *check_failure = json_extract_failure_stage(result.buffer, result.blen, "check"); + char *failure_owned = NULL; + size_t failure_len = 0; + const char *failure_json = json_response_payload(result.buffer, result.blen, &failure_owned, &failure_len); + char *check_failure = json_extract_failure_stage(failure_json, failure_len, "check"); + cloudsync_memory_free(failure_owned); if (check_failure) { if (out->check_failure_json) cloudsync_memory_free(out->check_failure_json); out->check_failure_json = check_failure; From f68c523f2e977bcc8a3d29de09058e698ecf2426 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 14:45:19 -0600 Subject: [PATCH 05/28] fix(apply): skip and report RLS-denied rows instead of stalling the cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A denial suppressed the receive checkpoint, which gave neither progress nor a signal. The rows are permanently not this site's to hold, so the next check re-delivered them, they were denied again, and nothing ever surfaced to break the cycle. One denied row also stalled every later change behind it. Worse in a chunked batch: policy_denied was a local of one apply call, but each chunk is a separate call. A denial in a non-final chunk suppressed a checkpoint that was already a no-op (non-final chunks pass CHECKPOINT_NONE), the flag died with the call, and the final chunk advanced the cursor past the denied rows — dropping them silently, the shape this was meant to prevent. Denied entries are now counted, skipped, and the cursor advances. The count accumulates across the drain on the context and is reported as receive.denied, so discarding stays visible: a non-zero denied with zero rows is the shape of an apply connection with no session identity. Deliberately not an error, even when every row is denied: a single-row payload belonging to another user is denied in full and is a correct outcome, which tests 27 and 29 already assert. Test 27 now checks the cursor moves past a denied apply. Verified it fails ("left the checkpoint at 4, expected > 4") with the old suppression restored. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- CHANGELOG.md | 3 +- src/cloudsync.c | 36 ++++++++++++++++++++---- src/cloudsync.h | 6 ++++ src/network/network.c | 38 +++++++++++++++----------- test/postgresql/27_rls_batch_merge.sql | 14 ++++++++++ 5 files changed, 75 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0adcd10..39112898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added +- **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. - **Network requests now have deadlines** — 30 seconds to connect and 300 seconds in total, applied to pooled handles as well as new ones. A stalled server previously left a sync call waiting indefinitely. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. - **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. ### Fixed -- **A failed payload write now reports the error and leaves the receive cursor where it was.** An error on one row could previously be overwritten by a later successful row, so `cloudsync_payload_apply` could report success after dropping changes and still advance the checkpoint — losing them silently. PostgreSQL's RLS `WITH CHECK` rejection remains a skippable policy outcome, distinct from generic SQL and permission errors: the call still reports the rows it processed, but does not advance the cursor when a policy denied rows. SQLite keeps its existing per-group partial-application behaviour. +- **A failed payload write now reports the error and leaves the receive cursor where it was.** An error on one row could previously be overwritten by a later successful row, so `cloudsync_payload_apply` could report success after dropping changes and still advance the checkpoint — losing them silently. SQLite keeps its existing per-group partial-application behaviour. - **Primary-key doubles keep their deployed little-endian IEEE754 byte order on every architecture.** The previous code combined host conversion with manual big-endian serialization; the historical format is now explicit. Integer keys are unchanged, and no migration is needed for little-endian deployments. - **PostgreSQL no longer frees a tuple table belonging to another open cursor.** A block write that failed while a second SPI cursor was active could release rows still in use; tuple tables are now owned per statement. - **Block-level LWW text writes roll back cleanly when a block write fails**, instead of leaving the row partially written. diff --git a/src/cloudsync.c b/src/cloudsync.c index 78468254..c0c5b577 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -192,6 +192,11 @@ struct cloudsync_context { // CLOUDSYNC_CHECKPOINT_LAST_APPLIED receive-checkpoint mode (-1 = none yet). int64_t apply_last_db_version; int64_t apply_last_seq; + + // payload entries rejected by a row-level security policy, accumulated across + // a receive drain so a denial in one chunk is still visible when a later chunk + // reports. Reset with cloudsync_apply_denied_reset. + int apply_denied; }; struct cloudsync_table_context { @@ -617,6 +622,14 @@ const char *cloudsync_errmsg (cloudsync_context *data) { return data->errmsg; } +void cloudsync_apply_denied_reset (cloudsync_context *data) { + if (data) data->apply_denied = 0; +} + +int cloudsync_apply_denied_count (cloudsync_context *data) { + return (data) ? data->apply_denied : 0; +} + int cloudsync_errcode (cloudsync_context *data) { return data->errcode; } @@ -4114,7 +4127,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b uint32_t nrows = header.nrows; int64_t last_payload_db_version = -1; int first_error = DBRES_OK; - bool policy_denied = false; + int denied_entries = 0; char first_error_message[1024] = {0}; cloudsync_pk_decode_bind_context decoded_context = {.vm = vm}; @@ -4152,8 +4165,9 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // Flush pending batch before any boundary change if (pk_changed || tbl_changed || db_version_changed) { + int pending_entries = batch.count; int flush_rc = merge_flush_pending(data); - if (flush_rc == DBRES_POLICY_DENIED) policy_denied = true; + if (flush_rc == DBRES_POLICY_DENIED) denied_entries += (pending_entries > 0) ? pending_entries : 1; else if (flush_rc != DBRES_OK && first_error == DBRES_OK) { first_error = flush_rc; snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); @@ -4200,7 +4214,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b last_tbl_len = decoded_context.tbl_len; rc = databasevm_step(vm); - if (rc == DBRES_POLICY_DENIED) { policy_denied = true; rc = DBRES_DONE; } + if (rc == DBRES_POLICY_DENIED) { denied_entries++; rc = DBRES_DONE; } if (rc != DBRES_DONE) { if (first_error == DBRES_OK) { first_error = rc; @@ -4217,8 +4231,9 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // Final flush after loop { + int pending_entries = batch.count; int flush_rc = merge_flush_pending(data); - if (flush_rc == DBRES_POLICY_DENIED) policy_denied = true; + if (flush_rc == DBRES_POLICY_DENIED) denied_entries += (pending_entries > 0) ? pending_entries : 1; else if (flush_rc != DBRES_OK && first_error == DBRES_OK) { first_error = flush_rc; snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); @@ -4238,6 +4253,17 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b } if (rc == DBRES_DONE) rc = DBRES_OK; + + data->apply_denied += denied_entries; + + // A policy denial is permanent: those rows are not this site's to hold, so the + // cursor must still advance. Holding it back would re-deliver the same rows on + // every check forever, and a single denied row would stall every later change + // behind it. Denials are counted and reported instead (receive.denied), so + // discarding stays visible without being an error: a payload can be entirely + // denied and still be a correct outcome, since a single-row payload that + // belongs to another user is denied in full. + if (rc == DBRES_OK) { // Record the last applied (db_version, seq) and advance the receive cursor // once, gated on the caller-supplied checkpoint. A non-final chunk passes @@ -4247,7 +4273,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b data->apply_last_db_version = decoded_context.db_version; data->apply_last_seq = decoded_context.seq; } - if (!policy_denied) cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); + cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); } cleanup: diff --git a/src/cloudsync.h b/src/cloudsync.h index 56077d8c..2c4bef5b 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -110,6 +110,12 @@ int cloudsync_set_dberror (cloudsync_context *data); const char *cloudsync_errmsg (cloudsync_context *data); int cloudsync_errcode (cloudsync_context *data); void cloudsync_reset_error (cloudsync_context *data); + +// Payload entries rejected by a row-level security policy. The count accumulates +// across a receive drain (reset once before it) so denials in an early chunk are +// still reported by the call that finishes the drain. +void cloudsync_apply_denied_reset (cloudsync_context *data); +int cloudsync_apply_denied_count (cloudsync_context *data); int cloudsync_commit_hook (void *ctx); void cloudsync_rollback_hook (void *ctx); void cloudsync_set_schema (cloudsync_context *data, const char *schema); diff --git a/src/network/network.c b/src/network/network.c index bed14dd8..7022ed2f 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -2192,6 +2192,7 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync // Result of a receive drain (see network_drain_changes). typedef struct { int rows; // cumulative rows applied across the drain + int denied; // payload entries rejected by a row-level security policy int chunks; // payload chunks applied this drain int64_t bytes; // serialized payload bytes received this drain bool complete; // true iff the receive stream is fully drained (nothing pending) @@ -2216,6 +2217,10 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, int64_t drain_prev_dbv = cloudsync_dbversion(data); sr->defer_tables = true; + // Denials accumulate on the context across every chunk of this drain, so a + // denial in an early chunk is still reported by the call that finishes it. + cloudsync_apply_denied_reset(data); + int ntries = 0; // counts only "nothing ready" (202) polls int nrows_total = 0; // cumulative rows applied across the whole drain int nchunks = 0; // payload chunks applied this call @@ -2282,6 +2287,7 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, } dr->rows = nrows_total; + dr->denied = cloudsync_apply_denied_count(data); dr->chunks = nchunks; dr->bytes = bytes_total; dr->complete = complete; @@ -2334,20 +2340,20 @@ void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retr char *recv_part; if (escaped_err && sr.check_failure_json) { recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}", - nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err, sr.check_failure_json); + "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}", + nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err, sr.check_failure_json); } else if (escaped_err) { recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}", - nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err); + "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}", + nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err); } else if (sr.check_failure_json) { recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}", - nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); + "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}", + nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); } else { recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}", - nrows_total, tables, dr.chunks, (long long)dr.bytes, complete_str); + "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}", + nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str); } char *buf = cloudsync_memory_mprintf("{%s,%s}", send_part, recv_part); @@ -2434,17 +2440,17 @@ static void network_receive_changes_impl (sqlite3_context *context, int max_chun char *escaped = receive_err ? json_escape_string(receive_err) : NULL; char *buf; if (escaped && sr.check_failure_json) { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}}", - nrows, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped, sr.check_failure_json); + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}}", + nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped, sr.check_failure_json); } else if (escaped) { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}}", - nrows, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped); + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}}", + nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped); } else if (sr.check_failure_json) { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}}", - nrows, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}}", + nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); } else { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}}", - nrows, tables, dr.chunks, (long long)dr.bytes, complete_str); + buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}}", + nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str); } sqlite3_result_text(context, buf, -1, cloudsync_memory_free); if (escaped) cloudsync_memory_free(escaped); diff --git a/test/postgresql/27_rls_batch_merge.sql b/test/postgresql/27_rls_batch_merge.sql index 2ab51bfd..0e03d52b 100644 --- a/test/postgresql/27_rls_batch_merge.sql +++ b/test/postgresql/27_rls_batch_merge.sql @@ -277,6 +277,8 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_5 FROM cloudsync_changes \gset -- Apply as test_rls_user with USER1 identity — should be denied (doc4 owned by USER2) \connect cloudsync_test_27_b \ir helper_psql_conn_setup.sql +-- read the receive cursor on the target, as superuser, before dropping to the RLS role +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before_denied \gset SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset @@ -294,6 +296,18 @@ SELECT (:apply_5::int = 3) AS apply_5_ok \gset SELECT (:fail::int + 1) AS fail \gset \endif +-- A denial is permanent, so the cursor must still move past those rows: holding it +-- back re-delivers them on every check forever, and in a chunked batch the final +-- chunk would checkpoint past them anyway, dropping them with no report. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after_denied \gset +SELECT (:ckpt_after_denied::bigint > :ckpt_before_denied::bigint) AS ckpt_ok \gset +\if :ckpt_ok +\echo [PASS] (:testid) RLS auth: denied apply still advanced the receive checkpoint +\else +\echo [FAIL] (:testid) RLS auth: denied apply left the checkpoint at :ckpt_after_denied (expected > :ckpt_before_denied) +SELECT (:fail::int + 1) AS fail \gset +\endif + -- Verify doc4 does NOT exist (superuser check) SELECT COUNT(*) AS doc4_count FROM documents WHERE id = 'doc4' \gset SELECT (:doc4_count::int = 0) AS test5_ok \gset From 6bf127b662b7faf36800200fcaf6f0a2ba1164a1 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 15:02:50 -0600 Subject: [PATCH 06/28] fix(block): report which table and column a block write failed on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading the row back is part of writing a block column: without its text there is nothing to split, and a row its own session cannot select could not sync anyway. So the failure stays fatal — but it has to be legible. Five paths across local_block_insert and block_migrate_existing_rows returned a bare code with no message. databasevm_step clears the error text on entry, so PG's cloudsync_insert raised the user's INSERT with an empty errmsg — the "not an error" confusion the comment in cloudsync_payload_apply already warns about. Two of them returned the raw negative from pk_decode_prikey, which is not a DBRES value at all (-1 is neither OK nor any known error), so callers testing for a known code fell through. Each now names the table and the column, and the unreadable-row case points at the SELECT policy. Aborting the migration stays recoverable: its Phase 1 scan skips already-migrated rows, so a re-run resumes. Test 57 covers the unreadable case end to end. Verified it fails ("reported a blank error") with the message removed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- CHANGELOG.md | 1 + src/cloudsync.c | 60 ++++++++++++++++++++---- test/postgresql/57_audit_regressions.sql | 35 ++++++++++++++ 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39112898..23af7b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Primary-key doubles keep their deployed little-endian IEEE754 byte order on every architecture.** The previous code combined host conversion with manual big-endian serialization; the historical format is now explicit. Integer keys are unchanged, and no migration is needed for little-endian deployments. - **PostgreSQL no longer frees a tuple table belonging to another open cursor.** A block write that failed while a second SPI cursor was active could release rows still in use; tuple tables are now owned per statement. - **Block-level LWW text writes roll back cleanly when a block write fails**, instead of leaving the row partially written. +- **Block-column failures now say which table and column failed**, instead of aborting the statement with a blank message. Reading the row back is part of writing a block column, so a row the session cannot `SELECT` — a row-level security policy narrower for reads than for writes, say — is reported with that cause rather than as an empty "not an error". - **64-bit clock values above `UINT32_MAX` are handled correctly on incoming changes** — column and database versions, causal length, and sequence. - **The Node package rejects `ia32` on every operating system** rather than selecting an incompatible binary. `x64` and `arm64-musl` selection is unchanged. diff --git a/src/cloudsync.c b/src/cloudsync.c index c0c5b577..2be02e5d 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -2202,21 +2202,44 @@ static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table } // Reuse the checked block writer; the scan above excludes migrated rows. + // As in local_block_insert, every failure carries a message: the migration aborts + // on one, and a bare code would surface as a blank error. Aborting is recoverable + // — the Phase 1 scan skips already-migrated rows, so a re-run resumes. + char errmsg[512]; dbvm_t *val_vm = table_column_lookup(table, col_name, false, NULL); - rc = val_vm ? DBRES_OK : DBRES_MISUSE; + if (!val_vm) { + snprintf(errmsg, sizeof(errmsg), "Missing value statement for block column \"%s\" of table \"%s\"", col_name, table->name); + rc = cloudsync_set_error(data, errmsg, DBRES_MISUSE); + } else { + rc = DBRES_OK; + } for (int p = 0; p < pk_count && rc == DBRES_OK; p++) { rc = pk_decode_prikey(pks[p], pklens[p], pk_decode_bind_callback, val_vm); - if (rc >= 0) rc = databasevm_step(val_vm); + if (rc < 0) { + snprintf(errmsg, sizeof(errmsg), "Unable to decode the primary key of a row in \"%s\" while migrating block column \"%s\"", table->name, col_name); + rc = cloudsync_set_error(data, errmsg, DBRES_ERROR); + databasevm_reset(val_vm); + break; + } + rc = databasevm_step(val_vm); if (rc == DBRES_ROW) { const char *text = database_column_text(val_vm, 0); bool has_text = text != NULL; char *copy = text ? cloudsync_string_dup(text) : NULL; databasevm_reset(val_vm); - rc = has_text && !copy ? DBRES_NOMEM : DBRES_OK; + if (has_text && !copy) { + snprintf(errmsg, sizeof(errmsg), "Not enough memory to migrate block column \"%s\" of table \"%s\"", col_name, table->name); + rc = cloudsync_set_error(data, errmsg, DBRES_NOMEM); + } else { + rc = DBRES_OK; + } if (rc == DBRES_OK && copy) rc = local_block_update(data, table, pks[p], pklens[p], col_idx, copy, db_version, true); cloudsync_memory_free(copy); - } else if (rc == DBRES_DONE) rc = DBRES_ERROR; + } else if (rc == DBRES_DONE) { + snprintf(errmsg, sizeof(errmsg), "Unable to read block column \"%s\" of table \"%s\" while migrating: a tracked row is not visible to this connection (check the table's row-level security SELECT policy)", col_name, table->name); + rc = cloudsync_set_error(data, errmsg, DBRES_ERROR); + } databasevm_reset(val_vm); } for (int i = 0; i < pk_count; i++) cloudsync_memory_free(pks[i]); @@ -4376,17 +4399,38 @@ int local_block_update(cloudsync_context *data, cloudsync_table_context *table, int local_block_insert(cloudsync_context *data, cloudsync_table_context *table, const void *pk, size_t pklen, int column, int64_t version) { - dbvm_t *vm = table_column_lookup(table, table_colname(table, column), false, NULL); + const char *colname = table_colname(table, column); + dbvm_t *vm = table_column_lookup(table, colname, false, NULL); if (!vm) return cloudsync_set_error(data, "Missing block column statement", DBRES_MISUSE); + + // Every failure below must carry a message: databasevm_step clears the error on + // entry, so a bare code reaches the caller as a blank "not an error". + char errmsg[512]; int rc = pk_decode_prikey((char *)pk, pklen, pk_decode_bind_callback, vm); - if (rc >= 0) rc = databasevm_step(vm); + if (rc < 0) { + databasevm_reset(vm); + snprintf(errmsg, sizeof(errmsg), "Unable to decode the primary key of a row in \"%s\" while writing block column \"%s\"", table->name, colname); + return cloudsync_set_error(data, errmsg, DBRES_ERROR); + } + + rc = databasevm_step(vm); char *copy = NULL; if (rc == DBRES_ROW) { const char *text = database_column_text(vm, 0); copy = cloudsync_string_dup(text ? text : ""); - rc = copy ? DBRES_OK : DBRES_NOMEM; + if (copy) rc = DBRES_OK; + else { + snprintf(errmsg, sizeof(errmsg), "Not enough memory to read block column \"%s\" of table \"%s\"", colname, table->name); + rc = cloudsync_set_error(data, errmsg, DBRES_NOMEM); + } + } + else if (rc == DBRES_DONE) { + // Reading the row back is part of the block write: without its text there is + // nothing to split. A row that its own session cannot select cannot sync at + // all, so report it here rather than leave the column silently untracked. + snprintf(errmsg, sizeof(errmsg), "Unable to read back block column \"%s\" of table \"%s\": the row just written is not visible to this connection (check the table's row-level security SELECT policy)", colname, table->name); + rc = cloudsync_set_error(data, errmsg, DBRES_ERROR); } - else if (rc == DBRES_DONE) rc = DBRES_ERROR; // End the read cursor before writes that can invoke nested triggers/SPI errors. databasevm_reset(vm); if (rc == DBRES_OK) rc = local_block_update(data, table, pk, pklen, column, copy, version, true); diff --git a/test/postgresql/57_audit_regressions.sql b/test/postgresql/57_audit_regressions.sql index 865f6ea8..93c4f3f1 100644 --- a/test/postgresql/57_audit_regressions.sql +++ b/test/postgresql/57_audit_regressions.sql @@ -72,6 +72,41 @@ BEGIN END IF; END $$; \echo [PASS] (57-audit) block insert/update failures roll back base rows and metadata + +DROP TRIGGER deny_block ON t_cloudsync_blocks; + +-- Reading the row back is part of the block write, so a row the session cannot +-- select is an error — but it must be a legible one. A bare code would reach the +-- caller blank, because databasevm_step clears the error text on entry. +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'audit_block_user') THEN + CREATE ROLE audit_block_user LOGIN; + END IF; +END $$; +GRANT USAGE ON SCHEMA public TO audit_block_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO audit_block_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO audit_block_user; +ALTER TABLE t ENABLE ROW LEVEL SECURITY; +CREATE POLICY t_ins ON t FOR INSERT WITH CHECK (true); +CREATE POLICY t_sel ON t FOR SELECT USING (false); +DO $$ +DECLARE msg TEXT := ''; failed BOOLEAN := false; +BEGIN + SET LOCAL ROLE audit_block_user; + BEGIN INSERT INTO t VALUES('invisible','text'); + EXCEPTION WHEN OTHERS THEN failed := true; msg := SQLERRM; END; + RESET ROLE; + IF NOT failed THEN RAISE EXCEPTION 'Unreadable block row did not report an error'; END IF; + IF coalesce(btrim(msg), '') = '' THEN RAISE EXCEPTION 'Unreadable block row reported a blank error'; END IF; + IF msg NOT LIKE '%not visible to this connection%' OR msg NOT LIKE '%value%' THEN + RAISE EXCEPTION 'Unreadable block row reported an unhelpful error: %', msg; + END IF; +END $$; +DROP POLICY t_sel ON t; +DROP POLICY t_ins ON t; +ALTER TABLE t DISABLE ROW LEVEL SECURITY; +\echo [PASS] (57-audit) an unreadable block row reports which table and column, not a blank error + \connect postgres DROP DATABASE cloudsync_audit_source; DROP DATABASE cloudsync_audit_target; From 7077df7fd298bda46e04813ec417b6baec2f9300 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 15:16:20 -0600 Subject: [PATCH 07/28] fix(network): bound artifact transfers on progress, not elapsed time CURLOPT_TIMEOUT was applied to both pooled handles, but they carry very different traffic. An S3 presigned URL is not one of the API endpoints, so artifact GETs and PUTs used the artifact handle and inherited the 300-second cap: 256 MiB (the new decompressed limit) inside 300s demands a sustained ~875 KB/s, so a healthy transfer on a slow link was killed mid-flight and reported as a timeout, indistinguishable from a dead server, on every retry. API calls keep the elapsed-time cap, which is the right shape for small JSON. Artifact transfers now abort after 60 seconds below 1 KB/s, which also catches a real stall five times sooner than the 300s cap did, and keeps a 1-hour backstop because there is no progress callback to cancel a transfer that trickles just fast enough to stay alive. The stalled-server test covered the artifact handle only (every endpoint in its stub context is NULL). It now runs both policies. Verified the artifact case fails when the low-speed options are removed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- CHANGELOG.md | 2 +- Makefile | 2 +- src/network/network.c | 37 ++++++++++++++++++++++++++--------- src/network/network_private.h | 13 ++++++++++++ test/network_unit.c | 9 ++++++--- 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23af7b8c..ccca42a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. -- **Network requests now have deadlines** — 30 seconds to connect and 300 seconds in total, applied to pooled handles as well as new ones. A stalled server previously left a sync call waiting indefinitely. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. +- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. ### Fixed diff --git a/Makefile b/Makefile index d22a2314..0860d307 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,7 @@ TEST_TARGET = $(patsubst %.c,$(DIST_DIR)/%$(EXE), $(notdir $(TEST_SRC))) # -dynamiclib on macOS) so it links as an executable, plus the test link libs. # The deadline regression uses a loopback socket; no external service is needed. BUILD_NETTEST = build/nettest -NT_CFLAGS = $(filter-out -DCLOUDSYNC_OMIT_NETWORK,$(T_CFLAGS)) -DCLOUDSYNC_REQUEST_TIMEOUT_SECONDS=1L -DCLOUDSYNC_CONNECT_TIMEOUT_SECONDS=1L +NT_CFLAGS = $(filter-out -DCLOUDSYNC_OMIT_NETWORK,$(T_CFLAGS)) -DCLOUDSYNC_REQUEST_TIMEOUT_SECONDS=1L -DCLOUDSYNC_CONNECT_TIMEOUT_SECONDS=1L -DCLOUDSYNC_ARTIFACT_LOW_SPEED_TIME=1L -DCLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS=30L NT_LDFLAGS = $(filter-out -shared -dynamiclib -headerpad_max_install_names,$(LDFLAGS)) $(T_LDFLAGS) NT_SRC = $(SRC_FILES) $(SQLITE_DIR)/sqlite3.c $(TEST_DIR)/network_unit.c NT_OBJ = $(patsubst %.c,$(BUILD_NETTEST)/%.o,$(notdir $(NT_SRC))) diff --git a/src/network/network.c b/src/network/network.c index 7022ed2f..372c4e15 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -371,27 +371,40 @@ static bool network_curl_pool_enabled(network_data *data) { return data->curl_pool_enabled > 0; } +// API calls carry small JSON, so a cap on elapsed time is the right shape for them. +// Artifact transfers are bulk and are bounded on progress instead: 256 MiB inside a +// 300s cap would demand a sustained ~875 KB/s, killing a healthy transfer on a slow +// link. Low-speed also detects a genuine stall sooner than the absolute cap does. +static void network_curl_apply_deadlines(CURL *handle, bool is_api) { + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); + curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); + if (is_api) { + curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); + return; + } + curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS); + curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT); + curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME); +} + static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, bool *pooled) { if (pooled) *pooled = false; + bool is_api = network_endpoint_is_api(data, endpoint); if (!network_curl_pool_enabled(data)) { CURL *handle = curl_easy_init(); if (!handle) return NULL; - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); - curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); - curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); + network_curl_apply_deadlines(handle, is_api); return handle; } - CURL **slot = network_endpoint_is_api(data, endpoint) ? &data->api_curl : &data->artifact_curl; + CURL **slot = is_api ? &data->api_curl : &data->artifact_curl; if (!*slot) { *slot = curl_easy_init(); } else { curl_easy_reset(*slot); } if (!*slot) return NULL; - curl_easy_setopt(*slot, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); - curl_easy_setopt(*slot, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); - curl_easy_setopt(*slot, CURLOPT_NOSIGNAL, 1L); + network_curl_apply_deadlines(*slot, is_api); curl_easy_setopt(*slot, CURLOPT_MAXCONNECTS, CLOUDSYNC_CURL_MAXCONNECTS); curl_easy_setopt(*slot, CURLOPT_MAXAGE_CONN, CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS); @@ -401,9 +414,12 @@ static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, } #if defined(CLOUDSYNC_UNITTEST) && !defined(CLOUDSYNC_OMIT_CURL) -bool network_test_curl_timeout(const char *url, bool use_pool) { +bool network_test_curl_timeout(const char *url, bool use_pool, bool as_api) { network_data data = {0}; data.curl_pool_enabled = use_pool ? 1 : -1; + // Classifying the url as the check endpoint selects the API deadline policy; + // leaving every endpoint NULL selects the artifact one. + if (as_api) data.check_endpoint = (char *)url; bool ok = true; // The second pooled call exercises curl_easy_reset as well as initialization. for (int i = 0; i < 2; i++) { @@ -415,7 +431,10 @@ bool network_test_curl_timeout(const char *url, bool use_pool) { CURLcode rc = curl_easy_perform(handle); double seconds = 0; curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &seconds); - ok = ok && rc == CURLE_OPERATION_TIMEDOUT && seconds < CLOUDSYNC_REQUEST_TIMEOUT_SECONDS + 2; + // curl reports a low-speed abort as CURLE_OPERATION_TIMEDOUT as well, so only + // the budget differs between the two policies. + long budget = as_api ? CLOUDSYNC_REQUEST_TIMEOUT_SECONDS : CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME; + ok = ok && rc == CURLE_OPERATION_TIMEDOUT && seconds < budget + 2; if (!pooled) curl_easy_cleanup(handle); } if (data.api_curl) curl_easy_cleanup(data.api_curl); diff --git a/src/network/network_private.h b/src/network/network_private.h index 113aab86..bd4fc92e 100644 --- a/src/network/network_private.h +++ b/src/network/network_private.h @@ -18,6 +18,19 @@ #ifndef CLOUDSYNC_REQUEST_TIMEOUT_SECONDS #define CLOUDSYNC_REQUEST_TIMEOUT_SECONDS 300L #endif +// Artifact transfers are bulk, so they are bounded by lack of progress rather than +// by elapsed time: a large payload on a slow link would otherwise be killed +// mid-flight. The absolute value is only a backstop against a transfer that +// trickles just fast enough to stay alive, since nothing can cancel one in flight. +#ifndef CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS +#define CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS 3600L +#endif +#ifndef CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT +#define CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT 1024L +#endif +#ifndef CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME +#define CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME 60L +#endif #define CLOUDSYNC_ENDPOINT_PREFIX "v2/cloudsync/databases" #define CLOUDSYNC_ENDPOINT_UPLOAD "upload" #define CLOUDSYNC_ENDPOINT_CHECK "check" diff --git a/test/network_unit.c b/test/network_unit.c index ce0911f4..d1e60fa8 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -215,7 +215,7 @@ static bool test_unicode(void) { #include #include #include -extern bool network_test_curl_timeout(const char *, bool); +extern bool network_test_curl_timeout(const char *, bool, bool); static bool test_stalled_http_timeout(void) { int fd = socket(AF_INET, SOCK_STREAM, 0); if (fd < 0) return false; @@ -228,7 +228,10 @@ static bool test_stalled_http_timeout(void) { char url[80]; snprintf(url, sizeof(url), "http://127.0.0.1:%u/", ntohs(address.sin_port)); // A listening socket that never sends HTTP simulates a stalled server. - if (ok) ok = network_test_curl_timeout(url, false) && network_test_curl_timeout(url, true); + // An API endpoint is bounded by elapsed time; an artifact transfer by a stall. + // Both shapes must abort against a server that accepts and then sends nothing. + if (ok) ok = network_test_curl_timeout(url, false, true) && network_test_curl_timeout(url, true, true); + if (ok) ok = network_test_curl_timeout(url, false, false) && network_test_curl_timeout(url, true, false); close(fd); return ok; } @@ -236,7 +239,7 @@ static bool test_stalled_http_timeout(void) { int main(void) { #if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) - check("HTTP deadlines for new and reset pooled handles:", test_stalled_http_timeout()); + check("HTTP deadlines: API elapsed cap and artifact stall cap:", test_stalled_http_timeout()); #endif check("JSON keys only match root object members:", test_json_scope()); check("Gateway data envelope is unwrapped before scoped lookups:", test_json_envelope()); From b4a5b3f107702476f2117ee566e020e9a385c94f Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 15:17:57 -0600 Subject: [PATCH 08/28] fix(merge): keep the real error when the flush savepoint fails to commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit error_message was snapshotted only on the way into cleanup, so arriving with rc OK left it empty. A database_commit_savepoint that then failed set rc but its message — a deadlock or serialization failure, say — was replaced by the generic "Unable to flush pending changes". Snapshot after the commit attempt as well, before the rollback, which touches the error state itself. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- src/cloudsync.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cloudsync.c b/src/cloudsync.c index 2be02e5d..6f64bbb5 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -1542,7 +1542,14 @@ static int merge_flush_pending (cloudsync_context *data) { if (rc != DBRES_OK) snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); merge_pending_free_entries(batch); if (flush_savepoint) { - if (rc == DBRES_OK) rc = database_commit_savepoint(data, "merge_flush"); + if (rc == DBRES_OK) { + rc = database_commit_savepoint(data, "merge_flush"); + // Snapshot here too: arriving with rc OK leaves error_message empty, and a + // commit that fails (a deadlock or serialization failure, say) sets a real + // message that the generic fallback below would otherwise replace. The + // rollback runs after, and touches the error state itself. + if (rc != DBRES_OK) snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); + } if (rc != DBRES_OK) database_rollback_savepoint(data, "merge_flush"); } if (rc != DBRES_OK) cloudsync_set_error(data, error_message[0] ? error_message : "Unable to flush pending changes", rc); From 067f3fbce9c9fa39fec5fec7b8f95be24f018d85 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 15:39:05 -0600 Subject: [PATCH 09/28] fix(apply): apply the denial policy to fragmented values, and report rows written MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three consequences of the denial work, which only covered the row path. The v3 fragment path had no denial branch, so a denied oversize value still returned POLICY_DENIED up through network_apply_payload_buffer, became a receive error, and aborted the whole drain — skipping the rest of the payload, stalling the cursor, and leaving the staged fragments undeleted to churn until stale cleanup. It gets the same treatment as the row path: count, skip, checkpoint. A denied value's fragments are as finished as an applied one's, so they are dropped too; any other failure still keeps them for the retry. receive.rows counted denied entries, because it came from the payload's entry count. An all-denied receive reported {"rows":N,"denied":N} while tables was correctly empty, and the diagnostic the CHANGELOG describes — a non-zero denied with a zero rows — could never occur. API.md has always documented the field as rows "received and applied", so the number now matches its own contract. Fixed in the drain rather than in the apply return value: that return counts payload entries including denied ones, which tests 27 and 29 asserts as part of the SQL surface. Both paths accumulate an accurate applied count on the context instead, next to the denied one. API.md documents denied in both receive shapes and all six samples. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- API.md | 20 +++++++++++--------- CHANGELOG.md | 2 +- src/cloudsync.c | 37 ++++++++++++++++++++++++++++++------- src/cloudsync.h | 11 +++++++---- src/network/network.c | 18 +++++++++++------- 5 files changed, 60 insertions(+), 28 deletions(-) diff --git a/API.md b/API.md index 9c3419f2..cb77655c 100644 --- a/API.md +++ b/API.md @@ -813,10 +813,11 @@ If the network is misconfigured or the remote server is unreachable, the functio **Returns:** A JSON string with the receive result: ```json -{"receive": {"rows": N, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} +{"receive": {"rows": N, "denied": D, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} ``` - `receive.rows`: The total number of rows received and applied to the local database, summed across all chunks drained this call. `0` when the receive phase failed, when nothing was available, or when only intermediate fragments were staged without completing a value. +- `receive.denied`: The number of entries a row-level security policy rejected, summed across all chunks drained this call. Denied entries are skipped and the receive cursor still advances past them: the rejection is permanent, so holding the cursor back would re-deliver the same entries on every call. They are not counted in `receive.rows`, so a non-zero `denied` alongside a `rows` of `0` means nothing was written — the shape of an apply connection whose session identity is not set. - `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. - `receive.chunks`: The number of payload chunks applied by this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a drained `N`-chunk stream (bounded by `max_chunks` if given). - `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; transport-independent, not the compressed wire size). Useful for byte-budgeted draining together with `max_chunks`. @@ -828,16 +829,16 @@ If the network is misconfigured or the remote server is unreachable, the functio ```sql SELECT cloudsync_network_receive_changes(); --- '{"receive":{"rows":3,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' +-- '{"receive":{"rows":3,"denied":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Capped drain with more pending (call again to continue): --- '{"receive":{"rows":40,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' +-- '{"receive":{"rows":40,"denied":0,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' -- With a client-side apply error: --- '{"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' +-- '{"receive":{"rows":0,"denied":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' -- With a server-reported check-job failure: --- '{"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' +-- '{"receive":{"rows":0,"denied":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' ``` --- @@ -867,7 +868,7 @@ When the server delivers changes as a stream of chunks, this function drains the ```json { "send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "chunks": C, "bytes": B, "lastFailure": {...}}, - "receive": {"rows": N, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} + "receive": {"rows": N, "denied": D, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} } ``` @@ -877,6 +878,7 @@ When the server delivers changes as a stream of chunks, this function drains the - `send.chunks` / `send.bytes`: Number of payload chunks sent and total serialized payload bytes sent during the send phase. Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes). - `send.lastFailure` (optional): Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) — forwarded verbatim from the server's `failures.apply` whenever a failed apply job is reported, regardless of `status`. - `receive.rows`: The **total** number of rows received and applied during the receive phase, summed across **all** chunks drained in this call. `0` when the receive phase failed. +- `receive.denied`: The **total** number of entries rejected by a row-level security policy across **all** chunks drained in this call. Skipped rather than retried, and not counted in `receive.rows` — see [Receive Changes](#receive-changes). - `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. - `receive.chunks`: The number of payload chunks applied in this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a fully drained `N`-chunk stream. `cloudsync_network_sync()` always drains the whole stream (it does not cap chunks). - `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; not the compressed wire size). Same semantics as in [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks). @@ -889,15 +891,15 @@ When the server delivers changes as a stream of chunks, this function drains the ```sql -- Perform a single synchronization cycle SELECT cloudsync_network_sync(); --- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"denied":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Perform a synchronization cycle with custom retry settings SELECT cloudsync_network_sync(500, 3); -- A large download drained as a multi-chunk stream in a single call: --- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' +-- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"denied":0,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' -- Receive phase failed but send phase completed — the error is surfaced in JSON, not as a SQL error: --- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"denied":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' ``` --- diff --git a/CHANGELOG.md b/CHANGELOG.md index ccca42a5..7f814bd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. +- **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. They are not counted in `receive.rows`, which reports what was actually written. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. - **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. diff --git a/src/cloudsync.c b/src/cloudsync.c index 6f64bbb5..f7c4d70e 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -193,9 +193,12 @@ struct cloudsync_context { int64_t apply_last_db_version; int64_t apply_last_seq; - // payload entries rejected by a row-level security policy, accumulated across - // a receive drain so a denial in one chunk is still visible when a later chunk - // reports. Reset with cloudsync_apply_denied_reset. + // Entries applied, and entries rejected by a row-level security policy, both + // accumulated across a receive drain so a denial in one chunk is still visible + // when a later chunk reports. Reset with cloudsync_apply_stats_reset. Kept here + // rather than derived from the apply return value, which reports payload entries + // (denied ones included) and is a tested part of the SQL surface. + int apply_rows; int apply_denied; }; @@ -622,8 +625,12 @@ const char *cloudsync_errmsg (cloudsync_context *data) { return data->errmsg; } -void cloudsync_apply_denied_reset (cloudsync_context *data) { - if (data) data->apply_denied = 0; +void cloudsync_apply_stats_reset (cloudsync_context *data) { + if (data) { data->apply_rows = 0; data->apply_denied = 0; } +} + +int cloudsync_apply_rows_count (cloudsync_context *data) { + return (data) ? data->apply_rows : 0; } int cloudsync_apply_denied_count (cloudsync_context *data) { @@ -3914,7 +3921,11 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data rc = cloudsync_payload_apply_single_decoded_row(data, tbl, tbl_len, pk, pk_len, col_name, col_name_len, value, (size_t)total_size, col_version, db_version, site_id, site_id_len, cl, seq, pnrows); - if (rc != DBRES_OK) goto cleanup; + // A denied value is permanently not ours to hold, so its staged fragments are as + // finished as an applied one's: drop them here rather than leave them churning + // until stale cleanup. Any other failure keeps them for the retry. + int apply_rc = rc; + if (rc != DBRES_OK && rc != DBRES_POLICY_DENIED) goto cleanup; rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_DELETE, &vm, 0); if (rc == DBRES_OK) { @@ -3922,6 +3933,7 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data int step_rc = databasevm_step(vm); if (step_rc == DBRES_DONE) rc = DBRES_OK; } + if (rc == DBRES_OK) rc = apply_rc; cleanup: if (vm) databasevm_finalize(vm); @@ -4115,6 +4127,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b if (header.version == CLOUDSYNC_PAYLOAD_VERSION_3) { int rc = DBRES_OK; int applied_rows = 0; + int denied_entries = 0; if (header.ncols != CLOUDSYNC_CHANGES_NCOLS) { if (clone) cloudsync_memory_free(clone); return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 column count", DBRES_MISUSE); @@ -4130,12 +4143,18 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b } int n = 0; rc = cloudsync_payload_apply_fragment_row(data, &row, &n); - if (rc != DBRES_OK) break; + // Same policy as the row path below: a denial is permanent, so skip it, + // count it, and let the cursor advance. Failing here would abort the whole + // drain and re-deliver the same value on every retry. + if (rc == DBRES_POLICY_DENIED) { denied_entries++; rc = DBRES_OK; } + else if (rc != DBRES_OK) break; applied_rows += n; buffer += seek; buf_len -= seek; } if (clone) cloudsync_memory_free(clone); + data->apply_denied += denied_entries; + if (rc == DBRES_OK) data->apply_rows += applied_rows; if (pnrows) *pnrows = applied_rows; // Advance the receive cursor only after the whole payload is applied, // gated on the caller-supplied checkpoint (a non-final chunk passes @@ -4285,6 +4304,10 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b if (rc == DBRES_DONE) rc = DBRES_OK; data->apply_denied += denied_entries; + if (rc == DBRES_OK) { + int applied = (int)nrows - denied_entries; + data->apply_rows += (applied > 0) ? applied : 0; + } // A policy denial is permanent: those rows are not this site's to hold, so the // cursor must still advance. Holding it back would re-deliver the same rows on diff --git a/src/cloudsync.h b/src/cloudsync.h index 2c4bef5b..b9ee1a7d 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -111,10 +111,13 @@ const char *cloudsync_errmsg (cloudsync_context *data); int cloudsync_errcode (cloudsync_context *data); void cloudsync_reset_error (cloudsync_context *data); -// Payload entries rejected by a row-level security policy. The count accumulates -// across a receive drain (reset once before it) so denials in an early chunk are -// still reported by the call that finishes the drain. -void cloudsync_apply_denied_reset (cloudsync_context *data); +// Entries applied, and entries rejected by a row-level security policy. Both counts +// accumulate across a receive drain (reset once before it) so denials in an early +// chunk are still reported by the call that finishes the drain. The applied count is +// tracked here rather than derived from the apply return value, which reports the +// payload's entry count (denied ones included) as part of the SQL surface. +void cloudsync_apply_stats_reset (cloudsync_context *data); +int cloudsync_apply_rows_count (cloudsync_context *data); int cloudsync_apply_denied_count (cloudsync_context *data); int cloudsync_commit_hook (void *ctx); void cloudsync_rollback_hook (void *ctx); diff --git a/src/network/network.c b/src/network/network.c index 372c4e15..30417699 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -2238,10 +2238,9 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, // Denials accumulate on the context across every chunk of this drain, so a // denial in an early chunk is still reported by the call that finishes it. - cloudsync_apply_denied_reset(data); + cloudsync_apply_stats_reset(data); int ntries = 0; // counts only "nothing ready" (202) polls - int nrows_total = 0; // cumulative rows applied across the whole drain int nchunks = 0; // payload chunks applied this call int64_t bytes_total = 0; // serialized payload bytes received this call bool complete = true; // false iff the stream is known to have more pending @@ -2265,14 +2264,13 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, request_max_chunks = safety_remaining; } - int nrows = 0; + int nrows = 0; // required out-param; the drain total comes from the context rc = cloudsync_network_check_internal(context, &nrows, sr, &receive_err, request_max_chunks); // a receive error (network or apply) won't fix itself across retries if (rc != SQLITE_OK) { complete = false; break; } if (sr->page_delivered) { - nrows_total += nrows; // a staged (incomplete) fragment contributes 0 - bytes_total += sr->bytes_received; + bytes_total += sr->bytes_received; // a staged (incomplete) fragment applies 0 rows nchunks += sr->chunks_received; complete = !sr->more_pending; // reflects whether the stream is finished if (!sr->more_pending) break; // final batch -> drained @@ -2301,11 +2299,17 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, } // Compute the affected-tables union once, over the whole drain window. - if (!receive_err && rc == SQLITE_OK && nrows_total > 0) { + // Report rows actually written, not payload entries: an all-denied receive would + // otherwise claim {"rows":N,"denied":N} while tables is correctly empty. The apply + // return value still counts payload entries, which is a tested part of the SQL + // surface, so the accurate count is accumulated on the context instead. + int applied_total = cloudsync_apply_rows_count(data); + + if (!receive_err && rc == SQLITE_OK && applied_total > 0) { sr->tables_json = network_get_affected_tables(db, drain_prev_dbv); } - dr->rows = nrows_total; + dr->rows = applied_total; dr->denied = cloudsync_apply_denied_count(data); dr->chunks = nchunks; dr->bytes = bytes_total; From 4a1cfd2b1a98f32ed396aa4cb1e9e90815fde850 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 15:58:01 -0600 Subject: [PATCH 10/28] fix(apply): revert the v3 denial skip, which left the transaction unusable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 067f3fb made the v3 fragment path skip a denied value and checkpoint past it, matching the row path. That is wrong on PostgreSQL: a denial leaves the transaction unusable, so the next statement — the checkpoint write — fails with "buffer pin is not owned by resource owner TopTransaction". The symptom is worse than the behaviour it replaced, which at least reported the denial cleanly. Neither a savepoint around the per-value apply nor dropping the staged-fragment delete recovers the state; both were tried and both still fail. The row path is safe only because merge_flush_pending rolls back its own savepoint around the write. So the v3 path goes back to failing on a denial, and the comment records why. The gap the revert leaves open is real and now covered: the cursor does not advance, so a denied oversize value is re-delivered on every drain. 58_v3_denied_checkpoint.sql builds a genuine fragmented payload, applies it under a WITH CHECK policy, and pins that behaviour, with a note to flip the assertion when the apply leaves a recoverable state. Closing it properly needs the fragment apply to roll back to a savepoint the way merge_flush_pending does, which is more than a follow-up to the reporting work. The CHANGELOG now scopes the skip-and-advance claim to the row path. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- CHANGELOG.md | 2 +- src/cloudsync.c | 50 +++++---- test/postgresql/58_v3_denied_checkpoint.sql | 117 ++++++++++++++++++++ test/postgresql/full_test.sql | 1 + 4 files changed, 149 insertions(+), 21 deletions(-) create mode 100644 test/postgresql/58_v3_denied_checkpoint.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f814bd1..c4426fec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. They are not counted in `receive.rows`, which reports what was actually written. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. +- **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. They are not counted in `receive.rows`, which reports what was actually written. This does not yet extend to a value large enough to be sent in fragments: a denial there is still reported as a receive error and the cursor does not advance. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. - **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. diff --git a/src/cloudsync.c b/src/cloudsync.c index f7c4d70e..68bdc66c 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -629,6 +629,13 @@ void cloudsync_apply_stats_reset (cloudsync_context *data) { if (data) { data->apply_rows = 0; data->apply_denied = 0; } } +// Saturating: only a receive drain resets these, so on the direct-SQL apply path they +// accumulate for the life of the connection and signed overflow would be undefined. +static void cloudsync_apply_stats_add (cloudsync_context *data, int rows, int denied) { + if (rows > 0) data->apply_rows = (data->apply_rows > INT_MAX - rows) ? INT_MAX : data->apply_rows + rows; + if (denied > 0) data->apply_denied = (data->apply_denied > INT_MAX - denied) ? INT_MAX : data->apply_denied + denied; +} + int cloudsync_apply_rows_count (cloudsync_context *data) { return (data) ? data->apply_rows : 0; } @@ -3921,19 +3928,23 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data rc = cloudsync_payload_apply_single_decoded_row(data, tbl, tbl_len, pk, pk_len, col_name, col_name_len, value, (size_t)total_size, col_version, db_version, site_id, site_id_len, cl, seq, pnrows); - // A denied value is permanently not ours to hold, so its staged fragments are as - // finished as an applied one's: drop them here rather than leave them churning - // until stale cleanup. Any other failure keeps them for the retry. - int apply_rc = rc; - if (rc != DBRES_OK && rc != DBRES_POLICY_DENIED) goto cleanup; + // A denial leaves the transaction unusable until the caller's savepoint rolls it + // back, so the staged fragments cannot be dropped here. They are bounded by the + // stale-fragment cleanup instead. + if (rc != DBRES_OK) goto cleanup; rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_DELETE, &vm, 0); if (rc == DBRES_OK) { databasevm_bind_text(vm, 1, value_id, -1); - int step_rc = databasevm_step(vm); - if (step_rc == DBRES_DONE) rc = DBRES_OK; + // A failed delete is deliberately tolerated rather than propagated: the value + // itself is already applied or permanently denied, so failing here would stall + // the cursor and re-deliver it, and a delete that fails once fails again on + // every retry. The leftover rows are bounded by the stale-fragment cleanup. + // (The former `if (step_rc == DBRES_DONE) rc = DBRES_OK;` only looked like a + // check: rc was already DBRES_OK from the prepare.) + databasevm_step(vm); } - if (rc == DBRES_OK) rc = apply_rc; + cleanup: if (vm) databasevm_finalize(vm); @@ -4127,7 +4138,6 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b if (header.version == CLOUDSYNC_PAYLOAD_VERSION_3) { int rc = DBRES_OK; int applied_rows = 0; - int denied_entries = 0; if (header.ncols != CLOUDSYNC_CHANGES_NCOLS) { if (clone) cloudsync_memory_free(clone); return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 column count", DBRES_MISUSE); @@ -4143,18 +4153,19 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b } int n = 0; rc = cloudsync_payload_apply_fragment_row(data, &row, &n); - // Same policy as the row path below: a denial is permanent, so skip it, - // count it, and let the cursor advance. Failing here would abort the whole - // drain and re-deliver the same value on every retry. - if (rc == DBRES_POLICY_DENIED) { denied_entries++; rc = DBRES_OK; } - else if (rc != DBRES_OK) break; + // A denial is NOT skipped here, unlike the row path. Continuing past one + // leaves PostgreSQL's transaction unusable — the next statement fails with + // "buffer pin is not owned by resource owner" — and neither a savepoint + // around this call nor dropping the staged-fragment delete recovers it. + // A denied oversize value is therefore still a hard receive error that + // stalls the cursor. See test 58_v3_denied_checkpoint.sql. + if (rc != DBRES_OK) break; applied_rows += n; buffer += seek; buf_len -= seek; } if (clone) cloudsync_memory_free(clone); - data->apply_denied += denied_entries; - if (rc == DBRES_OK) data->apply_rows += applied_rows; + cloudsync_apply_stats_add(data, (rc == DBRES_OK) ? applied_rows : 0, 0); if (pnrows) *pnrows = applied_rows; // Advance the receive cursor only after the whole payload is applied, // gated on the caller-supplied checkpoint (a non-final chunk passes @@ -4303,10 +4314,9 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b if (rc == DBRES_DONE) rc = DBRES_OK; - data->apply_denied += denied_entries; - if (rc == DBRES_OK) { - int applied = (int)nrows - denied_entries; - data->apply_rows += (applied > 0) ? applied : 0; + { + int applied = (rc == DBRES_OK) ? (int)nrows - denied_entries : 0; + cloudsync_apply_stats_add(data, (applied > 0) ? applied : 0, denied_entries); } // A policy denial is permanent: those rows are not this site's to hold, so the diff --git a/test/postgresql/58_v3_denied_checkpoint.sql b/test/postgresql/58_v3_denied_checkpoint.sql new file mode 100644 index 00000000..1e0d7d34 --- /dev/null +++ b/test/postgresql/58_v3_denied_checkpoint.sql @@ -0,0 +1,117 @@ +-- A denied v3 (fragmented) value is a hard error, NOT a skipped entry. +-- +-- The row path treats a row-level security denial as permanent and skippable: it is +-- counted, skipped, and the receive cursor advances past it. The v3 path cannot do +-- the same today. Continuing past a denial leaves PostgreSQL's transaction unusable, +-- and the next statement — the checkpoint write — fails with "buffer pin is not owned +-- by resource owner TopTransaction". Neither a savepoint around the per-value apply +-- nor skipping the staged-fragment delete recovers it. +-- +-- So this test pins the behaviour that actually holds: the denial surfaces as an +-- error and the cursor does not move. That is a known gap, not a desired outcome — +-- a denied oversize value is re-delivered on every drain. Closing it needs the v3 +-- apply to leave a recoverable transaction state. +-- +-- Test 27 covers the skip-and-advance guarantee for the v2 row path. + +\set testid '58-v3-denied' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_58_src; +DROP DATABASE IF EXISTS cloudsync_test_58_dst; +CREATE DATABASE cloudsync_test_58_src; +CREATE DATABASE cloudsync_test_58_dst; + +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'v3_denied_user') THEN + CREATE ROLE v3_denied_user LOGIN; + END IF; +END $$; + +-- Source: one oversized value, forced to fragment into several v3 chunks. +\connect cloudsync_test_58_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE frag_rls (id TEXT PRIMARY KEY NOT NULL, note TEXT DEFAULT ''); +SELECT cloudsync_init('frag_rls', 'CLS', 1) AS _init_src \gset +SELECT cloudsync_set('payload_max_chunk_size', '1'); -- clamps to the 256KB minimum +INSERT INTO frag_rls(id, note) +VALUES ('big', repeat('A', 262144) || repeat('B', 262144) || repeat('C', 131072)); + +SELECT count(*) FILTER (WHERE get_byte(payload, 4) = 3) AS v3_chunks +FROM cloudsync_payload_chunks() \gset +SELECT (:v3_chunks::int >= 2) AS fragmented_ok \gset +\if :fragmented_ok +\echo [PASS] (:testid) oversized value fragmented into :v3_chunks v3 chunks +\else +\echo [FAIL] (:testid) expected >=2 v3 fragments, got :v3_chunks (cannot exercise the v3 path) +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT string_agg(encode(payload, 'hex'), ',' ORDER BY chunk_index) AS chunks_hex +FROM cloudsync_payload_chunks() \gset + +-- Target: readable, but every insert is rejected by a WITH CHECK policy. +\connect cloudsync_test_58_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE frag_rls (id TEXT PRIMARY KEY NOT NULL, note TEXT DEFAULT ''); +SELECT cloudsync_init('frag_rls', 'CLS', 1) AS _init_dst \gset + +GRANT USAGE ON SCHEMA public TO v3_denied_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO v3_denied_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO v3_denied_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO v3_denied_user; +ALTER TABLE frag_rls ENABLE ROW LEVEL SECURITY; +CREATE POLICY frag_sel ON frag_rls FOR SELECT USING (true); +CREATE POLICY frag_ins ON frag_rls FOR INSERT WITH CHECK (false); + +CREATE TABLE chunk_transport(ord INT, payload BYTEA); +INSERT INTO chunk_transport(ord, payload) +SELECT ord, decode(hexval, 'hex') +FROM unnest(string_to_array(:'chunks_hex', ',')) WITH ORDINALITY AS t(hexval, ord); +GRANT ALL ON chunk_transport TO v3_denied_user; + +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset + +-- Apply every fragment as the restricted role, one top-level statement per chunk in +-- order. The non-final fragments stage; the final one reassembles and is denied. +-- ON_ERROR_STOP is disabled around it because the denial is expected to raise. +SET ROLE v3_denied_user; +\set ON_ERROR_STOP off +SELECT format('SELECT cloudsync_payload_apply(payload) FROM chunk_transport WHERE ord = %s;', ord) +FROM chunk_transport ORDER BY ord \gexec +\set ON_ERROR_STOP on +RESET ROLE; + +-- Reconnect for clean state after the expected denial. +\connect cloudsync_test_58_dst +\ir helper_psql_conn_setup.sql + +SELECT COUNT(*) AS applied_count FROM frag_rls WHERE id = 'big' \gset +SELECT (:applied_count::int = 0) AS denied_ok \gset +\if :denied_ok +\echo [PASS] (:testid) the fragmented value was denied by the WITH CHECK policy +\else +\echo [FAIL] (:testid) expected the value to be denied, found :applied_count rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Known gap: the cursor does not advance, so this value is re-delivered every drain. +-- Change this to expect an advance once the v3 apply leaves a recoverable state. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:ckpt_after::bigint = :ckpt_before::bigint) AS ckpt_pinned \gset +\if :ckpt_pinned +\echo [PASS] (:testid) known gap: a denied fragmented value leaves the checkpoint pinned at :ckpt_after +\else +\echo [FAIL] (:testid) checkpoint moved to :ckpt_after — the v3 denial gap may be closed; update this test +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_58_src; +DROP DATABASE IF EXISTS cloudsync_test_58_dst; +DROP ROLE IF EXISTS v3_denied_user; diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index da0d11e0..f4e9a97d 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -65,6 +65,7 @@ \ir 55_payload_chunks_positional_resume.sql \ir 56_many_columns.sql \ir 57_audit_regressions.sql +\ir 58_v3_denied_checkpoint.sql -- 'Test summary' \echo '\nTest summary:' From 5171b7713c7f53b13cc43424b5bce43cec0ed4c0 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 11 Sep 2026 15:58:24 -0600 Subject: [PATCH 11/28] docs: correct a comment about the fragment delete's reachability The denial path no longer reaches it, so "already applied or permanently denied" describes a state that cannot occur. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CCE6B54Qtf3UsaCVAJtVQF --- src/cloudsync.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cloudsync.c b/src/cloudsync.c index 68bdc66c..9a2d1724 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -3937,9 +3937,9 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data if (rc == DBRES_OK) { databasevm_bind_text(vm, 1, value_id, -1); // A failed delete is deliberately tolerated rather than propagated: the value - // itself is already applied or permanently denied, so failing here would stall - // the cursor and re-deliver it, and a delete that fails once fails again on - // every retry. The leftover rows are bounded by the stale-fragment cleanup. + // is already applied, so failing here would stall the cursor and re-deliver + // it, and a delete that fails once fails again on every retry. The leftover + // rows are bounded by the stale-fragment cleanup. // (The former `if (step_rc == DBRES_DONE) rc = DBRES_OK;` only looked like a // check: rc was already DBRES_OK from the prepare.) databasevm_step(vm); From 9cc60d21a1f9bff29f02003723f967920293219b Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:13:12 +0200 Subject: [PATCH 12/28] fix(apply): skip and report failed writes, keep group state atomic, preserve error origin Review follow-ups on the payload apply path (review points 1, 3, 4, 5, 7, 8, 9). - Failed writes: a change that fails on its data (constraint, raising trigger, type error) is skipped, logged as a warning and reported as receive.failed / receive.failedError, and the cursor advances. Transient failures (busy/locked, deadlock, serialization failure, cancel, out of memory/disk, I/O) and configuration failures (missing privilege, read-only database) fail the apply and keep the cursor. - PostgreSQL savepoints restore the caller's resource owner and memory context: SELECT cloudsync_payload_apply(data) FROM some_table no longer fails with a foreign buffer pin. Only the outermost savepoint swaps the active snapshot, so a rollback inside the caller's subtransaction no longer trips EnsurePortalSnapshotExists. - RLS denials: receive.denied is removed (denials never occur on the SQLite client); PostgreSQL raises one summary WARNING. Every payload row runs in its own savepoint and the cloudsync_changes trigger re-raises denials as 42501, so block-column and GOS denials are skipped instead of failing the apply. Denied rows are retried once the rest of the payload is in, so policies depending on later rows (memberships) apply. - Block columns and GOS tables write existing rows with an UPDATE (fallback to the upsert when no row changes), and a row's pending columns are flushed before its blocks, so they work under INSERT policies and NOT NULL columns. - Each PK group is applied under one savepoint covering the metadata its rows write, so a failed flush leaves nothing behind (a resurrected row is created when re-delivered) and skipped changes are counted per payload row, sentinel included. - Errors keep their origin: the SQLSTATE of the database error survives to the ereport (40001, 23505, ... instead of XX000); SQLite triggers report cloudsync's message and the real result code; block failures name the stage, column and table and are never blank. Tests: review_regressions (skip/transient/WAL busy, resurrected groups, block errors, allocation sweep), network_unit (receive JSON), PostgreSQL 39, 57 and new 59. Co-Authored-By: Claude Opus 5 --- API.md | 31 +- CHANGELOG.md | 11 +- docs/internal/audit-regressions.md | 50 +- src/cloudsync.c | 656 +++++++++++++++--- src/cloudsync.h | 22 +- src/database.h | 9 + src/network/network.c | 112 +-- src/postgresql/cloudsync_postgresql.c | 64 +- src/postgresql/database_postgresql.c | 141 +++- src/sqlite/cloudsync_sqlite.c | 21 +- src/sqlite/database_sqlite.c | 8 + test/network_unit.c | 17 + test/postgresql/39_concurrent_write_apply.sql | 12 + test/postgresql/57_audit_regressions.sql | 167 ++++- test/postgresql/59_rls_denial_retry.sql | 245 +++++++ test/postgresql/full_test.sql | 1 + test/review_regressions.c | 193 +++++- 17 files changed, 1513 insertions(+), 247 deletions(-) create mode 100644 test/postgresql/59_rls_denial_retry.sql diff --git a/API.md b/API.md index cb77655c..22150cfd 100644 --- a/API.md +++ b/API.md @@ -665,6 +665,14 @@ When a v3 fragment payload is received, CloudSync stores the fragment in an inte SELECT cloudsync_payload_apply(:payload); ``` +#### Failed writes + +A change whose write fails on its data — a constraint, a trigger that raises, a type error — fails the same way every time it is delivered, so it is skipped: the rest of the payload is applied and the receive checkpoint still advances. Each skipped change is logged as a warning (`sqlite3_log` on SQLite, a `WARNING` on PostgreSQL) and counted in `receive.failed` by the network functions. A transient or configuration failure — a locked or busy database, a deadlock or serialization failure, a cancel, running out of memory or disk, an I/O error, a missing privilege, a read-only database — is never skipped: the function fails and the checkpoint stays where it was, so the change is retried. + +#### Row-level security denials + +On PostgreSQL, a change rejected by a row-level security `WITH CHECK` policy is skipped as well, and the checkpoint advances. Because a policy can depend on rows that arrive later in the same payload — a membership row that grants access to the rows before it — denied changes are retried once the rest of the payload has been applied, repeating while a retry lets more changes through. A change still denied after that is skipped for good, and the function raises one `WARNING` with the number of changes skipped. A change that only a later payload would authorize is not retried. The return value still counts denied changes as payload rows. + --- ## Network Functions @@ -813,11 +821,12 @@ If the network is misconfigured or the remote server is unreachable, the functio **Returns:** A JSON string with the receive result: ```json -{"receive": {"rows": N, "denied": D, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} +{"receive": {"rows": N, "failed": F, "failedError": "...", "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} ``` - `receive.rows`: The total number of rows received and applied to the local database, summed across all chunks drained this call. `0` when the receive phase failed, when nothing was available, or when only intermediate fragments were staged without completing a value. -- `receive.denied`: The number of entries a row-level security policy rejected, summed across all chunks drained this call. Denied entries are skipped and the receive cursor still advances past them: the rejection is permanent, so holding the cursor back would re-deliver the same entries on every call. They are not counted in `receive.rows`, so a non-zero `denied` alongside a `rows` of `0` means nothing was written — the shape of an apply connection whose session identity is not set. +- `receive.failed`: The number of entries skipped because their write failed on the data itself — a constraint, a trigger that raised, a type error — summed across all chunks drained this call. Such a failure repeats identically on every retry, so the entry is skipped and the receive cursor still advances past it instead of stalling every later change; each skip is also logged as a warning. Not counted in `receive.rows`. Row-level security denials are not part of this count: they can only occur where the changes are applied under a PostgreSQL policy (see [Row-level security](#row-level-security-denials) below). A transient or configuration failure (a lock or busy database, a deadlock or serialization failure, a cancel, out of memory or disk, an I/O error, a missing privilege, a read-only database) is never skipped: it is reported in `receive.error` and the cursor stays in place so the next call retries it. +- `receive.failedError` (optional): The error message of the first entry counted in `receive.failed`, present only when `failed` is non-zero. - `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. - `receive.chunks`: The number of payload chunks applied by this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a drained `N`-chunk stream (bounded by `max_chunks` if given). - `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; transport-independent, not the compressed wire size). Useful for byte-budgeted draining together with `max_chunks`. @@ -829,16 +838,16 @@ If the network is misconfigured or the remote server is unreachable, the functio ```sql SELECT cloudsync_network_receive_changes(); --- '{"receive":{"rows":3,"denied":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' +-- '{"receive":{"rows":3,"failed":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Capped drain with more pending (call again to continue): --- '{"receive":{"rows":40,"denied":0,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' +-- '{"receive":{"rows":40,"failed":0,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' -- With a client-side apply error: --- '{"receive":{"rows":0,"denied":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' +-- '{"receive":{"rows":0,"failed":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' -- With a server-reported check-job failure: --- '{"receive":{"rows":0,"denied":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' +-- '{"receive":{"rows":0,"failed":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' ``` --- @@ -868,7 +877,7 @@ When the server delivers changes as a stream of chunks, this function drains the ```json { "send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "chunks": C, "bytes": B, "lastFailure": {...}}, - "receive": {"rows": N, "denied": D, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} + "receive": {"rows": N, "failed": F, "failedError": "...", "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} } ``` @@ -878,7 +887,7 @@ When the server delivers changes as a stream of chunks, this function drains the - `send.chunks` / `send.bytes`: Number of payload chunks sent and total serialized payload bytes sent during the send phase. Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes). - `send.lastFailure` (optional): Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) — forwarded verbatim from the server's `failures.apply` whenever a failed apply job is reported, regardless of `status`. - `receive.rows`: The **total** number of rows received and applied during the receive phase, summed across **all** chunks drained in this call. `0` when the receive phase failed. -- `receive.denied`: The **total** number of entries rejected by a row-level security policy across **all** chunks drained in this call. Skipped rather than retried, and not counted in `receive.rows` — see [Receive Changes](#receive-changes). +- `receive.failed` / `receive.failedError`: The **total** number of entries skipped because their write failed, and the first such error — see [Receive Changes](#receive-changes). - `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. - `receive.chunks`: The number of payload chunks applied in this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a fully drained `N`-chunk stream. `cloudsync_network_sync()` always drains the whole stream (it does not cap chunks). - `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; not the compressed wire size). Same semantics as in [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks). @@ -891,15 +900,15 @@ When the server delivers changes as a stream of chunks, this function drains the ```sql -- Perform a single synchronization cycle SELECT cloudsync_network_sync(); --- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"denied":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"failed":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Perform a synchronization cycle with custom retry settings SELECT cloudsync_network_sync(500, 3); -- A large download drained as a multi-chunk stream in a single call: --- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"denied":0,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' +-- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"failed":0,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' -- Receive phase failed but send phase completed — the error is surfaced in JSON, not as a SQL error: --- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"denied":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"failed":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' ``` --- diff --git a/CHANGELOG.md b/CHANGELOG.md index c4426fec..04440a5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **Rows rejected by a row-level security policy are now reported** as `receive.denied` in the JSON returned by `cloudsync_network_receive_changes()` and `cloudsync_network_sync()`, counted across every chunk of a receive. A denial is a permanent, expected outcome — those rows are not this site's to hold — so they are skipped and the receive cursor still advances past them; without a count, discarding them would be invisible. They are not counted in `receive.rows`, which reports what was actually written. This does not yet extend to a value large enough to be sent in fragments: a denial there is still reported as a receive error and the cursor does not advance. A non-zero `denied` with a zero `rows` is the shape of an apply connection whose session identity (`auth.uid()` / `app.current_user_id`) is not set. +- **PostgreSQL: rows rejected by a row-level security policy are retried within the payload, then skipped and reported.** A policy can depend on rows that arrive later in the same payload (a membership that grants access to the rows before it), so denied changes are retried once the rest of the payload is applied, repeating while a retry makes progress. A change still denied is skipped — it would be denied on every delivery — the receive cursor advances past it, and `cloudsync_payload_apply` raises one `WARNING` with the number of skipped changes. This now also covers block columns and grow-only-set tables, whose denial previously failed the whole apply. It does not yet extend to a value large enough to be sent in fragments: a denial there is still an error and the cursor does not advance, and a change authorized only by a later payload is not retried. - **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. ### Fixed -- **A failed payload write now reports the error and leaves the receive cursor where it was.** An error on one row could previously be overwritten by a later successful row, so `cloudsync_payload_apply` could report success after dropping changes and still advance the checkpoint — losing them silently. SQLite keeps its existing per-group partial-application behaviour. +- **A received change whose write fails is no longer dropped silently.** Previously the error was discarded, so the change was lost without a trace. A failure of the data itself (a constraint, a raising trigger, a type error) repeats on every retry, so the change is still skipped and the receive cursor still advances — holding it back would stall every later change forever — but it is now logged as a warning and reported as `receive.failed` / `receive.failedError`. A transient or configuration failure (lock or busy database, deadlock, serialization failure, cancel, out of memory or disk, I/O error, missing privilege, read-only database) now fails the apply and leaves the cursor in place, so the change is retried instead of lost. On PostgreSQL, a failure that cannot be rolled back on its own fails the whole apply. +- **PostgreSQL: applying a payload read from a table no longer fails.** `SELECT cloudsync_payload_apply(data) FROM some_table` failed with `buffer pin ... is not owned by resource owner TopTransaction` (and could abort an assertion-enabled server): the internal savepoints left the caller's resource owner and memory context switched. Both are now restored when a savepoint ends. +- **Block columns and grow-only-set tables merge under row-level security and NOT NULL constraints.** Both wrote a received column through an upsert holding only the primary key and that column: PostgreSQL checks an INSERT policy against that proposed row even when it becomes an update, and a NOT NULL column rejects it outright, so the column was never written. A block column's row now has its other received columns written first, and an existing row — of either kind — is updated in place. When the update finds no row to change (the row is gone, or an UPDATE policy hides it), the upsert is used as before, so the write is reported rather than recorded as applied without being stored. +- **A received row whose write fails leaves no partial state behind.** Applying a row wrote part of its metadata — for a deleted-and-recreated row, the new sentinel and the reset column clocks — before the row itself, outside the savepoint that protected the write. When the write then failed, the metadata claimed the row was there: delivered again, even after the cause was fixed (a lock that went away, a policy that now allows it), the row was silently never created, and an existing row kept zeroed clocks. Each primary key's changes are now applied under one savepoint that the failed write rolls back entirely. Skipped changes are also counted per payload entry, so a recreated row counts its sentinel too. +- **PostgreSQL: savepoints opened by cloudsync no longer disturb the caller's snapshot.** Ending a nested savepoint swapped the active snapshot inside an enclosing one, so rolling that back could leave the calling statement without its snapshot (an assertion failure on assert-enabled builds). +- **PostgreSQL: errors raised by cloudsync keep the SQLSTATE of the database error behind them.** A failure inside the tracking triggers or `cloudsync_payload_apply` — a serialization failure (`40001`), a deadlock (`40P01`), a unique violation (`23505`), a missing privilege (`42501`) — was re-raised as `XX000` internal_error, so application retry logic and error handling keyed on SQLSTATE could not react to it. cloudsync's own failures are still `XX000`. +- **SQLite: a failed tracking write in the insert, update and delete triggers keeps its result code and context.** It was reported as a bare `SQLITE_ERROR` with SQLite's last message; it now carries the real code (`SQLITE_BUSY`, `SQLITE_CONSTRAINT`, ...) and cloudsync's message. +- **Block column failures name the table and column** they were writing or reading, and what failed (reading, collecting or joining the blocks, writing the column), around the underlying database error. No block-column failure reports a blank message any more: allocation failures say so instead of posing as a read failure, and `cloudsync_text_materialize` keeps the error's result code on SQLite. - **Primary-key doubles keep their deployed little-endian IEEE754 byte order on every architecture.** The previous code combined host conversion with manual big-endian serialization; the historical format is now explicit. Integer keys are unchanged, and no migration is needed for little-endian deployments. - **PostgreSQL no longer frees a tuple table belonging to another open cursor.** A block write that failed while a second SPI cursor was active could release rows still in use; tuple tables are now owned per statement. - **Block-level LWW text writes roll back cleanly when a block write fails**, instead of leaving the row partially written. diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md index f90af4f1..11b94ef1 100644 --- a/docs/internal/audit-regressions.md +++ b/docs/internal/audit-regressions.md @@ -13,14 +13,42 @@ The previously reported `MAX_PARAMS` issue is outside this change. override `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE`; LZ4's `INT_MAX` bound still applies. Existing default chunk sizes are below this limit. Oversized legacy monolithic payloads must be rechunked or used with an explicitly raised limit. -- Curl requests now have a 30-second connection deadline and a 300-second total - deadline, including reused handles. Build overrides are - `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. -- Failed payload writes report an error and do not advance the receive cursor. - PostgreSQL's explicit RLS WITH CHECK rejection remains a skippable policy - outcome, distinct from generic SQL/permission errors; the call still reports - processed rows, but does not advance the cursor when a policy denied rows. - SQLite retains its existing per-group partial-application behavior. +- Payload writes that fail on their data (constraint, raising trigger, type + error) are skipped, logged as a warning and counted (`receive.failed`), and the + receive cursor advances: they fail identically on every retry. Transient + failures (busy/locked, deadlock, serialization failure, cancel, out of memory or + disk, I/O) and, on PostgreSQL, failures not contained by a savepoint fail the + apply and leave the cursor in place. PostgreSQL's RLS WITH CHECK rejection stays + a separate outcome: every row is applied in its own savepoint so a denial raised + inside the cloudsync_changes trigger (block columns, GOS tables) is contained too; + denied rows are retried after the rest of the payload while retries make progress, + then skipped with one summary WARNING. `receive.denied` was removed: denials never + occur on the SQLite client where the network functions run. +- Block materialization writes a row's pending columns first, and block and GOS + column writes update an existing row in place, so they pass INSERT policies and NOT + NULL constraints on other columns. An update that changes no row falls back to the + upsert, so a row hidden by an UPDATE policy is reported instead of silently skipped + (`databasevm_changes` exposes the affected row count on both backends). +- PostgreSQL savepoints restore the caller's resource owner and memory context, so + a payload applied from a table scan no longer trips a foreign buffer pin. +- Each PK group of a payload is applied under one savepoint (`cloudsync_merge_group`) + that also covers the metadata its rows write before the flush (sentinel, zeroed + clocks, block values, winner clocks); a failed flush rolls it all back, so a retried or + re-delivered row applies cleanly. The flush uses that savepoint instead of its own. + Skipped entries are counted from `merge_pending_batch.rows` (payload rows that joined + the batch, including an explicit sentinel, excluding one implied by a column row). +- Only the outermost savepoint opened by cloudsync swaps the active snapshot when it + ends; nested ones advance the command counter, keeping the caller's snapshot stack + balanced when an enclosing subtransaction rolls back. +- Every block materialization failure goes through one exit that names the stage, + column and table and composes the database error; allocation failures clear any + unrelated database error first so they are not misreported. +- Errors keep their origin. PostgreSQL records the SQLSTATE of every caught error on + the context (`cloudsync_sqlstate`), preserved across savepoint rollbacks, and every + `ereport` built from the context error uses it (internal_error only for cloudsync's + own failures). The SQLite tracking triggers report cloudsync's message and the real + result code. A missing privilege (`42501` outside a policy check, `SQLITE_PERM` / + `SQLITE_AUTH`) or a read-only database is not skipped by a payload apply. - Test database files live in a private per-run temporary directory, not HOME. The harness deletes only its own flat test files at shutdown. @@ -31,7 +59,11 @@ The previously reported `MAX_PARAMS` issue is outside this change. | 64-bit clocks | Incoming column/database versions, causal length and sequence above UINT32_MAX | | Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; forced big-endian conversion build | | Virtual-table planner | Unusable/unsupported constraints before an accepted constraint; no accepted constraints | -| Payload failures | First, middle and final PK errors; checkpoint unchanged; allocation limits | +| RLS denials | Block-column and GOS denials skipped; every column of a permitted block/GOS row written; a column hidden from UPDATE not recorded as applied; order-dependent denials applied on retry in both the batched and trigger paths; permanently denied rows skipped with checkpoint advanced | +| Block materialization errors | Write failure via cloudsync_text_materialize keeps cause, code/SQLSTATE and names stage, column, table; single-shot allocation failure at every allocation never yields a blank error | +| Error origin | SQLSTATE 40001/23505 through the block and metadata triggers; 40001 and a non-policy 42501 through payload apply, not skipped; SQLite trigger failure keeps SQLITE_CONSTRAINT and names column and table | +| Group atomicity | Resurrected row rejected (data and transient failure): no metadata left, re-delivery creates it, 3 entries counted; existing row keeps its clocks; RLS retry of a resurrected row; apply inside an aborted caller subtransaction | +| Payload failures | First, middle and final PK errors skipped with a warning and checkpoint advanced; locked database fails and keeps the checkpoint (SQLite rollback journal, WAL, PostgreSQL lock_timeout); allocation limits | | Metadata refill | Trigger rejects insertion of a missing column clock | | Block LWW | Insert/update rollback on block write failure; allocation failure at each split/list/diff allocation | | PostgreSQL ownership | Block failure while another SPI cursor is active; no invalid tuple-table cleanup | diff --git a/src/cloudsync.c b/src/cloudsync.c index 9a2d1724..3c63689d 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -119,6 +119,21 @@ typedef struct { bool cached_row_exists; int cached_col_count; const char **cached_col_names; // array of pointers into table_context (not owned) + + // Set by merge_flush_pending when a failure could not be contained by its savepoint + // (the savepoint could not be opened or rolled back), so the caller must not continue. + bool flush_uncontained; + + // Payload rows waiting in this batch (column rows and an explicit sentinel row), the + // count of changes lost if its flush fails. A sentinel implied by a column row is not + // a payload row of its own and is not counted. + int rows; + + // True while cloudsync_payload_apply holds a savepoint around the whole PK group: the + // flush then writes inside it instead of opening its own, so that a failed flush also + // rolls back what the group's rows already wrote (a resurrected row's sentinel and + // zeroed clocks, block values, winner clocks). + bool group_savepoint; } merge_pending_batch; // MARK: - @@ -143,6 +158,7 @@ struct cloudsync_context { void *db; char errmsg[1024]; int errcode; + int sqlstate; // SQLSTATE behind errmsg (PostgreSQL), 0 if none char *libversion; uint8_t site_id[UUID_LEN]; @@ -193,13 +209,15 @@ struct cloudsync_context { int64_t apply_last_db_version; int64_t apply_last_seq; - // Entries applied, and entries rejected by a row-level security policy, both - // accumulated across a receive drain so a denial in one chunk is still visible - // when a later chunk reports. Reset with cloudsync_apply_stats_reset. Kept here - // rather than derived from the apply return value, which reports payload entries - // (denied ones included) and is a tested part of the SQL surface. + // Entries applied, and entries skipped because their write failed, accumulated + // across a receive drain so an early chunk is still visible when a later chunk + // reports. Reset with cloudsync_apply_stats_reset. Kept here rather than derived + // from the apply return value, which reports payload entries (skipped ones + // included) and is a tested part of the SQL surface. Row-level security denials + // only occur on PostgreSQL, which has no receive drain: they are logged instead. int apply_rows; - int apply_denied; + int apply_failed; + char apply_failure[512]; // first skipped failure since the last reset ("" = none) }; struct cloudsync_table_context { @@ -210,6 +228,7 @@ struct cloudsync_table_context { char *base_ref; // schema-qualified base table name (e.g. "schema"."name") char **col_name; // array of column names dbvm_t **col_merge_stmt; // array of merge insert stmt (indexed by col_name) + dbvm_t **col_update_stmt; // lazily prepared UPDATE of one column by pk (block materialization) dbvm_t **col_value_stmt; // array of column value stmt (indexed by col_name) int *col_id; // array of column id col_algo_t *col_algo; // per-column algorithm (normal or block) @@ -603,7 +622,9 @@ int cloudsync_set_error (cloudsync_context *data, const char *err_user, int err_ char db_error_copy[sizeof(data->errmsg)]; int rc = database_errcode(data); if (rc == DBRES_OK) { + // no database error behind this one, so no SQLSTATE either snprintf(data->errmsg, sizeof(data->errmsg), "%s", err_user); + data->sqlstate = 0; } else { if (db_error == data->errmsg) { snprintf(db_error_copy, sizeof(db_error_copy), "%s", db_error); @@ -626,22 +647,26 @@ const char *cloudsync_errmsg (cloudsync_context *data) { } void cloudsync_apply_stats_reset (cloudsync_context *data) { - if (data) { data->apply_rows = 0; data->apply_denied = 0; } + if (data) { data->apply_rows = 0; data->apply_failed = 0; data->apply_failure[0] = 0; } } // Saturating: only a receive drain resets these, so on the direct-SQL apply path they // accumulate for the life of the connection and signed overflow would be undefined. -static void cloudsync_apply_stats_add (cloudsync_context *data, int rows, int denied) { +static void cloudsync_apply_stats_add (cloudsync_context *data, int rows, int failed) { if (rows > 0) data->apply_rows = (data->apply_rows > INT_MAX - rows) ? INT_MAX : data->apply_rows + rows; - if (denied > 0) data->apply_denied = (data->apply_denied > INT_MAX - denied) ? INT_MAX : data->apply_denied + denied; + if (failed > 0) data->apply_failed = (data->apply_failed > INT_MAX - failed) ? INT_MAX : data->apply_failed + failed; } -int cloudsync_apply_rows_count (cloudsync_context *data) { - return (data) ? data->apply_rows : 0; +int cloudsync_apply_failed_count (cloudsync_context *data) { + return (data) ? data->apply_failed : 0; } -int cloudsync_apply_denied_count (cloudsync_context *data) { - return (data) ? data->apply_denied : 0; +const char *cloudsync_apply_failure_message (cloudsync_context *data) { + return (data && data->apply_failure[0]) ? data->apply_failure : NULL; +} + +int cloudsync_apply_rows_count (cloudsync_context *data) { + return (data) ? data->apply_rows : 0; } int cloudsync_errcode (cloudsync_context *data) { @@ -651,6 +676,15 @@ int cloudsync_errcode (cloudsync_context *data) { void cloudsync_reset_error (cloudsync_context *data) { data->errmsg[0] = 0; data->errcode = DBRES_OK; + data->sqlstate = 0; +} + +void cloudsync_set_sqlstate (cloudsync_context *data, int sqlstate) { + if (data) data->sqlstate = sqlstate; +} + +int cloudsync_sqlstate (cloudsync_context *data) { + return (data) ? data->sqlstate : 0; } void *cloudsync_auxdata (cloudsync_context *data) { @@ -784,6 +818,12 @@ void table_free (cloudsync_table_context *table) { } cloudsync_memory_free(table->col_merge_stmt); } + if (table->col_update_stmt) { + for (int i=0; incols; ++i) { + if (table->col_update_stmt[i]) databasevm_finalize(table->col_update_stmt[i]); + } + cloudsync_memory_free(table->col_update_stmt); + } if (table->col_value_stmt) { for (int i=0; incols; ++i) { databasevm_finalize(table->col_value_stmt[i]); @@ -1138,6 +1178,9 @@ bool table_add_to_context (cloudsync_context *data, table_algo algo, const char table->col_merge_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); if (!table->col_merge_stmt) goto abort_add_table; + table->col_update_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); + if (!table->col_update_stmt) goto abort_add_table; + table->col_value_stmt = (dbvm_t **)cloudsync_memory_zeroalloc((uint64_t)(sizeof(void *) * ncols)); if (!table->col_value_stmt) goto abort_add_table; @@ -1360,6 +1403,7 @@ static int merge_pending_add (cloudsync_context *data, cloudsync_table_context * e->seq = seq; batch->count++; + batch->rows++; return DBRES_OK; } @@ -1382,15 +1426,18 @@ static void merge_pending_free_entries (merge_pending_batch *batch) { batch->sentinel_pending = false; batch->row_exists = false; batch->count = 0; + batch->rows = 0; } static int merge_flush_pending (cloudsync_context *data) { merge_pending_batch *batch = data->pending_batch; if (!batch) return DBRES_OK; + batch->flush_uncontained = false; int rc = DBRES_OK; bool flush_savepoint = false; char error_message[1024] = {0}; + int error_sqlstate = 0; // Nothing to write — handle sentinel-only case or skip if (batch->count == 0 && !(batch->sentinel_pending && batch->table)) { @@ -1400,9 +1447,12 @@ static int merge_flush_pending (cloudsync_context *data) { // Wrap database operations in a savepoint so that on failure (e.g. RLS // denial) the rollback properly releases all executor resources (open // relations, snapshots, plan cache) acquired during the failed statement. - rc = database_begin_savepoint(data, "merge_flush"); - if (rc != DBRES_OK) goto cleanup; - flush_savepoint = true; + // Inside a payload apply's PK-group savepoint the caller rolls back instead. + if (!batch->group_savepoint) { + rc = database_begin_savepoint(data, "merge_flush"); + if (rc != DBRES_OK) { batch->flush_uncontained = true; goto cleanup; } + flush_savepoint = true; + } if (batch->count == 0) { // Sentinel with no winning columns (PK-only row) @@ -1553,7 +1603,10 @@ static int merge_flush_pending (cloudsync_context *data) { } cleanup: - if (rc != DBRES_OK) snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); + if (rc != DBRES_OK) { + snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); + error_sqlstate = cloudsync_sqlstate(data); + } merge_pending_free_entries(batch); if (flush_savepoint) { if (rc == DBRES_OK) { @@ -1562,18 +1615,53 @@ static int merge_flush_pending (cloudsync_context *data) { // commit that fails (a deadlock or serialization failure, say) sets a real // message that the generic fallback below would otherwise replace. The // rollback runs after, and touches the error state itself. - if (rc != DBRES_OK) snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); + if (rc != DBRES_OK) { + snprintf(error_message, sizeof(error_message), "%s", cloudsync_errmsg(data)); + error_sqlstate = cloudsync_sqlstate(data); + } } - if (rc != DBRES_OK) database_rollback_savepoint(data, "merge_flush"); + if (rc != DBRES_OK && database_rollback_savepoint(data, "merge_flush") != DBRES_OK) batch->flush_uncontained = true; + } + if (rc != DBRES_OK && flush_savepoint) { + // the rollback cleared the error state: restore the failure's own message and SQLSTATE + cloudsync_reset_error(data); + cloudsync_set_error(data, error_message[0] ? error_message : "Unable to flush pending changes", rc); + cloudsync_set_sqlstate(data, error_sqlstate); } - if (rc != DBRES_OK) cloudsync_set_error(data, error_message[0] ? error_message : "Unable to flush pending changes", rc); return rc; } +// UPDATE of one column by primary key (pk params 1..npks, value npks+1), prepared on first +// use. Writing a column of an existing row through the column's upsert proposes a row +// holding only the primary key and that column; PostgreSQL checks an INSERT policy's +// WITH CHECK against that proposed row even when the conflict turns it into an update, +// so a policy on any other column rejects it, as does a NOT NULL constraint. An UPDATE +// is checked against the real row. Returns NULL when it cannot be prepared. +static dbvm_t *table_column_update_stmt (cloudsync_context *data, cloudsync_table_context *table, int col_idx) { + if (!table->col_update_stmt || col_idx < 0 || col_idx >= table->ncols) return NULL; + if (!table->col_update_stmt[col_idx]) { + const char *colnames[1] = {table->col_name[col_idx]}; + char *sql = sql_build_update_pk_and_multi_cols(data, table->name, colnames, 1, table->schema); + if (!sql) return NULL; + databasevm_prepare(data, sql, (void **)&table->col_update_stmt[col_idx], DBFLAG_PERSISTENT); + cloudsync_memory_free(sql); + } + return table->col_update_stmt[col_idx]; +} + int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *col_name, dbvalue_t *col_value, int64_t col_version, int64_t db_version, const char *site_id, int site_len, int64_t seq, int64_t *rowid) { int index; dbvm_t *vm = table_column_lookup(table, col_name, true, &index); if (vm == NULL) return cloudsync_set_error(data, "Unable to retrieve column merge precompiled statement in merge_insert_col", DBRES_MISUSE); + + // A grow-only set never deletes rows, so tracked metadata means the base row exists: + // write the column in place instead of through the upsert (see table_column_update_stmt). + bool update_only = false; + if (table->algo == table_algo_crdt_gos) { + int64_t local_cl = merge_get_local_cl(table, pk, pklen); + dbvm_t *update_vm = (local_cl > 0 && local_cl % 2 == 1) ? table_column_update_stmt(data, table, index) : NULL; + if (update_vm) { vm = update_vm; update_only = true; } + } // INSERT INTO table (pk1, pk2, col_name) VALUES (?, ?, ?) ON CONFLICT DO UPDATE SET col_name=?;" @@ -1588,10 +1676,10 @@ int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, c // bind value (always bind all expected parameters for correct prepared statement handling) if (col_value) { rc = databasevm_bind_value(vm, table->npks+1, col_value); - if (rc == DBRES_OK) rc = databasevm_bind_value(vm, table->npks+2, col_value); + if (rc == DBRES_OK && !update_only) rc = databasevm_bind_value(vm, table->npks+2, col_value); } else { rc = databasevm_bind_null(vm, table->npks+1); - if (rc == DBRES_OK) rc = databasevm_bind_null(vm, table->npks+2); + if (rc == DBRES_OK && !update_only) rc = databasevm_bind_null(vm, table->npks+2); } if (rc != DBRES_OK) { cloudsync_set_dberror(data); @@ -1609,7 +1697,21 @@ int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, c SYNCBIT_SET(data); rc = databasevm_step(vm); DEBUG_MERGE("merge_insert(%02x%02x): %s (%d)", data->site_id[UUID_LEN-2], data->site_id[UUID_LEN-1], databasevm_sql(vm), rc); + bool update_missed = update_only && rc == DBRES_DONE && databasevm_changes(vm) == 0; dbvm_reset(vm); + if (update_missed) { + // The row is not there to update — gone, or hidden by a policy's USING clause, + // which an UPDATE skips silently. Fall back to the upsert, which inserts a missing + // row or reports the policy, instead of recording a write that never happened. + vm = table_column_lookup(table, col_name, true, NULL); + rc = vm ? pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, vm) : DBRES_MISUSE; + if (rc >= 0 && vm) { + rc = col_value ? databasevm_bind_value(vm, table->npks+1, col_value) : databasevm_bind_null(vm, table->npks+1); + if (rc == DBRES_OK) rc = col_value ? databasevm_bind_value(vm, table->npks+2, col_value) : databasevm_bind_null(vm, table->npks+2); + if (rc == DBRES_OK) rc = databasevm_step(vm); + dbvm_reset(vm); + } + } SYNCBIT_RESET(data); if (table->algo == table_algo_crdt_gos) table->enabled = 1; @@ -1877,8 +1979,32 @@ static int block_delete_value (cloudsync_context *data, cloudsync_table_context } // Materialize all alive blocks for a base column into the base table +static int block_materialize_column_ex (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name, bool row_exists); + +// Every failure of block_materialize_column carries a message naming the stage, column +// and table: a bare code would reach the caller as a blank error. database_error is +// false for failures that involve no database call (an allocation), whose message must +// not pick up an unrelated database error left on the connection. +static int block_materialize_fail (cloudsync_context *data, cloudsync_table_context *table, const char *column, + const char *stage, int rc, bool database_error) { + char message[512]; + if (!database_error) cloudsync_reset_error(data); + if (rc == DBRES_NOMEM) { + snprintf(message, sizeof(message), "Not enough memory to %s the blocks of column \"%s\" of table \"%s\"", stage, column, table->name); + } else { + snprintf(message, sizeof(message), "Unable to %s the blocks of column \"%s\" of table \"%s\"", stage, column, table->name); + } + return cloudsync_set_error(data, message, (rc > 0) ? rc : DBRES_ERROR); +} + int block_materialize_column (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name) { - if (!table->block_list_stmt) return cloudsync_set_error(data, "block_materialize_column: blocks table not initialized", DBRES_MISUSE); + return block_materialize_column_ex(data, table, pk, pklen, base_col_name, false); +} + +// row_exists selects a plain UPDATE of the column instead of the column's upsert (see +// table_column_update_stmt). +static int block_materialize_column_ex (cloudsync_context *data, cloudsync_table_context *table, const void *pk, int pklen, const char *base_col_name, bool row_exists) { + if (!table->block_list_stmt) return block_materialize_fail(data, table, base_col_name, "read", DBRES_MISUSE, false); // Find column index and delimiter int col_idx = -1; @@ -1888,12 +2014,12 @@ int block_materialize_column (cloudsync_context *data, cloudsync_table_context * break; } } - if (col_idx < 0) return cloudsync_set_error(data, "block_materialize_column: column not found", DBRES_ERROR); + if (col_idx < 0) return block_materialize_fail(data, table, base_col_name, "locate", DBRES_ERROR, false); const char *delimiter = table->col_delimiter[col_idx] ? table->col_delimiter[col_idx] : BLOCK_DEFAULT_DELIMITER; // Build the LIKE pattern for block col_names: "base_col\x1F%" char *like_pattern = block_build_colname(base_col_name, "%"); - if (!like_pattern) return DBRES_NOMEM; + if (!like_pattern) return block_materialize_fail(data, table, base_col_name, "read", DBRES_NOMEM, false); // Query alive blocks from blocks table joined with metadata // block_list_stmt: SELECT b.col_value FROM blocks b JOIN meta m @@ -1902,14 +2028,15 @@ int block_materialize_column (cloudsync_context *data, cloudsync_table_context * // ORDER BY b.col_name dbvm_t *vm = table->block_list_stmt; int rc = databasevm_bind_blob(vm, 1, pk, pklen); - if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } - rc = databasevm_bind_text(vm, 2, like_pattern, -1); - if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 2, like_pattern, -1); // Bind pk again for the join condition (parameter 3) - rc = databasevm_bind_blob(vm, 3, pk, pklen); - if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } - rc = databasevm_bind_text(vm, 4, like_pattern, -1); - if (rc != DBRES_OK) { cloudsync_memory_free(like_pattern); databasevm_reset(vm); return rc; } + if (rc == DBRES_OK) rc = databasevm_bind_blob(vm, 3, pk, pklen); + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 4, like_pattern, -1); + if (rc != DBRES_OK) { + cloudsync_memory_free(like_pattern); + databasevm_reset(vm); + return block_materialize_fail(data, table, base_col_name, "read", rc, true); + } // Collect block values const char **block_values = NULL; @@ -1936,49 +2063,69 @@ int block_materialize_column (cloudsync_context *data, cloudsync_table_context * // Free collected values for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); if (block_values) cloudsync_memory_free((void *)block_values); - return cloudsync_set_error(data, "Unable to read block values", rc); + // an allocation failing while collecting is not a database read failure + if (rc == DBRES_NOMEM) return block_materialize_fail(data, table, base_col_name, "collect", rc, false); + return block_materialize_fail(data, table, base_col_name, "read", rc, true); } // Materialize text (NULL when no alive blocks) char *text = (block_count > 0) ? block_materialize_text(block_values, block_count, delimiter) : NULL; for (int i = 0; i < block_count; i++) cloudsync_memory_free((void *)block_values[i]); if (block_values) cloudsync_memory_free((void *)block_values); - if (block_count > 0 && !text) return DBRES_NOMEM; + if (block_count > 0 && !text) return block_materialize_fail(data, table, base_col_name, "join", DBRES_NOMEM, false); // Update the base table column via the col_merge_stmt (with triggers disabled) dbvm_t *merge_vm = table->col_merge_stmt[col_idx]; - if (!merge_vm) { cloudsync_memory_free(text); return DBRES_ERROR; } + bool update_only = false; + dbvm_t *update_vm = row_exists ? table_column_update_stmt(data, table, col_idx) : NULL; + if (update_vm) { merge_vm = update_vm; update_only = true; } + if (!merge_vm) { cloudsync_memory_free(text); return block_materialize_fail(data, table, base_col_name, "write", DBRES_MISUSE, false); } // Bind PKs rc = pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, merge_vm); - if (rc < 0) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return DBRES_ERROR; } + if (rc < 0) { + cloudsync_memory_free(text); + databasevm_reset(merge_vm); + return block_materialize_fail(data, table, base_col_name, "write", DBRES_ERROR, true); + } - // Bind the text value twice (INSERT value + ON CONFLICT UPDATE value) + // Bind the text value twice (INSERT value + ON CONFLICT UPDATE value); the plain + // UPDATE takes it once int npks = table->npks; - if (text) { - rc = databasevm_bind_text(merge_vm, npks + 1, text, -1); - if (rc != DBRES_OK) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return rc; } - rc = databasevm_bind_text(merge_vm, npks + 2, text, -1); - if (rc != DBRES_OK) { cloudsync_memory_free(text); databasevm_reset(merge_vm); return rc; } - } else { - rc = databasevm_bind_null(merge_vm, npks + 1); - if (rc != DBRES_OK) { databasevm_reset(merge_vm); return rc; } - rc = databasevm_bind_null(merge_vm, npks + 2); - if (rc != DBRES_OK) { databasevm_reset(merge_vm); return rc; } + rc = text ? databasevm_bind_text(merge_vm, npks + 1, text, -1) : databasevm_bind_null(merge_vm, npks + 1); + if (rc == DBRES_OK && !update_only) rc = text ? databasevm_bind_text(merge_vm, npks + 2, text, -1) : databasevm_bind_null(merge_vm, npks + 2); + if (rc != DBRES_OK) { + cloudsync_memory_free(text); + databasevm_reset(merge_vm); + return block_materialize_fail(data, table, base_col_name, "write", rc, true); } // Execute with triggers disabled table->enabled = 0; SYNCBIT_SET(data); rc = databasevm_step(merge_vm); + bool update_missed = update_only && rc == DBRES_DONE && databasevm_changes(merge_vm) == 0; databasevm_reset(merge_vm); + if (update_missed) { + // Nothing to update — the row is gone, or a policy's USING clause hides it and the + // UPDATE skipped it silently. Fall back to the upsert, which inserts a missing row + // or reports the policy, rather than leave the column unwritten without a word. + merge_vm = table->col_merge_stmt[col_idx]; + rc = merge_vm ? pk_decode_prikey((char *)pk, (size_t)pklen, pk_decode_bind_callback, merge_vm) : DBRES_MISUSE; + if (rc >= 0 && merge_vm) { + rc = text ? databasevm_bind_text(merge_vm, npks + 1, text, -1) : databasevm_bind_null(merge_vm, npks + 1); + if (rc == DBRES_OK) rc = text ? databasevm_bind_text(merge_vm, npks + 2, text, -1) : databasevm_bind_null(merge_vm, npks + 2); + if (rc == DBRES_OK) rc = databasevm_step(merge_vm); + databasevm_reset(merge_vm); + } + } SYNCBIT_RESET(data); table->enabled = 1; cloudsync_memory_free(text); if (rc == DBRES_DONE) rc = DBRES_OK; - if (rc != DBRES_OK) return cloudsync_set_dberror(data); + if (rc != DBRES_OK) return block_materialize_fail(data, table, base_col_name, "write", (rc > 0) ? rc : DBRES_ERROR, true); return DBRES_OK; } @@ -2060,6 +2207,7 @@ int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const int rc = merge_sentinel_only_insert(data, table, insert_pk, insert_pk_len, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_sentinel_only_insert", rc); + else if (data->pending_batch) data->pending_batch->rows++; // an explicit sentinel is a payload row return rc; } @@ -2110,9 +2258,10 @@ int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const // Materialize the full column from blocks into the base table char *base_col = block_extract_base_colname(insert_name); if (base_col) { - rc = block_materialize_column(data, table, insert_pk, insert_pk_len, base_col); + rc = block_materialize_column_ex(data, table, insert_pk, insert_pk_len, base_col, (local_cl % 2 == 1) && !needs_resurrect); cloudsync_memory_free(base_col); - if (rc != DBRES_OK) return cloudsync_set_error(data, "Unable to materialize block column", rc); + // the failure already names the stage, column and table + if (rc != DBRES_OK) return rc; } return DBRES_OK; @@ -4052,6 +4201,189 @@ static void cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t } } +// A failed write is skipped only when retrying could not change the outcome. A lock, +// deadlock, serialization failure, cancel or resource shortage can succeed on the next +// attempt, and a missing privilege or a read-only database is fixed by configuration, +// not by the data: those still fail the apply and leave the receive cursor in place. +static bool cloudsync_apply_error_is_transient (int rc) { + // SQLite can report an extended code (SQLITE_BUSY_SNAPSHOT is 517): its low byte + // is the primary code. DBRES_POLICY_DENIED's low byte matches no case below. + if (rc <= 0) return false; + switch (rc & 0xFF) { + case DBRES_PERM: + case DBRES_READONLY: + case DBRES_AUTH: + case DBRES_BUSY: + case DBRES_LOCKED: + case DBRES_NOMEM: + case DBRES_INTERRUPT: + case DBRES_IOERR: + case DBRES_FULL: + return true; + } + return false; +} + +typedef struct { + int denied; // entries rejected by a row-level security policy + int failed; // entries skipped because their write failed + int fatal_rc; // first failure that stops the apply (DBRES_OK = none) + int fatal_sqlstate; + char fatal_message[1024]; +} cloudsync_apply_outcome; + +// Classifies one failed write of cloudsync_payload_apply. contained is true when the +// failure has already been rolled back, leaving the transaction usable. Returns true +// when the apply must stop. +static bool cloudsync_apply_note_failure (cloudsync_context *data, cloudsync_apply_outcome *out, int rc, int entries, bool contained) { + if (entries < 1) entries = 1; + if (rc == DBRES_POLICY_DENIED && contained) { + out->denied += entries; + return false; + } + if (contained && !cloudsync_apply_error_is_transient(rc)) { + // A write that fails on its data (a constraint, a trigger, a type error) fails + // the same way on every retry. Holding the cursor back would re-deliver it + // forever and stall every later change behind it, so the change is skipped — + // but reported, never dropped silently. + const char *msg = cloudsync_errmsg(data); + if (!msg || !msg[0]) msg = "Unable to apply a received change"; + out->failed += entries; + if (!data->apply_failure[0]) snprintf(data->apply_failure, sizeof(data->apply_failure), "%s", msg); + char warning[1200]; + snprintf(warning, sizeof(warning), "skipped %d received change%s that failed to apply: %s", entries, (entries == 1) ? "" : "s", msg); + database_log_warning(data, warning); + return false; + } + if (out->fatal_rc == DBRES_OK) { + out->fatal_rc = rc; + out->fatal_sqlstate = cloudsync_sqlstate(data); + snprintf(out->fatal_message, sizeof(out->fatal_message), "%s", cloudsync_errmsg(data)); + } + return true; +} + +// Steps one decoded payload row (an INSERT into cloudsync_changes). *contained reports +// whether a failure was rolled back and left the transaction usable. On SQLite a failed +// statement always is. On PostgreSQL the merge runs inside the cloudsync_changes trigger, +// so the row gets its own savepoint: a denial or a failed write then rolls back cleanly +// — metadata included — and the apply can go on to the next row. +static int cloudsync_payload_apply_row (cloudsync_context *data, dbvm_t *vm, bool *contained) { +#ifdef CLOUDSYNC_POSTGRESQL_BUILD + int rc = database_begin_savepoint(data, "cloudsync_apply_row"); + if (rc != DBRES_OK) { *contained = false; return rc; } + rc = databasevm_step(vm); + if (rc == DBRES_DONE) { + int commit_rc = database_commit_savepoint(data, "cloudsync_apply_row"); + if (commit_rc == DBRES_OK) { *contained = true; return DBRES_DONE; } + rc = commit_rc; + } + char message[1024]; + snprintf(message, sizeof(message), "%s", cloudsync_errmsg(data)); + int sqlstate = cloudsync_sqlstate(data); + *contained = (database_rollback_savepoint(data, "cloudsync_apply_row") == DBRES_OK); + cloudsync_reset_error(data); + cloudsync_set_error(data, message[0] ? message : "Unable to apply a received change", rc); + cloudsync_set_sqlstate(data, sqlstate); + return rc; +#else + *contained = true; + return databasevm_step(vm); +#endif +} + +// True when the decoded payload row writes one block of a block column. Its merge +// materializes the column into the base table, so the row's other pending columns must +// be written first: materializing into a row that does not exist yet has to insert it +// with nothing but the primary key and that column. +static bool cloudsync_payload_row_is_block (const cloudsync_pk_decode_bind_context *row) { + return row->col_name && row->col_name_len > 0 && + memchr(row->col_name, BLOCK_SEPARATOR, (size_t)row->col_name_len) != NULL; +} + +// A run of payload rows rejected by a row-level security policy, kept so the apply can +// retry it once the rest of the payload is in: a policy can depend on rows that arrive +// later in the same payload (a membership row granting access to the rows before it). +typedef struct { + const char *start; // first row of the run inside the payload buffer + size_t avail; // bytes left in the buffer from start + uint32_t rows; // payload rows in the run + int entries; // entries counted as denied for the run +} cloudsync_denied_run; + +typedef struct { + cloudsync_denied_run *runs; + int count; + int capacity; +} cloudsync_denied_runs; + +// Records a denied run. A run for a whole PK group absorbs the single-row runs already +// recorded inside it, so no row is retried twice. Out of memory only loses the retry: +// the entries stay counted as denied. +static void cloudsync_denied_runs_add (cloudsync_denied_runs *list, const char *start, size_t avail, uint32_t rows, int entries) { + while (list->count > 0 && list->runs[list->count - 1].start >= start) { + entries += list->runs[--list->count].entries; + } + if (list->count == list->capacity) { + int capacity = list->capacity ? list->capacity * 2 : 8; + cloudsync_denied_run *runs = cloudsync_memory_realloc(list->runs, (uint64_t)capacity * sizeof(*runs)); + if (!runs) return; + list->runs = runs; + list->capacity = capacity; + } + list->runs[list->count++] = (cloudsync_denied_run){.start = start, .avail = avail, .rows = rows, .entries = entries}; +} + +// Opens the savepoint around a PK group of payload rows (see merge_pending_batch). If it +// cannot be opened the group runs without one and the flush falls back to its own. +static void cloudsync_payload_group_open (cloudsync_context *data, merge_pending_batch *batch) { + if (batch->group_savepoint) return; + batch->group_savepoint = (database_begin_savepoint(data, "cloudsync_merge_group") == DBRES_OK); + if (!batch->group_savepoint) cloudsync_reset_error(data); +} + +// Flushes the pending PK group and closes its savepoint: released when the flush +// succeeds, rolled back when it fails, so a failed group leaves no trace behind — its +// rows can then be retried or delivered again and apply cleanly. *entries receives the +// payload rows the flush covered and *contained whether a failure left the transaction +// usable. The failure's message and SQLSTATE survive the rollback. +static int cloudsync_payload_group_flush (cloudsync_context *data, merge_pending_batch *batch, int *entries, bool *contained) { + *entries = batch->rows; + int rc = merge_flush_pending(data); + *contained = !batch->flush_uncontained; + if (!batch->group_savepoint) return rc; + batch->group_savepoint = false; + if (rc == DBRES_OK) { + rc = database_commit_savepoint(data, "cloudsync_merge_group"); + if (rc == DBRES_OK) return DBRES_OK; + } + char message[1024]; + snprintf(message, sizeof(message), "%s", cloudsync_errmsg(data)); + int sqlstate = cloudsync_sqlstate(data); + *contained = (database_rollback_savepoint(data, "cloudsync_merge_group") == DBRES_OK); + cloudsync_reset_error(data); + cloudsync_set_error(data, message[0] ? message : "Unable to flush pending changes", rc); + cloudsync_set_sqlstate(data, sqlstate); + return rc; +} + +// Abandons an open PK group after the apply stopped: nothing it wrote is kept. +static void cloudsync_payload_group_abandon (cloudsync_context *data, merge_pending_batch *batch) { + merge_pending_free_entries(batch); + if (!batch->group_savepoint) return; + batch->group_savepoint = false; + char message[1024]; + snprintf(message, sizeof(message), "%s", cloudsync_errmsg(data)); + int sqlstate = cloudsync_sqlstate(data); + int code = cloudsync_errcode(data); + database_rollback_savepoint(data, "cloudsync_merge_group"); + if (code != DBRES_OK) { + cloudsync_reset_error(data); + cloudsync_set_error(data, message, code); + cloudsync_set_sqlstate(data, sqlstate); + } +} + int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *pnrows, int64_t checkpoint_db_version, int64_t checkpoint_seq) { // Guard against calling payload_apply before cloudsync_init: without this, // the settings lookups at the top of this function would each emit a @@ -4186,9 +4518,12 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b uint16_t ncols = header.ncols; uint32_t nrows = header.nrows; int64_t last_payload_db_version = -1; - int first_error = DBRES_OK; - int denied_entries = 0; - char first_error_message[1024] = {0}; + cloudsync_apply_outcome outcome = {0}; + bool stop = false; + cloudsync_denied_runs denied_runs = {0}; + const char *group_start = NULL; // first row of the current PK group + size_t group_avail = 0; + uint32_t group_rows = 0; cloudsync_pk_decode_bind_context decoded_context = {.vm = vm}; // Initialize deferred column-batch merge @@ -4202,9 +4537,11 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b for (uint32_t i=0; iskip_decode_idx, cloudsync_payload_decode_callback, &decoded_context); if (res == -1) { - merge_flush_pending(data); + cloudsync_payload_group_abandon(data, &batch); data->pending_batch = NULL; if (batch.cached_vm) { databasevm_finalize(batch.cached_vm); batch.cached_vm = NULL; } if (batch.cached_col_names) { cloudsync_memory_free(batch.cached_col_names); batch.cached_col_names = NULL; } @@ -4223,23 +4560,31 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b memcmp(last_tbl, decoded_context.tbl, last_tbl_len) != 0)); bool db_version_changed = (last_payload_db_version != decoded_context.db_version); - // Flush pending batch before any boundary change + // Flush pending batch before any boundary change, closing the PK group if (pk_changed || tbl_changed || db_version_changed) { - int pending_entries = batch.count; - int flush_rc = merge_flush_pending(data); - if (flush_rc == DBRES_POLICY_DENIED) denied_entries += (pending_entries > 0) ? pending_entries : 1; - else if (flush_rc != DBRES_OK && first_error == DBRES_OK) { - first_error = flush_rc; - snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); + int entries = 0; + bool contained = true; + int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); + if (flush_rc == DBRES_POLICY_DENIED && contained && group_start) { + cloudsync_denied_runs_add(&denied_runs, group_start, group_avail, group_rows, entries); + } + if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { + stop = true; + break; } + group_start = row_start; + group_avail = row_avail; + group_rows = 0; } + if (!group_start) { group_start = row_start; group_avail = row_avail; } + group_rows++; // Per-db_version savepoints group rows with the same source db_version // into one transaction. In SQLite autocommit mode, the RELEASE triggers // the commit hook which bumps data->db_version and resets seq, ensuring // unique (db_version, seq) tuples across groups. In PostgreSQL SPI, // database_in_transaction() is always true so this block is inactive — - // the inner per-PK savepoint in merge_flush_pending handles RLS instead. + // the per-PK group savepoint handles RLS and failed writes instead. if (in_savepoint && db_version_changed) { rc = database_commit_savepoint(data, "cloudsync_payload_apply"); if (rc != DBRES_OK) { @@ -4273,15 +4618,34 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b last_tbl = decoded_context.tbl; last_tbl_len = decoded_context.tbl_len; - rc = databasevm_step(vm); - if (rc == DBRES_POLICY_DENIED) { denied_entries++; rc = DBRES_DONE; } + if (batch.count > 0 && cloudsync_payload_row_is_block(&decoded_context)) { + int entries = 0; + bool contained = true; + int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); + if (flush_rc == DBRES_POLICY_DENIED && contained && group_start && group_rows > 1) { + // the rows of this PK group before the current one + cloudsync_denied_runs_add(&denied_runs, group_start, group_avail, group_rows - 1, entries); + } + if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { + stop = true; + break; + } + } + + cloudsync_payload_group_open(data, &batch); + bool contained = true; + rc = cloudsync_payload_apply_row(data, vm, &contained); if (rc != DBRES_DONE) { - if (first_error == DBRES_OK) { - first_error = rc; - snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); + if (rc == DBRES_POLICY_DENIED && contained) { + cloudsync_denied_runs_add(&denied_runs, row_start, row_avail, 1, 1); + } + if (cloudsync_apply_note_failure(data, &outcome, rc, 1, contained)) { + stop = true; + buffer += seek; + buf_len -= seek; + dbvm_reset(vm); + break; } - // don't "break;", the error can be due to a RLS policy. - // in case of error we try to apply the following changes } buffer += seek; @@ -4289,52 +4653,141 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b dbvm_reset(vm); } - // Final flush after loop - { - int pending_entries = batch.count; - int flush_rc = merge_flush_pending(data); - if (flush_rc == DBRES_POLICY_DENIED) denied_entries += (pending_entries > 0) ? pending_entries : 1; - else if (flush_rc != DBRES_OK && first_error == DBRES_OK) { - first_error = flush_rc; - snprintf(first_error_message, sizeof(first_error_message), "%s", cloudsync_errmsg(data)); + // Final flush after loop (a stopped apply discards the open group) + if (stop) { + cloudsync_payload_group_abandon(data, &batch); + } else { + int entries = 0; + bool contained = true; + int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); + if (flush_rc == DBRES_POLICY_DENIED && contained && group_start) { + cloudsync_denied_runs_add(&denied_runs, group_start, group_avail, group_rows, entries); } + if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) stop = true; + } + + // The receive checkpoint is the last row of the payload, whatever a retry decodes. + int64_t payload_last_db_version = decoded_context.db_version; + int64_t payload_last_seq = decoded_context.seq; + + // Retry denied runs now that every other row is in: a policy that depends on rows + // later in the payload lets them through on a later pass. Repeat while a pass makes + // progress, so a chain of such dependencies resolves; a run still denied after the + // last pass is final. Rows that already applied are re-merged as no-ops. A denied + // group left no trace (its savepoint was rolled back), so it retries cleanly. + for (int pass = 0; !stop && denied_runs.count > 0 && pass < 32; pass++) { + cloudsync_denied_runs pending = denied_runs; + denied_runs = (cloudsync_denied_runs){0}; + bool progress = false; + for (int r = 0; r < pending.count && !stop; r++) { + cloudsync_denied_run *run = &pending.runs[r]; + const char *p = run->start; + size_t avail = run->avail; + int denied_now = 0; + bool decode_failed = false; + for (uint32_t k = 0; k < run->rows && !stop; k++) { + size_t seek = 0; + if (pk_decode((char *)p, avail, ncols, &seek, data->skip_decode_idx, cloudsync_payload_decode_callback, &decoded_context) == -1) { + decode_failed = true; + break; + } + if (batch.count > 0 && cloudsync_payload_row_is_block(&decoded_context)) { + int entries = 0; + bool contained = true; + int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); + if (flush_rc == DBRES_POLICY_DENIED && contained) { + denied_now += entries; + } else if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { + stop = true; + break; + } + } + cloudsync_payload_group_open(data, &batch); + bool contained = true; + int step_rc = cloudsync_payload_apply_row(data, vm, &contained); + if (step_rc == DBRES_POLICY_DENIED && contained) { + denied_now++; + } else if (step_rc != DBRES_DONE && cloudsync_apply_note_failure(data, &outcome, step_rc, 1, contained)) { + stop = true; + } + p += seek; + avail -= seek; + dbvm_reset(vm); + } + if (!stop && !decode_failed) { + int entries = 0; + bool contained = true; + int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); + if (flush_rc == DBRES_POLICY_DENIED && contained) { + denied_now += entries; + } else if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { + stop = true; + } + } else { + cloudsync_payload_group_abandon(data, &batch); + } + if (stop || decode_failed) { + // Keep the run counted as it was; nothing more can be retried. + cloudsync_denied_runs_add(&denied_runs, run->start, run->avail, run->rows, run->entries); + continue; + } + outcome.denied -= run->entries; + if (denied_now > 0) { + outcome.denied += denied_now; + cloudsync_denied_runs_add(&denied_runs, run->start, run->avail, run->rows, denied_now); + } + if (denied_now < run->entries) progress = true; + } + cloudsync_memory_free(pending.runs); + if (!progress) break; } + if (outcome.denied < 0) outcome.denied = 0; data->pending_batch = NULL; + rc = DBRES_OK; if (in_savepoint) { int rc1 = database_commit_savepoint(data, "cloudsync_payload_apply"); - if (rc1 != DBRES_OK) rc = rc1; + if (rc1 != DBRES_OK && outcome.fatal_rc == DBRES_OK) { + outcome.fatal_rc = rc1; + outcome.fatal_sqlstate = cloudsync_sqlstate(data); + snprintf(outcome.fatal_message, sizeof(outcome.fatal_message), "%s", cloudsync_errmsg(data)); + } } - if (first_error != DBRES_OK) rc = first_error; - - // save last error (unused if function returns OK) - if (rc != DBRES_OK && rc != DBRES_DONE) { - cloudsync_set_error(data, first_error_message[0] ? first_error_message : "Unable to apply payload changes", rc); + if (outcome.fatal_rc != DBRES_OK) { + rc = outcome.fatal_rc; + // The captured message already carries the database error; clear it first so + // it is not appended to itself a second time. + cloudsync_reset_error(data); + cloudsync_set_error(data, outcome.fatal_message[0] ? outcome.fatal_message : "Unable to apply payload changes", rc); + cloudsync_set_sqlstate(data, outcome.fatal_sqlstate); } - if (rc == DBRES_DONE) rc = DBRES_OK; - { - int applied = (rc == DBRES_OK) ? (int)nrows - denied_entries : 0; - cloudsync_apply_stats_add(data, (applied > 0) ? applied : 0, denied_entries); + int applied = (rc == DBRES_OK) ? (int)nrows - outcome.denied - outcome.failed : 0; + cloudsync_apply_stats_add(data, (applied > 0) ? applied : 0, outcome.failed); } - // A policy denial is permanent: those rows are not this site's to hold, so the - // cursor must still advance. Holding it back would re-deliver the same rows on - // every check forever, and a single denied row would stall every later change - // behind it. Denials are counted and reported instead (receive.denied), so - // discarding stays visible without being an error: a payload can be entirely - // denied and still be a correct outcome, since a single-row payload that - // belongs to another user is denied in full. + // Policy denials and failed writes that survive the retry are permanent: delivering + // them again fails the same way. So the cursor still advances past them — holding it + // back would re-deliver them forever and stall every later change behind them. They + // are reported instead: a warning per failed write (receive.failed on the network + // path) and one summary warning for denials. Only a transient or uncontained failure + // fails the apply and keeps the cursor in place. + if (rc == DBRES_OK && outcome.denied > 0) { + char warning[256]; + snprintf(warning, sizeof(warning), "skipped %d received change%s denied by a row-level security policy", + outcome.denied, (outcome.denied == 1) ? "" : "s"); + database_log_warning(data, warning); + } if (rc == DBRES_OK) { // Record the last applied (db_version, seq) and advance the receive cursor // once, gated on the caller-supplied checkpoint. A non-final chunk passes // CLOUDSYNC_CHECKPOINT_NONE so the cursor never lands mid-db_version. - if (decoded_context.db_version > data->apply_last_db_version || - (decoded_context.db_version == data->apply_last_db_version && decoded_context.seq > data->apply_last_seq)) { - data->apply_last_db_version = decoded_context.db_version; - data->apply_last_seq = decoded_context.seq; + if (payload_last_db_version > data->apply_last_db_version || + (payload_last_db_version == data->apply_last_db_version && payload_last_seq > data->apply_last_seq)) { + data->apply_last_db_version = payload_last_db_version; + data->apply_last_seq = payload_last_seq; } cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); } @@ -4350,6 +4803,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // cleanup memory if (clone) cloudsync_memory_free(clone); + cloudsync_memory_free(denied_runs.runs); // error already saved in (save last error) if (rc != DBRES_OK) return rc; @@ -4433,7 +4887,11 @@ int local_block_update(cloudsync_context *data, cloudsync_table_context *table, block_diff_free(diff); block_list_free(old); block_list_free(next); - if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to update block metadata or values", rc); + if (rc != DBRES_OK) { + char message[512]; + snprintf(message, sizeof(message), "Unable to write the blocks of column \"%s\" of table \"%s\"", col, table->name); + cloudsync_set_error(data, message, rc); + } return rc; } diff --git a/src/cloudsync.h b/src/cloudsync.h index b9ee1a7d..f7a8dc22 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -110,15 +110,23 @@ int cloudsync_set_dberror (cloudsync_context *data); const char *cloudsync_errmsg (cloudsync_context *data); int cloudsync_errcode (cloudsync_context *data); void cloudsync_reset_error (cloudsync_context *data); - -// Entries applied, and entries rejected by a row-level security policy. Both counts -// accumulate across a receive drain (reset once before it) so denials in an early -// chunk are still reported by the call that finishes the drain. The applied count is -// tracked here rather than derived from the apply return value, which reports the -// payload's entry count (denied ones included) as part of the SQL surface. +// SQLSTATE of the database error behind the current error, encoded as PostgreSQL's +// MAKE_SQLSTATE integer; 0 when there is none (always 0 on SQLite). Reported by the +// PostgreSQL functions so a caller can still tell a serialization failure (40001) or a +// unique violation (23505) from an internal error. +void cloudsync_set_sqlstate (cloudsync_context *data, int sqlstate); +int cloudsync_sqlstate (cloudsync_context *data); + +// Entries applied and entries skipped because their write failed. Both counts accumulate +// across a receive drain (reset once before it) so an early chunk is still reported by +// the call that finishes the drain. The applied count is tracked here rather than +// derived from the apply return value, which reports the payload's entry count (skipped +// ones included) as part of the SQL surface. cloudsync_apply_failure_message returns the +// first skipped failure's message since the last reset, or NULL. void cloudsync_apply_stats_reset (cloudsync_context *data); int cloudsync_apply_rows_count (cloudsync_context *data); -int cloudsync_apply_denied_count (cloudsync_context *data); +int cloudsync_apply_failed_count (cloudsync_context *data); +const char *cloudsync_apply_failure_message (cloudsync_context *data); int cloudsync_commit_hook (void *ctx); void cloudsync_rollback_hook (void *ctx); void cloudsync_set_schema (cloudsync_context *data, const char *schema); diff --git a/src/database.h b/src/database.h index 9ccdc6fe..127ea1a9 100644 --- a/src/database.h +++ b/src/database.h @@ -19,10 +19,17 @@ typedef void dbvalue_t; typedef enum { DBRES_OK = 0, DBRES_ERROR = 1, + DBRES_PERM = 3, // missing privilege: fixed by a GRANT, not by the data DBRES_ABORT = 4, + DBRES_BUSY = 5, // transient: lock, deadlock, serialization failure, cancel + DBRES_LOCKED = 6, DBRES_NOMEM = 7, + DBRES_READONLY = 8, + DBRES_INTERRUPT = 9, DBRES_IOERR = 10, + DBRES_FULL = 13, DBRES_CONSTRAINT = 19, + DBRES_AUTH = 23, DBRES_MISUSE = 21, DBRES_ROW = 100, DBRES_DONE = 101, @@ -92,6 +99,7 @@ int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_ bool database_in_transaction (cloudsync_context *data); int database_errcode (cloudsync_context *data); const char *database_errmsg (cloudsync_context *data); +void database_log_warning (cloudsync_context *data, const char *message); // VM int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, int flags); @@ -100,6 +108,7 @@ void databasevm_finalize (dbvm_t *vm); void databasevm_reset (dbvm_t *vm); void databasevm_clear_bindings (dbvm_t *vm); const char *databasevm_sql (dbvm_t *vm); +int64_t databasevm_changes (dbvm_t *vm); // rows changed by the last completed INSERT/UPDATE/DELETE step // BINDING int databasevm_bind_blob (dbvm_t *vm, int index, const void *value, uint64_t size); diff --git a/src/network/network.c b/src/network/network.c index 30417699..184b1ce3 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -2211,7 +2211,8 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync // Result of a receive drain (see network_drain_changes). typedef struct { int rows; // cumulative rows applied across the drain - int denied; // payload entries rejected by a row-level security policy + int failed; // payload entries skipped because their write failed + char *failed_err; // owned; message of the first skipped failure, or NULL int chunks; // payload chunks applied this drain int64_t bytes; // serialized payload bytes received this drain bool complete; // true iff the receive stream is fully drained (nothing pending) @@ -2219,6 +2220,41 @@ typedef struct { char *receive_err; // owned by the caller; client-side apply error, or NULL } drain_result; +// Builds the "receive":{...} member shared by cloudsync_network_sync and +// cloudsync_network_receive_changes. receive_err and check_failure_json are optional; +// "failedError" is emitted only with a skipped failure's message. Returns a malloc'd +// string, or NULL when out of memory. +static char *network_receive_json (int rows, const drain_result *dr, const char *tables, + const char *receive_err, const char *check_failure_json) { + char *escaped_err = receive_err ? json_escape_string(receive_err) : NULL; + char *escaped_failed = dr->failed_err ? json_escape_string(dr->failed_err) : NULL; + char *error_part = escaped_err ? cloudsync_memory_mprintf(",\"error\":\"%s\"", escaped_err) : NULL; + char *failed_error_part = escaped_failed ? cloudsync_memory_mprintf(",\"failedError\":\"%s\"", escaped_failed) : NULL; + char *last_failure_part = check_failure_json ? cloudsync_memory_mprintf(",\"lastFailure\":%s", check_failure_json) : NULL; + char *json = NULL; + if ((!receive_err || error_part) && (!dr->failed_err || failed_error_part) && (!check_failure_json || last_failure_part)) { + json = cloudsync_memory_mprintf( + "\"receive\":{\"rows\":%d,\"failed\":%d%s,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s%s%s}", + rows, dr->failed, failed_error_part ? failed_error_part : "", tables ? tables : "[]", + dr->chunks, (long long)dr->bytes, dr->complete ? "true" : "false", + error_part ? error_part : "", last_failure_part ? last_failure_part : ""); + } + if (escaped_err) cloudsync_memory_free(escaped_err); + if (escaped_failed) cloudsync_memory_free(escaped_failed); + if (error_part) cloudsync_memory_free(error_part); + if (failed_error_part) cloudsync_memory_free(failed_error_part); + if (last_failure_part) cloudsync_memory_free(last_failure_part); + return json; +} + +#ifdef CLOUDSYNC_UNITTEST +char *network_test_receive_json (int rows, int failed, const char *failed_err, bool complete, + const char *receive_err, const char *check_failure_json) { + drain_result dr = {.rows = rows, .failed = failed, .failed_err = (char *)failed_err, .chunks = 1, .bytes = 10, .complete = complete}; + return network_receive_json(rows, &dr, "[\"t\"]", receive_err, check_failure_json); +} +#endif + // Drains chunked /check responses into the local database. Chunks that are already // available are fetched back-to-back with no delay; wait_ms/max_retries are spent // only while the server payload is not yet ready (HTTP 202). max_chunks > 0 caps how @@ -2236,8 +2272,8 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, int64_t drain_prev_dbv = cloudsync_dbversion(data); sr->defer_tables = true; - // Denials accumulate on the context across every chunk of this drain, so a - // denial in an early chunk is still reported by the call that finishes it. + // Apply counts accumulate on the context across every chunk of this drain, so a + // skipped write in an early chunk is still reported by the call that finishes it. cloudsync_apply_stats_reset(data); int ntries = 0; // counts only "nothing ready" (202) polls @@ -2299,10 +2335,9 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, } // Compute the affected-tables union once, over the whole drain window. - // Report rows actually written, not payload entries: an all-denied receive would - // otherwise claim {"rows":N,"denied":N} while tables is correctly empty. The apply - // return value still counts payload entries, which is a tested part of the SQL - // surface, so the accurate count is accumulated on the context instead. + // Report rows actually written, not payload entries: skipped writes are excluded. + // The apply return value still counts payload entries, which is a tested part of the + // SQL surface, so the accurate count is accumulated on the context instead. int applied_total = cloudsync_apply_rows_count(data); if (!receive_err && rc == SQLITE_OK && applied_total > 0) { @@ -2310,7 +2345,9 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, } dr->rows = applied_total; - dr->denied = cloudsync_apply_denied_count(data); + dr->failed = cloudsync_apply_failed_count(data); + const char *failure = cloudsync_apply_failure_message(data); + dr->failed_err = failure ? cloudsync_string_dup(failure) : NULL; dr->chunks = nchunks; dr->bytes = bytes_total; dr->complete = complete; @@ -2346,10 +2383,7 @@ void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retr } const char *tables = sr.tables_json ? sr.tables_json : "[]"; - const char *complete_str = dr.complete ? "true" : "false"; const char *status = sr.status ? sr.status : "error"; - char *escaped_err = receive_err ? json_escape_string(receive_err) : NULL; - // Build send and receive blocks separately to avoid combinatorial explosion // across optional fields (send.lastFailure, receive.error, receive.lastFailure). char *send_part = sr.apply_failure_json @@ -2360,31 +2394,15 @@ void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retr "\"send\":{\"status\":\"%s\",\"localVersion\":%lld,\"serverVersion\":%lld,\"chunks\":%d,\"bytes\":%lld}", status, (long long)sr.local_version, (long long)sr.server_version, sr.send_chunks, (long long)sr.send_bytes); - char *recv_part; - if (escaped_err && sr.check_failure_json) { - recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}", - nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err, sr.check_failure_json); - } else if (escaped_err) { - recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}", - nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped_err); - } else if (sr.check_failure_json) { - recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}", - nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); - } else { - recv_part = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}", - nrows_total, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str); - } + char *recv_part = network_receive_json(nrows_total, &dr, tables, receive_err, sr.check_failure_json); - char *buf = cloudsync_memory_mprintf("{%s,%s}", send_part, recv_part); - cloudsync_memory_free(send_part); - cloudsync_memory_free(recv_part); + char *buf = (send_part && recv_part) ? cloudsync_memory_mprintf("{%s,%s}", send_part, recv_part) : NULL; + if (send_part) cloudsync_memory_free(send_part); + if (recv_part) cloudsync_memory_free(recv_part); - sqlite3_result_text(context, buf, -1, cloudsync_memory_free); - if (escaped_err) cloudsync_memory_free(escaped_err); + if (buf) sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + else sqlite3_result_error_nomem(context); + if (dr.failed_err) cloudsync_memory_free(dr.failed_err); if (receive_err) cloudsync_memory_free(receive_err); if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); @@ -2427,6 +2445,7 @@ static void network_receive_changes_impl (sqlite3_context *context, int max_chun if (rc != SQLITE_OK && !receive_err) { if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); + if (dr.failed_err) cloudsync_memory_free(dr.failed_err); return; } @@ -2447,6 +2466,7 @@ static void network_receive_changes_impl (sqlite3_context *context, int max_chun if (code) cloudsync_memory_free(code); if (message) cloudsync_memory_free(message); if (receive_err) cloudsync_memory_free(receive_err); + if (dr.failed_err) cloudsync_memory_free(dr.failed_err); if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); return; @@ -2459,24 +2479,12 @@ static void network_receive_changes_impl (sqlite3_context *context, int max_chun } const char *tables = sr.tables_json ? sr.tables_json : "[]"; - const char *complete_str = dr.complete ? "true" : "false"; - char *escaped = receive_err ? json_escape_string(receive_err) : NULL; - char *buf; - if (escaped && sr.check_failure_json) { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\",\"lastFailure\":%s}}", - nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped, sr.check_failure_json); - } else if (escaped) { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"error\":\"%s\"}}", - nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, escaped); - } else if (sr.check_failure_json) { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s,\"lastFailure\":%s}}", - nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str, sr.check_failure_json); - } else { - buf = cloudsync_memory_mprintf("{\"receive\":{\"rows\":%d,\"denied\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s}}", - nrows, dr.denied, tables, dr.chunks, (long long)dr.bytes, complete_str); - } - sqlite3_result_text(context, buf, -1, cloudsync_memory_free); - if (escaped) cloudsync_memory_free(escaped); + char *recv_part = network_receive_json(nrows, &dr, tables, receive_err, sr.check_failure_json); + char *buf = recv_part ? cloudsync_memory_mprintf("{%s}", recv_part) : NULL; + if (recv_part) cloudsync_memory_free(recv_part); + if (buf) sqlite3_result_text(context, buf, -1, cloudsync_memory_free); + else sqlite3_result_error_nomem(context); + if (dr.failed_err) cloudsync_memory_free(dr.failed_err); if (receive_err) cloudsync_memory_free(receive_err); if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c index a310f96c..1ed17f7b 100644 --- a/src/postgresql/cloudsync_postgresql.c +++ b/src/postgresql/cloudsync_postgresql.c @@ -154,6 +154,14 @@ void _PG_fini (void) { // MARK: - Public SQL Functions - // cloudsync_version() - Returns extension version +// SQLSTATE for an error raised from the cloudsync context: the database error's own +// SQLSTATE when there is one, so callers can still react to a serialization failure, +// a deadlock or a constraint violation; internal_error for cloudsync's own failures. +static int cloudsync_error_sqlstate (cloudsync_context *data) { + int sqlstate = cloudsync_sqlstate(data); + return sqlstate ? sqlstate : ERRCODE_INTERNAL_ERROR; +} + PG_FUNCTION_INFO_V1(cloudsync_version); Datum cloudsync_version (PG_FUNCTION_ARGS) { UNUSED_PARAMETER(fcinfo); @@ -258,7 +266,7 @@ Datum cloudsync_db_version (PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cloudsync is not initialized: call SELECT cloudsync_init('') to enable sync on a table before calling cloudsync_db_version()."))); } - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to retrieve db_version (%s)", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("Unable to retrieve db_version (%s)", database_errmsg(data)))); } version = cloudsync_dbversion(data); @@ -305,7 +313,7 @@ Datum cloudsync_db_version_next (PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cloudsync is not initialized: call SELECT cloudsync_init('') to enable sync on a table before calling cloudsync_db_version_next()."))); } - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("Unable to retrieve next_db_version (%s)", database_errmsg(data)))); } } @@ -338,7 +346,7 @@ static bytea *cloudsync_init_internal (cloudsync_context *data, const char *tabl // Begin savepoint for transactional init int rc = database_begin_savepoint(data, "cloudsync_init"); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to create cloudsync_init savepoint: %s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("Unable to create cloudsync_init savepoint: %s", database_errmsg(data)))); } // Initialize table for sync @@ -348,7 +356,7 @@ static bytea *cloudsync_init_internal (cloudsync_context *data, const char *tabl if (rc == DBRES_OK) { rc = database_commit_savepoint(data, "cloudsync_init"); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_init savepoint: %s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("Unable to release cloudsync_init savepoint: %s", database_errmsg(data)))); } // Persist schema to settings now that the settings table exists @@ -510,7 +518,7 @@ Datum pg_cloudsync_cleanup (PG_FUNCTION_ARGS) { } if (spi_connected) SPI_finish(); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } PG_RETURN_BOOL(true); @@ -671,7 +679,7 @@ Datum cloudsync_set_column (PG_FUNCTION_ARGS) { if (key && value && strcmp(key, "algo") == 0 && strcmp(value, "block") == 0) { int rc = cloudsync_setup_block_column(data, tbl, col, NULL, true); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } } else { // Handle delimiter setting: cloudsync_set_column('tbl', 'col', 'delimiter', '\n\n') @@ -884,7 +892,7 @@ Datum pg_cloudsync_begin_alter (PG_FUNCTION_ARGS) { SPI_finish(); if (rc != DBRES_OK) { ereport(ERROR, - (errcode(ERRCODE_INTERNAL_ERROR), + (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } PG_RETURN_BOOL(true); @@ -931,13 +939,13 @@ Datum pg_cloudsync_commit_alter (PG_FUNCTION_ARGS) { if (rc != DBRES_OK) { // Rollback savepoint (SPI disconnected, no warning) database_rollback_savepoint(data, "cloudsync_alter"); - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } // Release savepoint (SPI disconnected, no warning) rc = database_commit_savepoint(data, "cloudsync_alter"); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Unable to release cloudsync_alter savepoint: %s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("Unable to release cloudsync_alter savepoint: %s", database_errmsg(data)))); } // Phase 2: reconnect SPI for post-commit work @@ -989,7 +997,7 @@ Datum cloudsync_payload_encode_transfn (PG_FUNCTION_ARGS) { if (argc > 0) { int rc = cloudsync_payload_encode_step(payload, data, argc, (dbvalue_t **)argv); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } } @@ -1014,7 +1022,7 @@ Datum cloudsync_payload_encode_finalfn (PG_FUNCTION_ARGS) { int rc = cloudsync_payload_encode_final(payload, data); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } int64_t blob_size = 0; @@ -1191,9 +1199,9 @@ static bytea *payload_chunks_emit_pg_fragment(PayloadChunksState *st, cloudsync_ VARDATA_ANY(st->site_id), VARSIZE_ANY_EXHDR(st->site_id), st->cl, st->seq, st->frag_checksum, st->frag_total, st->frag_part, st->frag_count); - if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + if (rc != DBRES_OK) ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); rc = cloudsync_payload_encode_final(payload, data); - if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + if (rc != DBRES_OK) ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); int64 blob_size = 0; char *blob = cloudsync_payload_blob(payload, &blob_size, rows); bytea *result = (bytea *)palloc(VARHDRSZ + blob_size); @@ -1289,7 +1297,7 @@ static bytea *payload_chunks_build_pg_next(PayloadChunksState *st, cloudsync_con payload_chunks_make_pgvalues(st, vals, owned_texts); int rc = cloudsync_payload_encode_step(payload, data, 9, (dbvalue_t **)vals); payload_chunks_free_pgvalues(vals, owned_texts); - if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + if (rc != DBRES_OK) ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); if (cloudsync_payload_context_nrows(payload) == 1) *dbv_min = st->db_version; *dbv_max = st->db_version; @@ -1301,7 +1309,7 @@ static bytea *payload_chunks_build_pg_next(PayloadChunksState *st, cloudsync_con return NULL; } int rc = cloudsync_payload_encode_final(payload, data); - if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + if (rc != DBRES_OK) ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); int64 blob_size = 0; char *blob = cloudsync_payload_blob(payload, &blob_size, rows); bytea *result = (bytea *)palloc(VARHDRSZ + blob_size); @@ -1650,7 +1658,7 @@ Datum cloudsync_payload_blob_checked(PG_FUNCTION_ARGS) { int rc = cloudsync_payload_encode_step(payload, data, 9, (dbvalue_t **)vals); payload_chunks_free_pgvalues(vals, owned_texts); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } payload_chunks_free_current(&encode_st); } @@ -1659,7 +1667,7 @@ Datum cloudsync_payload_blob_checked(PG_FUNCTION_ARGS) { encode_st.portal = NULL; int rc = cloudsync_payload_encode_final(payload, data); - if (rc != DBRES_OK) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + if (rc != DBRES_OK) ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); int64 blob_size = 0; char *blob = cloudsync_payload_blob(payload, &blob_size, NULL); if (spi_connected) { @@ -1741,7 +1749,7 @@ Datum cloudsync_payload_decode (PG_FUNCTION_ARGS) { if (spi_connected) SPI_finish(); if (rc != DBRES_OK) { if (payload_data) pfree(payload_data); - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } if (payload_data) pfree(payload_data); PG_RETURN_INT32(nrows); @@ -2135,7 +2143,7 @@ Datum cloudsync_insert (PG_FUNCTION_ARGS) { } if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", database_errmsg(data)))); } } PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); @@ -2214,7 +2222,7 @@ Datum cloudsync_delete (PG_FUNCTION_ARGS) { } if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", database_errmsg(data)))); } } PG_END_ENSURE_ERROR_CLEANUP(cloudsync_pg_cleanup, PointerGetDatum(&cleanup)); @@ -2463,7 +2471,7 @@ Datum cloudsync_update_finalfn (PG_FUNCTION_ARGS) { if (spi_connected) SPI_finish(); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", database_errmsg(data)))); } PG_RETURN_BOOL(true); @@ -2876,7 +2884,7 @@ Datum cloudsync_col_value(PG_FUNCTION_ARGS) { } databasevm_reset(vm); - ereport(ERROR, (errmsg("cloudsync_col_value error: %s", cloudsync_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("cloudsync_col_value error: %s", cloudsync_errmsg(data)))); PG_RETURN_NULL(); // unreachable, silences compiler } @@ -2937,7 +2945,7 @@ Datum cloudsync_text_materialize (PG_FUNCTION_ARGS) { int rc = block_materialize_column(data, table, cleanup.pk, (int)pklen, col_name); if (rc != DBRES_OK) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), errmsg("%s", cloudsync_errmsg(data)))); } } @@ -3467,8 +3475,16 @@ Datum cloudsync_changes_insert_trigger (PG_FUNCTION_ARGS) { } else { rc = merge_insert (data, table, VARDATA_ANY(insert_pk), insert_pk_len, insert_cl, insert_name, col_value, insert_col_version, insert_db_version, VARDATA_ANY(insert_site_id), insert_site_id_len, insert_seq, &rowid); } + if (rc == DBRES_POLICY_DENIED) { + // Keep a row-level security denial recognizable to the caller that inserted + // into cloudsync_changes (cloudsync_payload_apply): it is skipped and + // counted, unlike other merge failures. + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("Error during merge_insert: %s", database_errmsg(data)))); + } if (rc != DBRES_OK) { - ereport(ERROR, (errmsg("Error during merge_insert: %s", database_errmsg(data)))); + ereport(ERROR, (errcode(cloudsync_error_sqlstate(data)), + errmsg("Error during merge_insert: %s", database_errmsg(data)))); } pgvalue_free(col_value); diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c index d81fa3bf..436f17da 100644 --- a/src/postgresql/database_postgresql.c +++ b/src/postgresql/database_postgresql.c @@ -30,6 +30,7 @@ #include "utils/datum.h" #include "utils/lsyscache.h" #include "utils/memutils.h" +#include "utils/resowner.h" #include "utils/snapmgr.h" #include "pgvalue.h" @@ -75,6 +76,7 @@ typedef struct { Datum *values; char *nulls; bool executed_nonselect; // non-select executed already + uint64 changes; // rows processed by the last non-select execution // Memory MemoryContext stmt_mcxt; // lifetime = pg_stmt_t @@ -560,6 +562,39 @@ char *sql_build_insert_missing_pks_query(const char *schema, const char *table_n // MARK: - HELPER FUNCTIONS - +// Map a PostgreSQL SQLSTATE to DBRES. Only the distinction between a transient failure +// (worth retrying later) and a failure of the data itself matters to callers: a payload +// apply skips a change whose write fails, but must never skip one that failed only +// because of a lock, a deadlock, a cancel or a resource shortage. +static int map_sqlerrcode (int sqlerrcode) { + switch (sqlerrcode) { + case ERRCODE_INSUFFICIENT_PRIVILEGE: // a policy denial is recognized by the caller + return DBRES_PERM; + case ERRCODE_READ_ONLY_SQL_TRANSACTION: + return DBRES_READONLY; + case ERRCODE_LOCK_NOT_AVAILABLE: + case ERRCODE_OBJECT_IN_USE: + return DBRES_BUSY; + case ERRCODE_QUERY_CANCELED: + return DBRES_INTERRUPT; + case ERRCODE_OUT_OF_MEMORY: + return DBRES_NOMEM; + case ERRCODE_DISK_FULL: + return DBRES_FULL; + } + switch (ERRCODE_TO_CATEGORY(sqlerrcode)) { + case ERRCODE_TRANSACTION_ROLLBACK: // 40: serialization failure, deadlock + case ERRCODE_OPERATOR_INTERVENTION: // 57: cancel, shutdown + case ERRCODE_CONNECTION_EXCEPTION: // 08 + return DBRES_BUSY; + case ERRCODE_INSUFFICIENT_RESOURCES: // 53 + return DBRES_NOMEM; + case ERRCODE_SYSTEM_ERROR: // 58: I/O + return DBRES_IOERR; + } + return DBRES_ERROR; +} + // Map SPI result codes to DBRES static int map_spi_result (int rc) { switch (rc) { @@ -855,7 +890,8 @@ static bool database_system_exists (cloudsync_context *data, const char *name, c { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - cloudsync_set_error(data, edata->message, DBRES_ERROR); + cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); exists = false; @@ -889,7 +925,8 @@ int database_exec (cloudsync_context *data, const char *sql) { { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); if (SPI_tuptable) { @@ -925,7 +962,8 @@ int database_exec_callback (cloudsync_context *data, const char *sql, int (*call { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); is_error = true; @@ -1144,6 +1182,10 @@ const char *database_errmsg (cloudsync_context *data) { return cloudsync_errmsg(data); } +void database_log_warning (cloudsync_context *data, const char *message) { + ereport(WARNING, (errmsg("cloudsync: %s", message ? message : ""))); +} + bool database_in_transaction (cloudsync_context *data) { // In SPI context, we're always in a transaction return IsTransactionState(); @@ -2083,7 +2125,8 @@ int databasevm_prepare (cloudsync_context *data, const char *sql, dbvm_t **vm, i { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); if (stmt->stmt_mcxt) MemoryContextDelete(stmt->stmt_mcxt); @@ -2133,7 +2176,8 @@ int databasevm_step0 (pg_stmt_t *stmt) { // Switch to safe context for CopyErrorData (can't be ErrorContext) MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); @@ -2286,6 +2330,7 @@ int databasevm_step (dbvm_t *vm) { rc = cloudsync_set_error(data, "SPI_execute_plan failed", DBRES_ERROR); break; } + stmt->changes = SPI_processed; if (SPI_tuptable) { SPI_freetuptable(SPI_tuptable); SPI_tuptable = NULL; @@ -2304,10 +2349,16 @@ int databasevm_step (dbvm_t *vm) { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); // PostgreSQL uses 42501 for both missing privileges and RLS. Only the - // executor's WITH CHECK policy rejection is safe to skip during merge. - bool policy_denied = edata->sqlerrcode == ERRCODE_INSUFFICIENT_PRIVILEGE && - edata->funcname && strcmp(edata->funcname, "ExecWithCheckOptions") == 0; - int err = cloudsync_set_error(data, edata->message, policy_denied ? DBRES_POLICY_DENIED : DBRES_ERROR); + // executor's WITH CHECK policy rejection is safe to skip during merge — raised + // directly, or re-raised by the cloudsync_changes trigger around the merge. + // The trigger re-raises every merge error with its original SQLSTATE, so a 42501 + // from it is a policy denial only when the merge itself reported one. + bool policy_denied = edata->sqlerrcode == ERRCODE_INSUFFICIENT_PRIVILEGE && edata->funcname && + (strcmp(edata->funcname, "ExecWithCheckOptions") == 0 || + (strcmp(edata->funcname, "cloudsync_changes_insert_trigger") == 0 && + cloudsync_errcode(data) == DBRES_POLICY_DENIED)); + int err = cloudsync_set_error(data, edata->message, policy_denied ? DBRES_POLICY_DENIED : map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); @@ -2360,6 +2411,7 @@ void databasevm_reset (dbvm_t *vm) { // Reset execution state stmt->executed_nonselect = false; + stmt->changes = 0; // Reset parameter values but keep the plan, types, and nparams intact. // The prepared plan can be reused with new values of the same types, @@ -2389,6 +2441,10 @@ void databasevm_clear_bindings (dbvm_t *vm) { } } +int64_t databasevm_changes (dbvm_t *vm) { + return vm ? (int64_t)((pg_stmt_t *)vm)->changes : 0; +} + const char *databasevm_sql (dbvm_t *vm) { if (!vm) return NULL; @@ -2978,20 +3034,65 @@ static int database_refresh_snapshot (void) { return DBRES_OK; } +// BeginInternalSubTransaction leaves CurrentResourceOwner and CurrentMemoryContext +// pointing at the subtransaction, and releasing or rolling it back leaves them at the +// parent *transaction's* owner and context — not at the ones the caller was running +// with. Any resource the calling statement acquires or releases afterwards (a buffer +// pin of the scan feeding cloudsync_payload_apply, say) is then charged to the wrong +// owner: "buffer pin ... is not owned by resource owner TopTransaction". So, as the +// procedural languages do, remember both at the start of each subtransaction and +// restore them when it ends. Indexed by nesting level so an owner never leaks across +// a subtransaction that was aborted elsewhere. +#define CLOUDSYNC_SAVEPOINT_MAX_DEPTH 128 +static ResourceOwner savepoint_owner[CLOUDSYNC_SAVEPOINT_MAX_DEPTH]; +static MemoryContext savepoint_context[CLOUDSYNC_SAVEPOINT_MAX_DEPTH]; + +// Only the outermost savepoint opened here may swap the active snapshot when it ends: a +// snapshot pushed while an enclosing savepoint is still open belongs to that +// subtransaction, and rolling the enclosing one back pops it — in place of the caller's +// snapshot the swap popped, leaving the caller's portal without one (an assertion +// failure in EnsurePortalSnapshotExists). Nested, advancing the command counter is +// enough to make the changes visible: every SPI statement takes a fresh snapshot. +// True when the subtransaction at level is nested inside another savepoint opened here. +static bool savepoint_is_nested (int level) { + for (int k = 2; k < level && k < CLOUDSYNC_SAVEPOINT_MAX_DEPTH; k++) { + if (savepoint_owner[k]) return true; + } + return false; +} + +static void savepoint_restore_caller (int level) { + if (level <= 0 || level >= CLOUDSYNC_SAVEPOINT_MAX_DEPTH || !savepoint_owner[level]) return; + MemoryContextSwitchTo(savepoint_context[level]); + CurrentResourceOwner = savepoint_owner[level]; + savepoint_owner[level] = NULL; + savepoint_context[level] = NULL; +} + int database_begin_savepoint (cloudsync_context *data, const char *savepoint_name) { cloudsync_reset_error(data); int rc = DBRES_OK; MemoryContext oldcontext = CurrentMemoryContext; + ResourceOwner oldowner = CurrentResourceOwner; PG_TRY(); { BeginInternalSubTransaction(NULL); + int level = GetCurrentTransactionNestLevel(); + if (level > 0 && level < CLOUDSYNC_SAVEPOINT_MAX_DEPTH) { + savepoint_owner[level] = oldowner; + savepoint_context[level] = oldcontext; + } + // Keep allocating in the caller's context; the subtransaction's resource owner + // stays current so what the savepoint acquires is released with it. + MemoryContextSwitchTo(oldcontext); } PG_CATCH(); { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - rc = cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); } @@ -3006,19 +3107,23 @@ int database_commit_savepoint (cloudsync_context *data, const char *savepoint_na int rc = DBRES_OK; MemoryContext oldcontext = CurrentMemoryContext; + int level = GetCurrentTransactionNestLevel(); PG_TRY(); { ReleaseCurrentSubTransaction(); - database_refresh_snapshot(); + bool nested = savepoint_is_nested(level); + savepoint_restore_caller(level); + if (nested) CommandCounterIncrement(); + else database_refresh_snapshot(); } PG_CATCH(); { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); - rc = DBRES_ERROR; } PG_END_TRY(); @@ -3031,19 +3136,23 @@ int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_ int rc = DBRES_OK; MemoryContext oldcontext = CurrentMemoryContext; + int level = GetCurrentTransactionNestLevel(); PG_TRY(); { RollbackAndReleaseCurrentSubTransaction(); - database_refresh_snapshot(); + bool nested = savepoint_is_nested(level); + savepoint_restore_caller(level); + if (nested) CommandCounterIncrement(); + else database_refresh_snapshot(); } PG_CATCH(); { MemoryContextSwitchTo(oldcontext); ErrorData *edata = CopyErrorData(); - cloudsync_set_error(data, edata->message, DBRES_ERROR); + rc = cloudsync_set_error(data, edata->message, map_sqlerrcode(edata->sqlerrcode)); + cloudsync_set_sqlstate(data, edata->sqlerrcode); FreeErrorData(edata); FlushErrorState(); - rc = DBRES_ERROR; } PG_END_TRY(); diff --git a/src/sqlite/cloudsync_sqlite.c b/src/sqlite/cloudsync_sqlite.c index aa31761e..25cdd7c3 100644 --- a/src/sqlite/cloudsync_sqlite.c +++ b/src/sqlite/cloudsync_sqlite.c @@ -47,6 +47,17 @@ typedef struct { int capacity; } cloudsync_update_payload; +// Reports a failed tracking write from the insert/update/delete triggers: cloudsync's +// own message when it set one (it names the table and column and carries the database +// error), else SQLite's, and the real result code, so a caller can still tell SQLITE_BUSY +// from a constraint violation. The context error is reset on entry to each trigger, so +// the message is never a stale one. +static void dbsync_result_trigger_error (sqlite3_context *context, cloudsync_context *data, int rc) { + const char *message = cloudsync_errmsg(data); + sqlite3_result_error(context, (message && message[0]) ? message : database_errmsg(data), -1); + sqlite3_result_error_code(context, (rc > 0) ? rc : SQLITE_ERROR); +} + void dbsync_set_error (sqlite3_context *context, const char *format, ...) { char buffer[2048]; @@ -428,6 +439,7 @@ void dbsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { // retrieve context cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_reset_error(data); // lookup table const char *table_name = (const char *)database_value_text(argv[0]); @@ -481,7 +493,7 @@ void dbsync_insert (sqlite3_context *context, int argc, sqlite3_value **argv) { } cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + if (rc != SQLITE_OK) dbsync_result_trigger_error(context, data, rc); // free memory if the primary key was dynamically allocated if (pk != buffer) cloudsync_memory_free(pk); } @@ -492,6 +504,7 @@ void dbsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { // retrieve context cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_reset_error(data); // lookup table const char *table_name = (const char *)database_value_text(argv[0]); @@ -528,7 +541,7 @@ void dbsync_delete (sqlite3_context *context, int argc, sqlite3_value **argv) { if (rc != SQLITE_OK) goto cleanup; cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + if (rc != SQLITE_OK) dbsync_result_trigger_error(context, data, rc); // free memory if the primary key was dynamically allocated if (pk != buffer) cloudsync_memory_free(pk); } @@ -618,6 +631,7 @@ void dbsync_update_final (sqlite3_context *context) { // retrieve context cloudsync_context *data = (cloudsync_context *)sqlite3_user_data(context); + cloudsync_reset_error(data); // lookup table const char *table_name = (const char *)database_value_text(payload->table_name); @@ -712,7 +726,7 @@ void dbsync_update_final (sqlite3_context *context) { } cleanup: - if (rc != SQLITE_OK) sqlite3_result_error(context, database_errmsg(data), -1); + if (rc != SQLITE_OK) dbsync_result_trigger_error(context, data, rc); if (pk != buffer) cloudsync_memory_free(pk); if (oldpk && (oldpk != buffer2)) cloudsync_memory_free(oldpk); @@ -1836,6 +1850,7 @@ void dbsync_text_materialize (sqlite3_context *context, int argc, sqlite3_value int rc = block_materialize_column(data, table, pk, (int)pklen, col_name); if (rc != DBRES_OK) { sqlite3_result_error(context, cloudsync_errmsg(data), -1); + sqlite3_result_error_code(context, rc); } else { sqlite3_result_int(context, 1); } diff --git a/src/sqlite/database_sqlite.c b/src/sqlite/database_sqlite.c index b7864bbd..c0f1c3d9 100644 --- a/src/sqlite/database_sqlite.c +++ b/src/sqlite/database_sqlite.c @@ -588,6 +588,10 @@ int database_errcode (cloudsync_context *data) { return sqlite3_errcode((sqlite3 *)cloudsync_db(data)); } +void database_log_warning (cloudsync_context *data, const char *message) { + sqlite3_log(SQLITE_WARNING, "cloudsync: %s", message ? message : ""); +} + bool database_in_transaction (cloudsync_context *data) { sqlite3 *db = (sqlite3 *)cloudsync_db(data); bool in_transaction = (sqlite3_get_autocommit(db) != true); @@ -1147,6 +1151,10 @@ void databasevm_clear_bindings (dbvm_t *vm) { sqlite3_clear_bindings((sqlite3_stmt *)vm); } +int64_t databasevm_changes (dbvm_t *vm) { + return (int64_t)sqlite3_changes64(sqlite3_db_handle((sqlite3_stmt *)vm)); +} + const char *databasevm_sql (dbvm_t *vm) { return sqlite3_sql((sqlite3_stmt *)vm); // the following allocates memory that needs to be freed diff --git a/test/network_unit.c b/test/network_unit.c index d1e60fa8..2758b1a2 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -198,6 +198,22 @@ static bool test_json_envelope(void) { cloudsync_memory_free(url); return ok; } +extern char *network_test_receive_json(int, int, const char *, bool, const char *, const char *); +static bool receive_json_is(char *json, const char *expected) { + bool ok = json && strcmp(json, expected) == 0; + if (!ok) printf("\n got: %s\n expected: %s\n", json ? json : "(null)", expected); + cloudsync_memory_free(json); + return ok; +} +static bool test_receive_json(void) { + bool ok = receive_json_is(network_test_receive_json(3, 0, NULL, true, NULL, NULL), + "\"receive\":{\"rows\":3,\"failed\":0,\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":true}"); + ok = receive_json_is(network_test_receive_json(2, 1, "rejected \"here\"", true, NULL, NULL), + "\"receive\":{\"rows\":2,\"failed\":1,\"failedError\":\"rejected \\\"here\\\"\",\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":true}") && ok; + ok = receive_json_is(network_test_receive_json(0, 0, NULL, false, "boom", "{\"code\":\"x\"}"), + "\"receive\":{\"rows\":0,\"failed\":0,\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":false,\"error\":\"boom\",\"lastFailure\":{\"code\":\"x\"}}") && ok; + return ok; +} static bool test_unicode(void) { char *s = network_test_unescape("caf\\u00e9 \\u20ac \\ud83d\\ude80 \\/\\n"); bool ok = s && strcmp(s, "caf\xc3\xa9 \xe2\x82\xac \xf0\x9f\x9a\x80 /\n") == 0; @@ -244,6 +260,7 @@ int main(void) { check("JSON keys only match root object members:", test_json_scope()); check("Gateway data envelope is unwrapped before scoped lookups:", test_json_envelope()); check("JSON Unicode, surrogate pairs and malformed escapes:", test_unicode()); + check("receive JSON members (failed, failedError, error, lastFailure):", test_receive_json()); printf("\nNetwork unit tests\n"); check("optimistic/confirmed version folds latest-valid (allows rollback):", test_optimistic_version_rollback()); check("non-buffer response is a no-op:", test_non_buffer_is_noop()); diff --git a/test/postgresql/39_concurrent_write_apply.sql b/test/postgresql/39_concurrent_write_apply.sql index a9c850ec..8b59cbde 100644 --- a/test/postgresql/39_concurrent_write_apply.sql +++ b/test/postgresql/39_concurrent_write_apply.sql @@ -76,6 +76,7 @@ SELECT dblink_exec('locker', 'LOCK TABLE concurrent_tbl IN EXCLUSIVE MODE') AS _ \if :{?_lock} -- ===== Lock acquired — run lock-contention test ===== +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before_blocked \gset BEGIN; \set ON_ERROR_ROLLBACK on @@ -102,6 +103,17 @@ SELECT (:'row1_val_check' = 'val_a') AS blocked_ok \gset SELECT (:fail::int + 1) AS fail \gset \endif +-- A lock timeout is transient: the change must not be skipped, so the receive +-- cursor stays where it was and the next apply retries it. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after_blocked \gset +SELECT (:ckpt_after_blocked::bigint = :ckpt_before_blocked::bigint) AS blocked_ckpt_ok \gset +\if :blocked_ckpt_ok +\echo [PASS] (:testid) Lock-blocked apply left the receive checkpoint in place +\else +\echo [FAIL] (:testid) Lock-blocked apply moved the checkpoint from :ckpt_before_blocked to :ckpt_after_blocked +SELECT (:fail::int + 1) AS fail \gset +\endif + -- Release the table lock SELECT dblink_exec('locker', 'COMMIT') AS _release \gset SELECT dblink_disconnect('locker') AS _disconn \gset diff --git a/test/postgresql/57_audit_regressions.sql b/test/postgresql/57_audit_regressions.sql index 93c4f3f1..7c9c9cc2 100644 --- a/test/postgresql/57_audit_regressions.sql +++ b/test/postgresql/57_audit_regressions.sql @@ -1,4 +1,6 @@ --- Audit: database errors must not be mistaken for skippable RLS denials. +-- Audit: a change whose write fails on its data is skipped and reported, never +-- silently dropped and never allowed to stall the cursor; transient failures are +-- covered by 39_concurrent_write_apply.sql. \set ON_ERROR_STOP on \connect postgres DROP DATABASE IF EXISTS cloudsync_audit_source; @@ -11,6 +13,16 @@ CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); SELECT cloudsync_init('t'); INSERT INTO t VALUES ('1','a'),('2','b'),('3','c'); SELECT encode(cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq),'hex') AS audit_payload FROM cloudsync_changes \gset +CREATE TABLE plain_audit(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('plain_audit') \gset +INSERT INTO plain_audit VALUES ('p1','x'); +SELECT encode(cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq),'hex') AS plain_payload_hex FROM cloudsync_changes WHERE tbl = 'plain_audit' \gset +CREATE TABLE revived(id TEXT PRIMARY KEY NOT NULL, a TEXT, b TEXT); +SELECT cloudsync_init('revived') \gset +INSERT INTO revived VALUES ('r1','x','y'); +DELETE FROM revived WHERE id = 'r1'; +INSERT INTO revived VALUES ('r1','x2','y2'); +SELECT encode(cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq),'hex') AS revived_payload_hex FROM cloudsync_changes WHERE tbl = 'revived' \gset \connect cloudsync_audit_target CREATE EXTENSION cloudsync; @@ -26,28 +38,114 @@ BEGIN RETURN NEW; END $$; CREATE TRIGGER deny_audit BEFORE INSERT ON t FOR EACH ROW EXECUTE FUNCTION deny_audit_row(); +SET client_min_messages = error; -- each skipped change raises a WARNING by design +-- The payload is read from a table on purpose: the apply's internal savepoints must +-- not disturb the resource owner of the scan feeding it. DO $$ -DECLARE denied INTEGER; failed BOOLEAN; +DECLARE denied INTEGER; BEGIN FOR denied IN 1..3 LOOP + DELETE FROM t; + DELETE FROM t_cloudsync; + DELETE FROM cloudsync_settings WHERE key IN ('check_dbversion','check_seq'); PERFORM set_config('audit.denied_id', denied::TEXT, false); - failed := false; - BEGIN - PERFORM cloudsync_payload_apply(data) FROM audit_payload; - EXCEPTION WHEN OTHERS THEN - IF SQLERRM NOT LIKE '%audit write denied%' THEN RAISE; END IF; - failed := true; - END; - IF NOT failed THEN RAISE EXCEPTION 'Payload silently ignored error at row %', denied; END IF; - IF EXISTS (SELECT FROM t) THEN RAISE EXCEPTION 'Failed payload committed partial data'; END IF; - IF coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) <> 0 THEN - RAISE EXCEPTION 'Failed payload advanced its checkpoint'; + PERFORM cloudsync_payload_apply(data) FROM audit_payload; + IF (SELECT count(*) FROM t) <> 2 THEN + RAISE EXCEPTION 'Row % failing must not discard the other rows (found %)', denied, (SELECT count(*) FROM t); + END IF; + IF EXISTS (SELECT FROM t WHERE id = denied::TEXT) THEN + RAISE EXCEPTION 'Failed row % was written', denied; + END IF; + IF coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) = 0 THEN + RAISE EXCEPTION 'A skipped failure at row % must still advance the checkpoint', denied; END IF; END LOOP; END $$; -\echo [PASS] (57-audit) first, middle and final merge errors are propagated without checkpoint advancement +SET client_min_messages = warning; +\echo [PASS] (57-audit) first, middle and final write failures are skipped without discarding the rest or stalling the checkpoint + +CREATE FUNCTION raise_sqlstate() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN RAISE EXCEPTION USING ERRCODE = TG_ARGV[0], MESSAGE = 'simulated ' || TG_ARGV[0]; END $$; +CREATE TABLE plain_audit(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('plain_audit') \gset +CREATE TABLE revived(id TEXT PRIMARY KEY NOT NULL, a TEXT, b TEXT); +SELECT cloudsync_init('revived') \gset + +-- A payload apply keeps the SQLSTATE too, and does not skip these failures: a serialization +-- failure is transient and a missing privilege is fixed by a GRANT, so both fail the +-- apply and leave the checkpoint where it was. +CREATE TRIGGER fail_apply BEFORE INSERT ON plain_audit FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('40001'); +CREATE TEMP TABLE plain_payload(data BYTEA); +INSERT INTO plain_payload VALUES (decode(:'plain_payload_hex','hex')); +DO $$ +DECLARE state TEXT; msg TEXT; code TEXT; +BEGIN + FOREACH code IN ARRAY ARRAY['40001', '42501'] LOOP + EXECUTE 'DROP TRIGGER fail_apply ON plain_audit'; + EXECUTE format('CREATE TRIGGER fail_apply BEFORE INSERT ON plain_audit FOR EACH ROW EXECUTE FUNCTION raise_sqlstate(%L)', code); + DELETE FROM cloudsync_settings WHERE key IN ('check_dbversion','check_seq'); + state := NULL; + BEGIN PERFORM cloudsync_payload_apply(data) FROM plain_payload; + EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE, msg = MESSAGE_TEXT; END; + IF state IS DISTINCT FROM code THEN + RAISE EXCEPTION 'Apply failing with % surfaced as SQLSTATE % (%)', code, coalesce(state, 'none: the failure was skipped'), msg; + END IF; + IF EXISTS (SELECT FROM plain_audit) OR coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) <> 0 THEN + RAISE EXCEPTION 'Apply failing with % wrote rows or moved the checkpoint', code; + END IF; + END LOOP; +END $$; +DROP TRIGGER fail_apply ON plain_audit; +\echo [PASS] (57-audit) apply keeps the SQLSTATE of transient and privilege failures and does not skip them + +-- A resurrected row (sentinel plus columns) whose write fails leaves nothing behind — +-- no sentinel, no zeroed clocks — so once the cause is gone the same payload creates it. +CREATE TRIGGER fail_revived BEFORE INSERT ON revived FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('23514'); +CREATE TEMP TABLE revived_payload(data BYTEA); +INSERT INTO revived_payload VALUES (decode(:'revived_payload_hex','hex')); +SET client_min_messages = error; +SELECT cloudsync_payload_apply(data) FROM revived_payload \gset +SET client_min_messages = warning; +DROP TRIGGER fail_revived ON revived; +DO $$ BEGIN + IF EXISTS (SELECT FROM revived) THEN RAISE EXCEPTION 'Rejected resurrected row was written'; END IF; + IF EXISTS (SELECT FROM revived_cloudsync) THEN RAISE EXCEPTION 'Rejected resurrected row left metadata behind'; END IF; +END $$; +SELECT cloudsync_payload_apply(data) FROM revived_payload \gset +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM revived WHERE id = 'r1' AND a = 'x2' AND b = 'y2') THEN + RAISE EXCEPTION 'Resurrected row was not created once its write could succeed'; + END IF; +END $$; +\echo [PASS] (57-audit) a failed resurrected row leaves no metadata and applies when delivered again + +-- Savepoints opened by the apply must leave the caller's snapshots intact, including +-- when a group is rolled back and when the caller's own subtransaction is then aborted: +-- statements after the apply keep running (an unbalanced active-snapshot stack trips an +-- assertion in EnsurePortalSnapshotExists and takes the backend down). +DELETE FROM revived; DELETE FROM revived_cloudsync; +CREATE TRIGGER fail_revived BEFORE INSERT ON revived FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('23514'); +SET client_min_messages = error; +DO $$ +DECLARE n INTEGER; +BEGIN + BEGIN + PERFORM cloudsync_payload_apply(data) FROM revived_payload; + PERFORM count(*) FROM revived; + RAISE EXCEPTION 'abort the caller subtransaction'; + EXCEPTION WHEN raise_exception THEN NULL; + END; + SELECT count(*) INTO n FROM revived_payload; + PERFORM cloudsync_payload_apply(data) FROM revived_payload; + SELECT count(*) INTO n FROM revived; + IF n <> 0 THEN RAISE EXCEPTION 'Rejected resurrected row was written'; END IF; +END $$; +SET client_min_messages = warning; +DROP TRIGGER fail_revived ON revived; +\echo [PASS] (57-audit) apply savepoints leave the snapshots of the caller intact across rollbacks DROP TRIGGER deny_audit ON t; +DELETE FROM t; -- the block cases below start from an empty table SELECT cloudsync_set_column('t','value','algo','block'); CREATE FUNCTION deny_audit_block() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'audit block write denied'; END $$; @@ -107,6 +205,47 @@ DROP POLICY t_ins ON t; ALTER TABLE t DISABLE ROW LEVEL SECURITY; \echo [PASS] (57-audit) an unreadable block row reports which table and column, not a blank error +-- Materializing a block column names the stage, column and table of a failed write +-- and keeps its SQLSTATE, also when called directly rather than from a merge. +CREATE FUNCTION reject_value() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN RAISE EXCEPTION USING ERRCODE = '23514', MESSAGE = 'value rejected'; END $$; +CREATE TRIGGER reject_value BEFORE UPDATE ON t FOR EACH ROW EXECUTE FUNCTION reject_value(); +DO $$ +DECLARE state TEXT; msg TEXT; +BEGIN + BEGIN PERFORM cloudsync_text_materialize('t', 'value', 'block'); + EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE, msg = MESSAGE_TEXT; END; + IF state IS DISTINCT FROM '23514' THEN RAISE EXCEPTION 'Materialize write failure surfaced as SQLSTATE % (%)', state, msg; END IF; + IF msg NOT LIKE 'Unable to write the blocks of column "value" of table "t"%' OR msg NOT LIKE '%value rejected%' THEN + RAISE EXCEPTION 'Materialize write failure message lacks stage, context or cause: %', msg; + END IF; +END $$; +DROP TRIGGER reject_value ON t; +\echo [PASS] (57-audit) a failed block materialization names the stage, column and table and keeps its SQLSTATE + +-- A failure inside cloudsync's tracking triggers keeps its SQLSTATE, so an application +-- can still retry a serialization failure or handle a unique violation, and names the +-- table and column it was writing. +CREATE TRIGGER fail_block BEFORE INSERT ON t_cloudsync_blocks FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('40001'); +CREATE TRIGGER fail_meta BEFORE INSERT ON plain_audit_cloudsync FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('23505'); +DO $$ +DECLARE state TEXT; msg TEXT; +BEGIN + BEGIN INSERT INTO t VALUES('sqlstate','text'); + EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE, msg = MESSAGE_TEXT; END; + IF state IS DISTINCT FROM '40001' THEN RAISE EXCEPTION 'Block write failure surfaced as SQLSTATE % (%), expected 40001', state, msg; END IF; + IF msg NOT LIKE '%column "value" of table "t"%' OR msg NOT LIKE '%simulated 40001%' THEN + RAISE EXCEPTION 'Block write failure message lacks context or cause: %', msg; + END IF; + state := NULL; + BEGIN INSERT INTO plain_audit VALUES('sqlstate','text'); + EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE, msg = MESSAGE_TEXT; END; + IF state IS DISTINCT FROM '23505' THEN RAISE EXCEPTION 'Metadata write failure surfaced as SQLSTATE % (%), expected 23505', state, msg; END IF; +END $$; +DROP TRIGGER fail_block ON t_cloudsync_blocks; +DROP TRIGGER fail_meta ON plain_audit_cloudsync; +\echo [PASS] (57-audit) tracking-trigger failures keep their SQLSTATE and name the column and table + \connect postgres DROP DATABASE cloudsync_audit_source; DROP DATABASE cloudsync_audit_target; diff --git a/test/postgresql/59_rls_denial_retry.sql b/test/postgresql/59_rls_denial_retry.sql new file mode 100644 index 00000000..0d931106 --- /dev/null +++ b/test/postgresql/59_rls_denial_retry.sql @@ -0,0 +1,245 @@ +-- Row-level security denials outside the batched column path, and denials that depend +-- on the order of the payload. +-- +-- 1. A denial raised inside the cloudsync_changes trigger (a block column, a GOS table) +-- is skipped like one from the batched path: the rest of the payload applies and the +-- receive checkpoint advances, instead of the whole apply failing. +-- 2. A row denied only because a row granting access arrives later in the same payload +-- is retried once the payload is in, and is applied. +-- 3. A row still denied after the retry stays skipped. +-- Along the way, block columns and GOS tables must write every column of a permitted row +-- although their policies reference another column. + +\set testid '59-rls-retry' +\ir helper_test_init.sql + +\set USER1 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' +\set USER2 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb' + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_59_src; +DROP DATABASE IF EXISTS cloudsync_test_59_dst; +CREATE DATABASE cloudsync_test_59_src; +CREATE DATABASE cloudsync_test_59_dst; +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'rls_retry_user') THEN + CREATE ROLE rls_retry_user LOGIN; + END IF; +END $$; + +-- ------------------------------------------------------------------ source +\connect cloudsync_test_59_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, user_id UUID, body TEXT); +CREATE TABLE events (id TEXT PRIMARY KEY NOT NULL, user_id UUID, kind TEXT); +CREATE TABLE members (project_id TEXT NOT NULL, user_id UUID NOT NULL, PRIMARY KEY (project_id, user_id)); +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, project_id TEXT, title TEXT); +CREATE TABLE activity (id TEXT PRIMARY KEY NOT NULL, project_id TEXT, kind TEXT); +CREATE TABLE logs (id TEXT PRIMARY KEY NOT NULL, user_id UUID, msg TEXT); +SELECT cloudsync_init('notes') AS _i1 \gset +SELECT cloudsync_init('events', 'gos') AS _i2 \gset +SELECT cloudsync_init('members') AS _i3 \gset +SELECT cloudsync_init('tasks') AS _i4 \gset +SELECT cloudsync_init('activity', 'gos') AS _i5 \gset +SELECT cloudsync_init('logs', 'gos') AS _i6 \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _b \gset + +INSERT INTO notes VALUES ('n_own', :'USER1', E'line 1\nline 2'); +INSERT INTO notes VALUES ('n_other', :'USER2', E'secret 1\nsecret 2'); +INSERT INTO events VALUES ('e_own', :'USER1', 'login'); +INSERT INTO events VALUES ('e_other', :'USER2', 'login'); +-- The task is written BEFORE the membership that authorizes it, so it comes first +-- in the payload. +INSERT INTO tasks VALUES ('t_p1', 'p1', 'task in p1'); +-- Same dependency on a GOS table, whose merge runs inside the cloudsync_changes trigger: +-- only a denial recognized as such is retried. +INSERT INTO activity VALUES ('a_p1', 'p1', 'created'); +-- A resurrected task (sentinel plus columns) in the same project, also authorized only +-- by the membership that follows. +INSERT INTO tasks VALUES ('t_rev', 'p1', 'first'); +DELETE FROM tasks WHERE id = 't_rev'; +INSERT INTO tasks VALUES ('t_rev', 'p1', 'revived'); +INSERT INTO members VALUES ('p1', :'USER1'); +INSERT INTO logs VALUES ('l_own', :'USER1', 'hello'); +-- A task in a project USER1 never joins: denied for good. +INSERT INTO tasks VALUES ('t_p2', 'p2', 'task in p2'); + +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset + +-- ------------------------------------------------------------------ target +\connect cloudsync_test_59_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE notes (id TEXT PRIMARY KEY NOT NULL, user_id UUID, body TEXT); +CREATE TABLE events (id TEXT PRIMARY KEY NOT NULL, user_id UUID, kind TEXT); +CREATE TABLE members (project_id TEXT NOT NULL, user_id UUID NOT NULL, PRIMARY KEY (project_id, user_id)); +CREATE TABLE tasks (id TEXT PRIMARY KEY NOT NULL, project_id TEXT, title TEXT); +CREATE TABLE activity (id TEXT PRIMARY KEY NOT NULL, project_id TEXT, kind TEXT); +CREATE TABLE logs (id TEXT PRIMARY KEY NOT NULL, user_id UUID, msg TEXT); +SELECT cloudsync_init('notes') AS _i1 \gset +SELECT cloudsync_init('events', 'gos') AS _i2 \gset +SELECT cloudsync_init('members') AS _i3 \gset +SELECT cloudsync_init('tasks') AS _i4 \gset +SELECT cloudsync_init('activity', 'gos') AS _i5 \gset +SELECT cloudsync_init('logs', 'gos') AS _i6 \gset +SELECT cloudsync_set_column('notes', 'body', 'algo', 'block') AS _b \gset + +CREATE SCHEMA IF NOT EXISTS auth; +CREATE OR REPLACE FUNCTION auth.uid() RETURNS UUID LANGUAGE sql STABLE +AS $$ SELECT NULLIF(current_setting('app.current_user_id', true), '')::UUID $$; + +DO $$ DECLARE t TEXT; BEGIN + FOREACH t IN ARRAY ARRAY['notes', 'events', 'members'] LOOP + EXECUTE format('ALTER TABLE %I ENABLE ROW LEVEL SECURITY', t); + EXECUTE format('CREATE POLICY own_select ON %I FOR SELECT USING (auth.uid() = user_id)', t); + EXECUTE format('CREATE POLICY own_insert ON %I FOR INSERT WITH CHECK (auth.uid() = user_id)', t); + EXECUTE format('CREATE POLICY own_update ON %I FOR UPDATE USING (auth.uid() = user_id) WITH CHECK (auth.uid() = user_id)', t); + END LOOP; +END $$; +-- logs can be inserted and read but has no UPDATE policy: a GOS table writes its +-- second column as an update, which such a policy silently turns into a no-op. +ALTER TABLE logs ENABLE ROW LEVEL SECURITY; +CREATE POLICY own_select ON logs FOR SELECT USING (auth.uid() = user_id); +CREATE POLICY own_insert ON logs FOR INSERT WITH CHECK (auth.uid() = user_id); +ALTER TABLE tasks ENABLE ROW LEVEL SECURITY; +CREATE POLICY member_select ON tasks FOR SELECT + USING (EXISTS (SELECT 1 FROM members m WHERE m.project_id = tasks.project_id AND m.user_id = auth.uid())); +CREATE POLICY member_insert ON tasks FOR INSERT + WITH CHECK (EXISTS (SELECT 1 FROM members m WHERE m.project_id = tasks.project_id AND m.user_id = auth.uid())); +ALTER TABLE activity ENABLE ROW LEVEL SECURITY; +CREATE POLICY member_select ON activity FOR SELECT + USING (EXISTS (SELECT 1 FROM members m WHERE m.project_id = activity.project_id AND m.user_id = auth.uid())); +CREATE POLICY member_insert ON activity FOR INSERT + WITH CHECK (EXISTS (SELECT 1 FROM members m WHERE m.project_id = activity.project_id AND m.user_id = auth.uid())); +CREATE POLICY member_update ON activity FOR UPDATE + USING (EXISTS (SELECT 1 FROM members m WHERE m.project_id = activity.project_id AND m.user_id = auth.uid())) + WITH CHECK (EXISTS (SELECT 1 FROM members m WHERE m.project_id = activity.project_id AND m.user_id = auth.uid())); +CREATE POLICY member_update ON tasks FOR UPDATE + USING (EXISTS (SELECT 1 FROM members m WHERE m.project_id = tasks.project_id AND m.user_id = auth.uid())) + WITH CHECK (EXISTS (SELECT 1 FROM members m WHERE m.project_id = tasks.project_id AND m.user_id = auth.uid())); + +GRANT USAGE ON SCHEMA public, auth TO rls_retry_user; +GRANT ALL ON ALL TABLES IN SCHEMA public TO rls_retry_user; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO rls_retry_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public, auth TO rls_retry_user; + +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset + +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +\unset apply_rows +SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_rows \gset +\set ON_ERROR_STOP on +RESET ROLE; + +\if :{?apply_rows} +\echo [PASS] (:testid) apply with trigger-path and order-dependent denials completed +\else +\echo [FAIL] (:testid) apply failed instead of skipping the denied rows +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 1. Trigger-path denials are skipped, the permitted rows are applied. +SELECT (SELECT count(*) FROM notes WHERE id = 'n_own') = 1 + AND (SELECT count(*) FROM notes WHERE id = 'n_other') = 0 AS notes_ok \gset +\if :notes_ok +\echo [PASS] (:testid) block column: own row applied, other user row skipped +\else +\echo [FAIL] (:testid) block column: unexpected rows in notes +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (SELECT body FROM notes WHERE id = 'n_own') = E'line 1\nline 2' AS notes_body_ok \gset +\if :notes_body_ok +\echo [PASS] (:testid) block column: own row materialized in full +\else +\echo [FAIL] (:testid) block column: own row body not materialized +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (SELECT count(*) FROM notes_cloudsync_blocks WHERE pk = cloudsync_pk_encode('n_other')) = 0 AS notes_blocks_ok \gset +\if :notes_blocks_ok +\echo [PASS] (:testid) block column: a denied row leaves no blocks behind +\else +\echo [FAIL] (:testid) block column: denied row left blocks behind +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (SELECT count(*) FROM events WHERE id = 'e_own' AND kind = 'login') = 1 + AND (SELECT count(*) FROM events WHERE id = 'e_other') = 0 AS events_ok \gset +\if :events_ok +\echo [PASS] (:testid) GOS table: own row applied with every column, other user row skipped +\else +\echo [FAIL] (:testid) GOS table: unexpected rows in events +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 2. The task denied on the first pass is applied once its membership is in. +SELECT (SELECT count(*) FROM members WHERE project_id = 'p1') = 1 AS member_ok \gset +SELECT (SELECT title FROM tasks WHERE id = 't_p1') IS NOT DISTINCT FROM 'task in p1' AS retry_ok \gset +\if :member_ok +\if :retry_ok +\echo [PASS] (:testid) order-dependent denial: task applied after its membership on retry +\else +\echo [FAIL] (:testid) order-dependent denial: task authorized later in the payload was lost +SELECT (:fail::int + 1) AS fail \gset +\endif +\else +\echo [FAIL] (:testid) membership row was not applied +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (SELECT kind FROM activity WHERE id = 'a_p1') IS NOT DISTINCT FROM 'created' AS retry_gos_ok \gset +\if :retry_gos_ok +\echo [PASS] (:testid) order-dependent denial in the trigger path (GOS): applied on retry +\else +\echo [FAIL] (:testid) order-dependent denial in the trigger path (GOS): row was lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- A column the policy does not let this session write is reported as denied, never +-- recorded as applied while the table does not hold it. +SELECT NOT EXISTS (SELECT FROM logs_cloudsync WHERE col_name = 'msg') + AND (SELECT msg FROM logs WHERE id = 'l_own') IS NULL AS no_silent_loss_ok \gset +\if :no_silent_loss_ok +\echo [PASS] (:testid) a column hidden from UPDATE is not recorded as applied +\else +\echo [FAIL] (:testid) a column the table does not hold was recorded as applied +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (SELECT title FROM tasks WHERE id = 't_rev') IS NOT DISTINCT FROM 'revived' AS retry_revived_ok \gset +\if :retry_revived_ok +\echo [PASS] (:testid) order-dependent denial of a resurrected row: applied on retry +\else +\echo [FAIL] (:testid) order-dependent denial of a resurrected row: row was lost +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 3. A row still denied after the retry stays out, and the cursor moves on. +SELECT (SELECT count(*) FROM tasks WHERE id = 't_p2') = 0 AS still_denied_ok \gset +\if :still_denied_ok +\echo [PASS] (:testid) a row still denied after the retry is skipped +\else +\echo [FAIL] (:testid) a permanently denied row was applied +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:ckpt_after::bigint > :ckpt_before::bigint) AS ckpt_ok \gset +\if :ckpt_ok +\echo [PASS] (:testid) receive checkpoint advanced past the skipped rows +\else +\echo [FAIL] (:testid) receive checkpoint stayed at :ckpt_after +SELECT (:fail::int + 1) AS fail \gset +\endif + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_59_src; +DROP DATABASE IF EXISTS cloudsync_test_59_dst; +DROP ROLE IF EXISTS rls_retry_user; diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index f4e9a97d..4b9c2058 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -66,6 +66,7 @@ \ir 56_many_columns.sql \ir 57_audit_regressions.sql \ir 58_v3_denied_checkpoint.sql +\ir 59_rls_denial_retry.sql -- 'Test summary' \echo '\nTest summary:' diff --git a/test/review_regressions.c b/test/review_regressions.c index af40466e..bc2fae5c 100644 --- a/test/review_regressions.c +++ b/test/review_regressions.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "sqlite3.h" #include "cloudsync.h" #include "cloudsync_sqlite.h" @@ -62,7 +63,30 @@ static void test_best_index(void) { CHECK(strcmp(info.idxStr, " ORDER BY db_version, seq ASC") == 0); sqlite3_free(info.idxStr); } +static int skipped_warnings; +static int skipped_changes; // sum of N over "skipped N received change(s) that failed to apply" +static void log_callback(void *arg, int code, const char *message) { + (void)arg; + if (code == SQLITE_WARNING && message && strstr(message, "failed to apply")) { + skipped_warnings++; + const char *n = strstr(message, "skipped "); + if (n) skipped_changes += atoi(n + 8); + } +} +static int apply_payload(sqlite3 *source, sqlite3 *target) { + sqlite3_stmt *read = NULL, *write = NULL; + CHECK(sqlite3_prepare_v2(source, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes", -1, &read, NULL) == SQLITE_OK); + CHECK(sqlite3_step(read) == SQLITE_ROW); + CHECK(sqlite3_prepare_v2(target, "SELECT cloudsync_payload_decode(?1)", -1, &write, NULL) == SQLITE_OK); + CHECK(sqlite3_bind_value(write, 1, sqlite3_column_value(read, 0)) == SQLITE_OK); + int rc = sqlite3_step(write); + sqlite3_finalize(write); + sqlite3_finalize(read); + return rc; +} static void test_payload_errors(void) { + // A write that fails on its data fails the same way on every retry: it is skipped + // and reported, and the cursor still advances past it. for (int denied = 1; denied <= 3; denied++) { sqlite3 *source = open_db(), *target = open_db(); const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL,value TEXT); SELECT cloudsync_init('t');"; @@ -71,20 +95,53 @@ static void test_payload_errors(void) { char trigger[256]; snprintf(trigger, sizeof(trigger), "CREATE TRIGGER deny BEFORE INSERT ON t WHEN NEW.id='%d' BEGIN SELECT RAISE(ABORT,'denied'); END", denied); CHECK(sql(target, trigger) == SQLITE_OK); - sqlite3_stmt *read = NULL, *write = NULL; - CHECK(sqlite3_prepare_v2(source, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes", -1, &read, NULL) == SQLITE_OK); - CHECK(sqlite3_step(read) == SQLITE_ROW); - CHECK(sqlite3_prepare_v2(target, "SELECT cloudsync_payload_decode(?1)", -1, &write, NULL) == SQLITE_OK); - CHECK(sqlite3_bind_value(write, 1, sqlite3_column_value(read, 0)) == SQLITE_OK); - CHECK(sqlite3_step(write) != SQLITE_ROW); - CHECK(strstr(sqlite3_errmsg(target), "denied") != NULL); - sqlite3_finalize(write); - sqlite3_finalize(read); + skipped_warnings = 0; + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(skipped_warnings == 1); + CHECK(scalar(target, "SELECT count(*) FROM t") == 2); + char query[128]; + snprintf(query, sizeof(query), "SELECT count(*) FROM t WHERE id='%d'", denied); + CHECK(scalar(target, query) == 0); + CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") > 0); + CHECK(sqlite3_get_autocommit(target)); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); + } + + // A transient failure (the database is locked by another connection) could succeed + // on a retry, so it must fail the apply and leave the cursor in place. + { + char path[512]; + const char *dir = getenv("TMPDIR"); + unsigned int nonce = 0; + sqlite3_randomness(sizeof(nonce), &nonce); + snprintf(path, sizeof(path), "%s/cloudsync-rr-busy-%08x.db", (dir && *dir) ? dir : ".", nonce); + sqlite3 *source = open_db(), *target = NULL, *locker = NULL; + CHECK(sqlite3_open(path, &target) == SQLITE_OK); + CHECK(sqlite3_cloudsync_init(target, NULL, NULL) == SQLITE_OK); + const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL,value TEXT); SELECT cloudsync_init('t');"; + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('1','a'),('2','b');") == SQLITE_OK); + CHECK(sqlite3_open(path, &locker) == SQLITE_OK); + CHECK(sql(locker, "BEGIN IMMEDIATE") == SQLITE_OK); + skipped_warnings = 0; + CHECK(apply_payload(source, target) != SQLITE_ROW); + CHECK(skipped_warnings == 0); + CHECK(sql(locker, "ROLLBACK") == SQLITE_OK); + CHECK(sqlite3_close(locker) == SQLITE_OK); CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") == 0); CHECK(sqlite3_get_autocommit(target)); + // Once the lock is gone the same payload applies in full. + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(scalar(target, "SELECT count(*) FROM t") == 2); CHECK(close_db(source) == SQLITE_OK); CHECK(close_db(target) == SQLITE_OK); + char aux[600]; + remove(path); + snprintf(aux, sizeof(aux), "%s-journal", path); + remove(aux); } + sqlite3 *db = open_db(); CHECK(sql(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL); SELECT cloudsync_init('t');") == SQLITE_OK); // v1 header: request a 4GB decompression without checksum/schema requirements. @@ -94,6 +151,61 @@ static void test_payload_errors(void) { CHECK(strstr(sqlite3_errmsg(db), "exceeds limit") != NULL); CHECK(close_db(db) == SQLITE_OK); } +static void fail_busy(sqlite3_context *context, int argc, sqlite3_value **argv) { + (void)argc; (void)argv; + sqlite3_result_error(context, "simulated busy", -1); + sqlite3_result_error_code(context, SQLITE_BUSY); +} +static void test_resurrected_group_rollback(void) { + const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, a TEXT, b TEXT); SELECT cloudsync_init('t');"; + + // A resurrected row arrives as a sentinel plus its columns. When its write fails the + // whole group is skipped and counted (3 changes, not 2), and nothing it wrote remains: + // delivered again once the cause is gone, the row is created. + for (int transient = 0; transient < 2; transient++) { + sqlite3 *source = open_db(), *target = open_db(); + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('r1','x','y'); DELETE FROM t WHERE id='r1'; INSERT INTO t VALUES('r1','x2','y2');") == SQLITE_OK); + CHECK(sqlite3_create_function(target, "fail_busy", 0, SQLITE_UTF8, NULL, fail_busy, NULL, NULL) == SQLITE_OK); + CHECK(sql(target, transient + ? "CREATE TRIGGER rej BEFORE INSERT ON t WHEN NEW.id='r1' BEGIN SELECT fail_busy(); END" + : "CREATE TRIGGER rej BEFORE INSERT ON t WHEN NEW.id='r1' BEGIN SELECT RAISE(ABORT,'rejected r1'); END") == SQLITE_OK); + skipped_warnings = 0; skipped_changes = 0; + int rc = apply_payload(source, target); + if (transient) { + CHECK(rc != SQLITE_ROW); // transient: the apply fails and is retried + CHECK(skipped_warnings == 0); + } else { + CHECK(rc == SQLITE_ROW); // data failure: skipped and reported + CHECK(skipped_warnings == 1); + CHECK(skipped_changes == 3); + } + CHECK(scalar(target, "SELECT count(*) FROM t_cloudsync") == 0); + CHECK(sql(target, "DROP TRIGGER rej") == SQLITE_OK); + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(scalar(target, "SELECT count(*) FROM t WHERE id='r1' AND a='x2' AND b='y2'") == 1); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); + } + + // A row the target already holds keeps its clocks when resurrecting it fails: the + // zeroed clocks and the new sentinel are rolled back with the failed write. + sqlite3 *source = open_db(), *target = open_db(); + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('r1','x','y')") == SQLITE_OK); + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(sql(source, "DELETE FROM t WHERE id='r1'; INSERT INTO t VALUES('r1','x2','y2');") == SQLITE_OK); + CHECK(sql(target, "CREATE TRIGGER rej BEFORE UPDATE ON t BEGIN SELECT RAISE(ABORT,'rejected update'); END;" + "CREATE TRIGGER rej2 BEFORE INSERT ON t BEGIN SELECT RAISE(ABORT,'rejected insert'); END;") == SQLITE_OK); + CHECK(sql(target, "CREATE TEMP TABLE before_clocks AS SELECT col_name, col_version FROM t_cloudsync") == SQLITE_OK); + skipped_warnings = 0; + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(skipped_warnings >= 1); // the resurrection was attempted and failed + CHECK(scalar(target, "SELECT count(*) FROM (SELECT col_name, col_version FROM t_cloudsync EXCEPT SELECT col_name, col_version FROM before_clocks)") == 0); + CHECK(scalar(target, "SELECT count(*) FROM t WHERE id='r1' AND a='x' AND b='y'") == 1); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); +} static void test_block_write_errors(void) { for (int update = 0; update < 2; update++) { sqlite3 *db = open_db(); @@ -101,10 +213,32 @@ static void test_block_write_errors(void) { if (update) CHECK(sql(db, "INSERT INTO docs VALUES('1','old')") == SQLITE_OK); CHECK(sql(db, "CREATE TRIGGER deny_block BEFORE INSERT ON docs_cloudsync_blocks BEGIN SELECT RAISE(ABORT,'block write denied'); END") == SQLITE_OK); CHECK(sql(db, update ? "UPDATE docs SET body='new' WHERE id='1'" : "INSERT INTO docs VALUES('1','new')") != SQLITE_OK); + // the failure names the column and table, keeps the database's own message, and + // keeps its result code (RAISE(ABORT) is a constraint failure, not SQLITE_ERROR) + CHECK(strstr(sqlite3_errmsg(db), "column \"body\" of table \"docs\"") != NULL); + CHECK(strstr(sqlite3_errmsg(db), "block write denied") != NULL); + CHECK((sqlite3_errcode(db) & 0xFF) == SQLITE_CONSTRAINT); CHECK(scalar(db, update ? "SELECT count(*) FROM docs WHERE body='old'" : "SELECT count(*) FROM docs") == update); CHECK(close_db(db) == SQLITE_OK); } } +static void test_block_not_null_payload(void) { + // A received block is materialized into a row whose other columns arrive in the same + // payload; a constraint on one of them (here a trigger requiring an owner, as a NOT + // NULL column or an RLS policy would) must not reject the block write. + sqlite3 *source = open_db(), *target = open_db(); + const char *schema = "CREATE TABLE docs(id TEXT PRIMARY KEY NOT NULL, owner TEXT NOT NULL DEFAULT 'x', body TEXT);" + "SELECT cloudsync_init('docs'); SELECT cloudsync_set_column('docs','body','algo','block');"; + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(target, "CREATE TRIGGER no_null_owner BEFORE INSERT ON docs WHEN NEW.owner = 'x' BEGIN SELECT RAISE(ABORT,'owner required'); END") == SQLITE_OK); + CHECK(sql(source, "INSERT INTO docs VALUES('1','alice','line 1' || char(10) || 'line 2')") == SQLITE_OK); + skipped_warnings = 0; + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(skipped_warnings == 0); + CHECK(scalar(target, "SELECT count(*) FROM docs WHERE id='1' AND owner='alice' AND body='line 1' || char(10) || 'line 2'") == 1); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); +} static void test_refill_error(void) { sqlite3 *db = open_db(); CHECK(sql(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL,a TEXT,b TEXT); SELECT cloudsync_init('t'); INSERT INTO t VALUES('1','a','b'),('2','a','b'); DELETE FROM t_cloudsync WHERE col_name='a';") == SQLITE_OK); @@ -119,9 +253,10 @@ static void test_refill_error(void) { static sqlite3_mem_methods memory; static int fail_after = -1; +static bool fail_once = false; // fail only the selected allocation, not every one after it static bool fail_alloc(void) { if (fail_after < 0) return false; - if (fail_after == 0) return true; + if (fail_after == 0) { if (fail_once) fail_after = -1; return true; } fail_after--; return false; } @@ -163,17 +298,55 @@ static void test_block_oom(void) { CHECK(succeeded); } } +static void test_block_materialize_errors(void) { + sqlite3 *db = open_db(); + CHECK(sql(db, "CREATE TABLE docs(id TEXT PRIMARY KEY NOT NULL, body TEXT); SELECT cloudsync_init('docs');" + "SELECT cloudsync_set_column('docs','body','algo','block'); INSERT INTO docs VALUES('1','a' || char(10) || 'b');") == SQLITE_OK); + + // A failed write names the stage, column and table, keeps the cause and its code. + CHECK(sql(db, "CREATE TRIGGER deny_body BEFORE UPDATE ON docs BEGIN SELECT RAISE(ABORT,'body rejected'); END") == SQLITE_OK); + CHECK(sql(db, "SELECT cloudsync_text_materialize('docs','body','1')") != SQLITE_OK); + CHECK(strstr(sqlite3_errmsg(db), "Unable to write the blocks of column \"body\" of table \"docs\"") != NULL); + CHECK(strstr(sqlite3_errmsg(db), "body rejected") != NULL); + CHECK((sqlite3_errcode(db) & 0xFF) == SQLITE_CONSTRAINT); + CHECK(sql(db, "DROP TRIGGER deny_body") == SQLITE_OK); + + // Fail each allocation of the call in turn: the error is never blank, and cloudsync's + // own allocation failures say so instead of posing as a read failure. + int ours = 0, blank = 0; + bool succeeded = false; + for (int n = 0; n < 400 && !succeeded; n++) { + fail_once = true; + fail_after = n; + int rc = sql(db, "SELECT cloudsync_text_materialize('docs','body','1')"); + fail_after = -1; + fail_once = false; + if (rc == SQLITE_OK) { succeeded = true; break; } + const char *msg = sqlite3_errmsg(db); + if (!msg || !msg[0] || strcmp(msg, "not an error") == 0) blank++; + if (msg && strstr(msg, "Not enough memory to") && strstr(msg, "column \"body\" of table \"docs\"")) ours++; + } + CHECK(succeeded); + CHECK(blank == 0); + CHECK(ours > 0); + CHECK(scalar(db, "SELECT body = 'a' || char(10) || 'b' FROM docs WHERE id='1'") == 1); + CHECK(close_db(db) == SQLITE_OK); +} int main(void) { CHECK(sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memory) == SQLITE_OK); sqlite3_mem_methods faults = memory; faults.xMalloc = fault_malloc; faults.xRealloc = fault_realloc; CHECK(sqlite3_config(SQLITE_CONFIG_MALLOC, &faults) == SQLITE_OK); + CHECK(sqlite3_config(SQLITE_CONFIG_LOG, log_callback, NULL) == SQLITE_OK); CHECK(sqlite3_initialize() == SQLITE_OK); test_clocks_and_double(); test_best_index(); test_payload_errors(); + test_resurrected_group_rollback(); test_block_write_errors(); + test_block_materialize_errors(); + test_block_not_null_payload(); test_refill_error(); test_block_oom(); cloudsync_memory_finalize(); From af8e5b09c2f18b140255ca36a3ee2b7841a58230 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:14:20 +0200 Subject: [PATCH 13/28] fix(block): skip rows whose base row is gone when migrating to block columns Review point 2. The migration scans the metadata table, so it can return a pk whose base row no longer exists (deleted while sync was disabled) or is hidden by a SELECT policy. Failing on it made cloudsync_set_column(..., 'algo', 'block') impossible for that table, and on SQLite left algo=block persisted with a half-done migration. Such rows are skipped again, as before the refactor; the row's next local write creates its blocks. Tests: review_regressions and PostgreSQL 57 convert a column with orphaned metadata. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 1 + docs/internal/audit-regressions.md | 3 ++- src/cloudsync.c | 12 +++++++++--- test/postgresql/57_audit_regressions.sql | 22 ++++++++++++++++++++++ test/review_regressions.c | 16 ++++++++++++++++ 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04440a5e..dc04acea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **PostgreSQL: errors raised by cloudsync keep the SQLSTATE of the database error behind them.** A failure inside the tracking triggers or `cloudsync_payload_apply` — a serialization failure (`40001`), a deadlock (`40P01`), a unique violation (`23505`), a missing privilege (`42501`) — was re-raised as `XX000` internal_error, so application retry logic and error handling keyed on SQLSTATE could not react to it. cloudsync's own failures are still `XX000`. - **SQLite: a failed tracking write in the insert, update and delete triggers keeps its result code and context.** It was reported as a bare `SQLITE_ERROR` with SQLite's last message; it now carries the real code (`SQLITE_BUSY`, `SQLITE_CONSTRAINT`, ...) and cloudsync's message. - **Block column failures name the table and column** they were writing or reading, and what failed (reading, collecting or joining the blocks, writing the column), around the underlying database error. No block-column failure reports a blank message any more: allocation failures say so instead of posing as a read failure, and `cloudsync_text_materialize` keeps the error's result code on SQLite. +- **Converting a column to the block algorithm skips rows whose base row no longer exists** (for example deleted while sync was disabled) instead of failing the conversion. - **Primary-key doubles keep their deployed little-endian IEEE754 byte order on every architecture.** The previous code combined host conversion with manual big-endian serialization; the historical format is now explicit. Integer keys are unchanged, and no migration is needed for little-endian deployments. - **PostgreSQL no longer frees a tuple table belonging to another open cursor.** A block write that failed while a second SPI cursor was active could release rows still in use; tuple tables are now owned per statement. - **Block-level LWW text writes roll back cleanly when a block write fails**, instead of leaving the row partially written. diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md index 11b94ef1..2b3d02e1 100644 --- a/docs/internal/audit-regressions.md +++ b/docs/internal/audit-regressions.md @@ -31,6 +31,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. (`databasevm_changes` exposes the affected row count on both backends). - PostgreSQL savepoints restore the caller's resource owner and memory context, so a payload applied from a table scan no longer trips a foreign buffer pin. +- Block-column migration skips tracked rows whose base row cannot be read. - Each PK group of a payload is applied under one savepoint (`cloudsync_merge_group`) that also covers the metadata its rows write before the flush (sentinel, zeroed clocks, block values, winner clocks); a failed flush rolls it all back, so a retried or @@ -65,7 +66,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. | Group atomicity | Resurrected row rejected (data and transient failure): no metadata left, re-delivery creates it, 3 entries counted; existing row keeps its clocks; RLS retry of a resurrected row; apply inside an aborted caller subtransaction | | Payload failures | First, middle and final PK errors skipped with a warning and checkpoint advanced; locked database fails and keeps the checkpoint (SQLite rollback journal, WAL, PostgreSQL lock_timeout); allocation limits | | Metadata refill | Trigger rejects insertion of a missing column clock | -| Block LWW | Insert/update rollback on block write failure; allocation failure at each split/list/diff allocation | +| Block LWW | Insert/update rollback on block write failure; migration past orphaned metadata; allocation failure at each split/list/diff allocation | | PostgreSQL ownership | Block failure while another SPI cursor is active; no invalid tuple-table cleanup | | JSON | Root-only member lookup, string values resembling keys, nested keys, Unicode and invalid surrogates | | Curl | Stalled loopback HTTP server, unpooled handle and pooled handle before/after reset | diff --git a/src/cloudsync.c b/src/cloudsync.c index 3c63689d..fdd0cdbf 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -2374,7 +2374,8 @@ static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table // Reuse the checked block writer; the scan above excludes migrated rows. // As in local_block_insert, every failure carries a message: the migration aborts // on one, and a bare code would surface as a blank error. Aborting is recoverable - // — the Phase 1 scan skips already-migrated rows, so a re-run resumes. + // — the Phase 1 scan skips already-migrated rows, so a re-run resumes. A row that + // cannot be read back is skipped rather than failed (see below). char errmsg[512]; dbvm_t *val_vm = table_column_lookup(table, col_name, false, NULL); if (!val_vm) { @@ -2407,8 +2408,13 @@ static int block_migrate_existing_rows (cloudsync_context *data, cloudsync_table rc = local_block_update(data, table, pks[p], pklens[p], col_idx, copy, db_version, true); cloudsync_memory_free(copy); } else if (rc == DBRES_DONE) { - snprintf(errmsg, sizeof(errmsg), "Unable to read block column \"%s\" of table \"%s\" while migrating: a tracked row is not visible to this connection (check the table's row-level security SELECT policy)", col_name, table->name); - rc = cloudsync_set_error(data, errmsg, DBRES_ERROR); + // The scan reads the metadata table, not the base table, so it can return a + // pk whose row is gone: deleted while sync was disabled, say, or hidden from + // this session by a row-level security SELECT policy. There is no current + // value to split, and failing here would make the column impossible to + // convert. Skip it as the migration always has; if the row reappears, its + // next local write creates the blocks. + rc = DBRES_OK; } databasevm_reset(val_vm); } diff --git a/test/postgresql/57_audit_regressions.sql b/test/postgresql/57_audit_regressions.sql index 7c9c9cc2..1c617ce6 100644 --- a/test/postgresql/57_audit_regressions.sql +++ b/test/postgresql/57_audit_regressions.sql @@ -246,6 +246,28 @@ DROP TRIGGER fail_block ON t_cloudsync_blocks; DROP TRIGGER fail_meta ON plain_audit_cloudsync; \echo [PASS] (57-audit) tracking-trigger failures keep their SQLSTATE and name the column and table +-- Converting a column to block must skip metadata whose base row is gone, rather +-- than fail the conversion. +CREATE TABLE orphan_docs(id TEXT PRIMARY KEY NOT NULL, body TEXT); +SELECT cloudsync_init('orphan_docs') \gset +INSERT INTO orphan_docs VALUES ('a','hello world'), ('b','x y'); +SELECT cloudsync_disable('orphan_docs') \gset +DELETE FROM orphan_docs WHERE id = 'b'; +SELECT cloudsync_enable('orphan_docs') \gset +SELECT cloudsync_set_column('orphan_docs','body','algo','block') \gset +DO $$ BEGIN + IF NOT EXISTS (SELECT FROM cloudsync_table_settings WHERE tbl_name='orphan_docs' AND key='algo' AND value='block') THEN + RAISE EXCEPTION 'Block conversion was not persisted'; + END IF; + IF (SELECT count(*) FROM orphan_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('a')) <> 1 THEN + RAISE EXCEPTION 'Readable row was not migrated'; + END IF; + IF EXISTS (SELECT FROM orphan_docs_cloudsync_blocks WHERE pk = cloudsync_pk_encode('b')) THEN + RAISE EXCEPTION 'Orphan row produced blocks'; + END IF; +END $$; +\echo [PASS] (57-audit) block conversion skips metadata whose base row is gone + \connect postgres DROP DATABASE cloudsync_audit_source; DROP DATABASE cloudsync_audit_target; diff --git a/test/review_regressions.c b/test/review_regressions.c index bc2fae5c..0356b42c 100644 --- a/test/review_regressions.c +++ b/test/review_regressions.c @@ -222,6 +222,21 @@ static void test_block_write_errors(void) { CHECK(close_db(db) == SQLITE_OK); } } +static void test_block_migration_orphan(void) { + // Metadata can outlive its base row (deleted while sync was disabled). Converting + // the column must skip that row, not fail, and still migrate the readable ones. + sqlite3 *db = open_db(); + CHECK(sql(db, "CREATE TABLE docs(id TEXT PRIMARY KEY NOT NULL, body TEXT); SELECT cloudsync_init('docs');" + "INSERT INTO docs VALUES('a','hello world'),('b','x y');" + "SELECT cloudsync_disable('docs'); DELETE FROM docs WHERE id='b'; SELECT cloudsync_enable('docs');") == SQLITE_OK); + CHECK(sql(db, "SELECT cloudsync_set_column('docs','body','algo','block')") == SQLITE_OK); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_table_settings WHERE tbl_name='docs' AND key='algo' AND value='block'") == 1); + CHECK(scalar(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk=cloudsync_pk_encode('a')") == 1); + CHECK(scalar(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk=cloudsync_pk_encode('b')") == 0); + CHECK(sql(db, "UPDATE docs SET body='hello world' || char(10) || 'again' WHERE id='a'") == SQLITE_OK); + CHECK(scalar(db, "SELECT count(*) FROM docs_cloudsync_blocks WHERE pk=cloudsync_pk_encode('a')") == 2); + CHECK(close_db(db) == SQLITE_OK); +} static void test_block_not_null_payload(void) { // A received block is materialized into a row whose other columns arrive in the same // payload; a constraint on one of them (here a trigger requiring an owner, as a NOT @@ -346,6 +361,7 @@ int main(void) { test_resurrected_group_rollback(); test_block_write_errors(); test_block_materialize_errors(); + test_block_migration_orphan(); test_block_not_null_payload(); test_refill_error(); test_block_oom(); From 268011b5898a35d16324f24e0b34982fa9d453dd Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:15:27 +0200 Subject: [PATCH 14/28] fix(payload): bound the declared expanded size by LZ4's ratio, not a fixed 256 MiB cap Review point 6. The fixed cap rejected payloads the library itself produces (cloudsync_payload_encode / cloudsync_payload_save have no such limit), so a large export could no longer be loaded, and it only applied to compressed payloads. The real risk is a few forged header bytes claiming a huge allocation: a compressed payload cannot expand beyond LZ4's 255:1 ratio, so a declared size above compressed_len * 255 + 64 (or above INT_MAX, where LZ4 gets a negative capacity) is rejected up front, and any genuine payload stays loadable. Tests: forged 4 GB and 268 MB headers, the exact 255:1 boundary, and a genuine payload compressing to 254:1. A 300 MiB payload was also verified manually. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 +- docs/internal/audit-regressions.md | 15 ++++++++++----- src/cloudsync.c | 11 ++++++++--- src/cloudsync.h | 6 +++--- src/network/network.c | 6 +++--- test/review_regressions.c | 28 +++++++++++++++++++++++++--- 6 files changed, 50 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc04acea..a4c0e9a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **PostgreSQL: rows rejected by a row-level security policy are retried within the payload, then skipped and reported.** A policy can depend on rows that arrive later in the same payload (a membership that grants access to the rows before it), so denied changes are retried once the rest of the payload is applied, repeating while a retry makes progress. A change still denied is skipped — it would be denied on every delivery — the receive cursor advances past it, and `cloudsync_payload_apply` raises one `WARNING` with the number of skipped changes. This now also covers block columns and grow-only-set tables, whose denial previously failed the whole apply. It does not yet extend to a value large enough to be sent in fragments: a denial there is still an error and the cursor does not advance, and a change authorized only by a later payload is not retried. - **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. -- **Decompressed payloads are capped at 256 MiB before allocation**, overridable with `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE` (LZ4's own `INT_MAX` bound still applies). Default chunk sizes are far below this limit; an oversized legacy monolithic payload must be rechunked or the limit raised explicitly. +- **A compressed payload can no longer make the extension allocate an arbitrary amount of memory.** The decompressed size is read from the payload header and allocated before decompressing, so a few forged bytes could previously claim gigabytes. A declared size larger than LZ4 can actually produce from the compressed bytes (255:1) is now rejected up front, as is one beyond LZ4's own `INT_MAX` limit, where the previous code handed LZ4 a negative capacity. There is no fixed size limit: any payload the library produces, however large, still applies. ### Fixed diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md index 2b3d02e1..8b7fa0b8 100644 --- a/docs/internal/audit-regressions.md +++ b/docs/internal/audit-regressions.md @@ -9,10 +9,15 @@ The previously reported `MAX_PARAMS` issue is outside this change. byte swapping now makes the historical format explicit on every architecture. Integers keep their existing encoding. No migration is needed for supported little-endian deployments. -- Decompressed payloads are limited to 256 MiB before allocation. Builds may - override `CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE`; LZ4's `INT_MAX` bound still - applies. Existing default chunk sizes are below this limit. Oversized legacy - monolithic payloads must be rechunked or used with an explicitly raised limit. +- A compressed payload's declared expanded size must not exceed what its compressed + bytes can decompress to under LZ4's 255:1 maximum ratio (plus a small constant), nor + LZ4's `INT_MAX` API limit. There is no fixed size cap, so large payloads produced by + `cloudsync_payload_encode` / `cloudsync_payload_save` remain loadable; a forged header + can no longer trigger a large allocation. A genuine repetitive 8 MiB value compresses + to 254.3:1, inside the bound. +- Curl requests now have a 30-second connection deadline and a 300-second total + deadline, including reused handles. Build overrides are + `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. - Payload writes that fail on their data (constraint, raising trigger, type error) are skipped, logged as a warning and counted (`receive.failed`), and the receive cursor advances: they fail identically on every retry. Transient @@ -64,7 +69,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. | Block materialization errors | Write failure via cloudsync_text_materialize keeps cause, code/SQLSTATE and names stage, column, table; single-shot allocation failure at every allocation never yields a blank error | | Error origin | SQLSTATE 40001/23505 through the block and metadata triggers; 40001 and a non-policy 42501 through payload apply, not skipped; SQLite trigger failure keeps SQLITE_CONSTRAINT and names column and table | | Group atomicity | Resurrected row rejected (data and transient failure): no metadata left, re-delivery creates it, 3 entries counted; existing row keeps its clocks; RLS retry of a resurrected row; apply inside an aborted caller subtransaction | -| Payload failures | First, middle and final PK errors skipped with a warning and checkpoint advanced; locked database fails and keeps the checkpoint (SQLite rollback journal, WAL, PostgreSQL lock_timeout); allocation limits | +| Payload failures | First, middle and final PK errors skipped with a warning and checkpoint advanced; locked database fails and keeps the checkpoint (SQLite rollback journal, WAL, PostgreSQL lock_timeout); expanded-size bound (forged 4GB/268MB headers, exact 255:1 boundary, genuine 254:1 payload) | | Metadata refill | Trigger rejects insertion of a missing column clock | | Block LWW | Insert/update rollback on block write failure; migration past orphaned metadata; allocation failure at each split/list/diff allocation | | PostgreSQL ownership | Block failure while another SPI cursor is active; no invalid tuple-table cleanup | diff --git a/src/cloudsync.c b/src/cloudsync.c index fdd0cdbf..28370595 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -4456,9 +4456,14 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // check if payload is compressed char *clone = NULL; if (header.expanded_size != 0) { - // Bound untrusted allocation sizes before passing them to LZ4's int API. - if (header.expanded_size > CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE || header.expanded_size > INT_MAX) { - return cloudsync_set_error(data, "Error on cloudsync_payload_apply: expanded payload exceeds limit", DBRES_MISUSE); + // The declared size is untrusted and is allocated before decompressing, so bound + // it by what the compressed bytes can actually expand to: a few forged header bytes + // must not make us allocate gigabytes. A genuinely large payload stays loadable, + // whatever its size. INT_MAX is LZ4's own API limit: past it the cast below turns + // negative and LZ4_decompress_safe does not validate a negative capacity. + if (header.expanded_size > INT_MAX || + (uint64_t)header.expanded_size > (uint64_t)buf_len * CLOUDSYNC_PAYLOAD_LZ4_MAX_RATIO + 64) { + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: declared expanded size is inconsistent with the compressed payload", DBRES_MISUSE); } clone = (char *)cloudsync_memory_alloc(header.expanded_size); if (!clone) return cloudsync_set_error(data, "Unable to allocate memory to uncompress payload", DBRES_NOMEM); diff --git a/src/cloudsync.h b/src/cloudsync.h index f7a8dc22..44ba0ca6 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -19,9 +19,9 @@ extern "C" { #endif #define CLOUDSYNC_VERSION "1.1.4" -#ifndef CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE -#define CLOUDSYNC_MAX_PAYLOAD_EXPANDED_SIZE (256U * 1024U * 1024U) -#endif +// LZ4's block format cannot expand input by more than 255:1, so a compressed payload +// declaring a larger expansion is forged or corrupt (see cloudsync_payload_apply). +#define CLOUDSYNC_PAYLOAD_LZ4_MAX_RATIO 255 #define CLOUDSYNC_MAX_TABLENAME_LEN 512 #define CLOUDSYNC_VALUE_NOTSET -1 diff --git a/src/network/network.c b/src/network/network.c index 184b1ce3..d8fe74e4 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -372,9 +372,9 @@ static bool network_curl_pool_enabled(network_data *data) { } // API calls carry small JSON, so a cap on elapsed time is the right shape for them. -// Artifact transfers are bulk and are bounded on progress instead: 256 MiB inside a -// 300s cap would demand a sustained ~875 KB/s, killing a healthy transfer on a slow -// link. Low-speed also detects a genuine stall sooner than the absolute cap does. +// Artifact transfers are bulk and are bounded on progress instead: a large payload +// inside a 300s cap would demand a sustained transfer rate, killing a healthy transfer +// on a slow link. Low-speed also detects a genuine stall sooner than the absolute cap does. static void network_curl_apply_deadlines(CURL *handle, bool is_api) { curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); diff --git a/test/review_regressions.c b/test/review_regressions.c index 0356b42c..27cf66aa 100644 --- a/test/review_regressions.c +++ b/test/review_regressions.c @@ -144,13 +144,34 @@ static void test_payload_errors(void) { sqlite3 *db = open_db(); CHECK(sql(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL); SELECT cloudsync_init('t');") == SQLITE_OK); - // v1 header: request a 4GB decompression without checksum/schema requirements. + // v1 headers (no checksum or schema requirements) over a 2-byte compressed body. + // Declaring 4GB, or 268MB, from 2 bytes is inconsistent with LZ4's maximum ratio. CHECK(sql(db, "SELECT cloudsync_payload_decode(x'434C535901000000FFFFFFFF00090000000000000000000000000000000000000000')") != SQLITE_OK); - CHECK(strstr(sqlite3_errmsg(db), "exceeds limit") != NULL); + CHECK(strstr(sqlite3_errmsg(db), "inconsistent") != NULL); CHECK(sql(db, "SELECT cloudsync_payload_decode(x'434C5359010000001000000100090000000000000000000000000000000000000000')") != SQLITE_OK); - CHECK(strstr(sqlite3_errmsg(db), "exceeds limit") != NULL); + CHECK(strstr(sqlite3_errmsg(db), "inconsistent") != NULL); + // At the bound (2 * 255 + 64 = 574 bytes) the size is plausible and decompression is + // attempted, failing on the bogus data; one byte past it is rejected up front. + CHECK(sql(db, "SELECT cloudsync_payload_decode(x'434C5359010000000000023E00090000000000000000000000000000000000000000')") != SQLITE_OK); + CHECK(strstr(sqlite3_errmsg(db), "unable to decompress") != NULL); + CHECK(sql(db, "SELECT cloudsync_payload_decode(x'434C5359010000000000023F00090000000000000000000000000000000000000000')") != SQLITE_OK); + CHECK(strstr(sqlite3_errmsg(db), "inconsistent") != NULL); CHECK(close_db(db) == SQLITE_OK); } +static void test_payload_high_compression(void) { + // A genuine payload compresses close to LZ4's maximum ratio when its values repeat; + // the size check must never reject what the library itself produced. + sqlite3 *source = open_db(), *target = open_db(); + const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); SELECT cloudsync_init('t');"; + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('big', replace(hex(zeroblob(4*1024*1024)), '0', 'A'))") == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('small', 'x')") == SQLITE_OK); + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(scalar(target, "SELECT length(value) FROM t WHERE id='big'") == 8 * 1024 * 1024); + CHECK(scalar(target, "SELECT count(*) FROM t") == 2); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); +} static void fail_busy(sqlite3_context *context, int argc, sqlite3_value **argv) { (void)argc; (void)argv; sqlite3_result_error(context, "simulated busy", -1); @@ -358,6 +379,7 @@ int main(void) { test_clocks_and_double(); test_best_index(); test_payload_errors(); + test_payload_high_compression(); test_resurrected_group_rollback(); test_block_write_errors(); test_block_materialize_errors(); From 1ade3e3bb88b0f1aadf43a1c8ac20245577d9eda Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:17:34 +0200 Subject: [PATCH 15/28] test: run the SQLite suites on a real big-endian host (make unittest-s390x) Review point 10. Primary-key and payload encodings are byte-order sensitive, but every suite ran on little-endian hosts only, where the pre-1.1.4 host-dependent double encoding is indistinguishable from the fixed one. The new target builds and runs dist/unit and dist/review_regressions in a linux/s390x Alpine container under QEMU, in separate build directories, and aborts unless the host really is big-endian. With the pre-1.1.4 double encoding restored it fails on s390x on the golden bytes while still passing on macOS. No migration is added for big-endian data written by earlier versions: no such deployment exists. Co-Authored-By: Claude Opus 5 --- Makefile | 16 ++++++++++++++++ docker/s390x/Dockerfile | 4 ++++ docs/internal/audit-regressions.md | 10 +++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 docker/s390x/Dockerfile diff --git a/Makefile b/Makefile index 0860d307..003e94a7 100644 --- a/Makefile +++ b/Makefile @@ -314,6 +314,22 @@ $(DIST_DIR)/review_regressions_big_endian$(EXE): $(TEST_OBJ) $(BUILD_TEST)/pk_fo endian-unittest: $(DIST_DIR)/review_regressions_big_endian$(EXE) @./$(DIST_DIR)/review_regressions_big_endian$(EXE) +# Run the SQLite unit and regression suites on a real big-endian host (s390x) under QEMU +# emulation. The payload and primary-key encodings are byte-order sensitive; this is the +# only build that executes them on big-endian hardware semantics. Needs Docker with +# linux/s390x emulation. Objects go to separate directories so the host build is untouched. +S390X_IMAGE ?= sqlite-sync-s390x-test +.PHONY: unittest-s390x +unittest-s390x: + docker build --platform linux/s390x -t $(S390X_IMAGE) docker/s390x + docker run --rm --platform linux/s390x -v "$(CURDIR)":/src -w /src $(S390X_IMAGE) sh -ec '\ + test "$$(uname -m)" = s390x; \ + test "$$(printf "\001\000" | od -An -tu2 | tr -d " ")" = 256; \ + echo "Host: $$(uname -m), big-endian"; \ + make BUILD_TEST=build/s390x/test DIST_DIR=dist/s390x dist/s390x/unit dist/s390x/review_regressions; \ + ./dist/s390x/unit; \ + ./dist/s390x/review_regressions' + # Network-enabled unit test binary. Link it via a file rule (like dist/unit), not in # the run recipe below: on Android `make test` runs binaries on the emulator from a # script generated by `make test -n`, so a link command inside the recipe would be diff --git a/docker/s390x/Dockerfile b/docker/s390x/Dockerfile new file mode 100644 index 00000000..02b5aa67 --- /dev/null +++ b/docker/s390x/Dockerfile @@ -0,0 +1,4 @@ +# Minimal big-endian toolchain for `make unittest-s390x`. Runs under QEMU emulation +# (Docker Desktop, or docker/setup-qemu-action on CI); the sources are mounted, not copied. +FROM alpine:3.20 +RUN apk add --no-cache build-base diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md index 8b7fa0b8..9311f067 100644 --- a/docs/internal/audit-regressions.md +++ b/docs/internal/audit-regressions.md @@ -4,7 +4,11 @@ The previously reported `MAX_PARAMS` issue is outside this change. ## Compatibility and limits -- PK doubles retain their deployed little-endian IEEE754 bytes. The old code +- PK doubles retain their deployed little-endian IEEE754 bytes. `make unittest-s390x` + runs the SQLite suites on s390x (Docker + QEMU): the golden-byte checks fail there with + the pre-1.1.4 host-dependent encoding, which a little-endian host cannot detect. + No migration is provided for data written by earlier versions on big-endian hosts: + no such deployment exists. The old code combined host conversion with manual big-endian serialization; unconditional byte swapping now makes the historical format explicit on every architecture. Integers keep their existing encoding. No migration is needed for supported @@ -63,7 +67,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. | Area | Focused coverage | | --- | --- | | 64-bit clocks | Incoming column/database versions, causal length and sequence above UINT32_MAX | -| Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; forced big-endian conversion build | +| Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; forced big-endian conversion build; the unit and regression suites on a real big-endian host (`make unittest-s390x`, s390x under QEMU) | | Virtual-table planner | Unusable/unsupported constraints before an accepted constraint; no accepted constraints | | RLS denials | Block-column and GOS denials skipped; every column of a permitted block/GOS row written; a column hidden from UPDATE not recorded as applied; order-dependent denials applied on retry in both the batched and trigger paths; permanently denied rows skipped with checkpoint advanced | | Block materialization errors | Write failure via cloudsync_text_materialize keeps cause, code/SQLSTATE and names stage, column, table; single-shot allocation failure at every allocation never yields a blank error | @@ -79,7 +83,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. | Fractional indexing | 4096-byte common prefix plus the existing module suite | | Test harness | Temporary directory cleanup, memory accounting and sanitizer-safe RowID generation | -Run `make unittest`, `make endian-unittest`, `make network-unittest`, and +Run `make unittest`, `make unittest-s390x`, `make endian-unittest`, `make network-unittest`, and `make -C modules/fractional-indexing/test run`. The network test needs permission to bind a loopback socket; it does not contact an external service. Run Node checks from `packages/node` with `npm test -- --run`, `npm run typecheck`, and From c4a2b0797a33f2f8ed09e5977152bd47786fb381 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:18:40 +0200 Subject: [PATCH 16/28] refactor: remove code left unused by the block write refactor Review point 11. Centralizing block writes in local_block_insert/local_block_update left SQL_BLOCKS_INSERT_IGNORE, SQL_META_INSERT_BLOCK_IGNORE, block_initial_positions() and table_block_list_stmt() without callers. Their names suggested the migration was idempotent through INSERT OR IGNORE, which is no longer how it works. Co-Authored-By: Claude Opus 5 --- src/block.c | 5 ----- src/block.h | 6 +----- src/cloudsync.c | 1 - src/cloudsync.h | 1 - src/postgresql/sql_postgresql.c | 7 ------- src/sql.h | 2 -- src/sqlite/sql_sqlite.c | 7 ------- 7 files changed, 1 insertion(+), 28 deletions(-) diff --git a/src/block.c b/src/block.c index 7087b46d..f7a2ed10 100644 --- a/src/block.c +++ b/src/block.c @@ -162,11 +162,6 @@ char *block_position_between(const char *before, const char *after) { return generate_key_between(before, after); } -char **block_initial_positions(int count) { - if (count <= 0) return NULL; - return generate_n_keys_between(NULL, NULL, count); -} - // MARK: - Block diff - static block_diff_t *block_diff_create(void) { diff --git a/src/block.h b/src/block.h index fa43369b..fe29663d 100644 --- a/src/block.h +++ b/src/block.h @@ -61,7 +61,7 @@ typedef struct { } block_diff_t; // Initialize the fractional-indexing library to use cloudsync's allocator. -// Must be called once before any block_position_between / block_initial_positions calls. +// Must be called once before any block_position_between call. void block_init_allocator(void); // Check if a col_name is a block entry (contains BLOCK_SEPARATOR) @@ -85,10 +85,6 @@ block_list_t *block_split(const char *text, const char *delimiter); // Free a block list void block_list_free(block_list_t *list); -// Generate fractional index position IDs for N initial blocks -// Returns array of N strings (caller must free each + the array) -char **block_initial_positions(int count); - // Generate a position ID that sorts between 'before' and 'after' // Either can be NULL (meaning beginning/end of sequence) // Caller must free the result diff --git a/src/cloudsync.c b/src/cloudsync.c index 28370595..f948554f 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -2149,7 +2149,6 @@ const char *table_col_delimiter (cloudsync_table_context *table, int index) { // Block column struct accessors (for use outside cloudsync.c where struct is opaque) dbvm_t *table_block_value_read_stmt (cloudsync_table_context *table) { return table ? table->block_value_read_stmt : NULL; } dbvm_t *table_block_value_write_stmt (cloudsync_table_context *table) { return table ? table->block_value_write_stmt : NULL; } -dbvm_t *table_block_list_stmt (cloudsync_table_context *table) { return table ? table->block_list_stmt : NULL; } const char *table_blocks_ref (cloudsync_table_context *table) { return table ? table->blocks_ref : NULL; } void table_set_col_delimiter (cloudsync_table_context *table, int col_idx, const char *delimiter) { diff --git a/src/cloudsync.h b/src/cloudsync.h index 44ba0ca6..f69ffa16 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -216,7 +216,6 @@ int cloudsync_setup_block_column (cloudsync_context *data, const char *table_nam // Block column accessors (avoids accessing opaque struct from outside cloudsync.c) dbvm_t *table_block_value_read_stmt (cloudsync_table_context *table); dbvm_t *table_block_value_write_stmt (cloudsync_table_context *table); -dbvm_t *table_block_list_stmt (cloudsync_table_context *table); const char *table_blocks_ref (cloudsync_table_context *table); void table_set_col_delimiter (cloudsync_table_context *table, int col_idx, const char *delimiter); diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c index 9106b1b1..2f2dd433 100644 --- a/src/postgresql/sql_postgresql.c +++ b/src/postgresql/sql_postgresql.c @@ -475,14 +475,7 @@ const char * const SQL_BLOCKS_LIST_ALIVE = "AND m.pk = $3 AND m.col_name LIKE $4 AND m.col_version %% 2 = 1 " "ORDER BY b.col_name COLLATE \"C\""; -const char * const SQL_BLOCKS_INSERT_IGNORE = - "INSERT INTO %s (pk, col_name, col_value) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING"; - const char * const SQL_META_SCAN_COL_FOR_MIGRATION = "SELECT DISTINCT m.pk FROM %s m " "WHERE m.col_name = $1 AND m.col_version %% 2 = 1 " "AND NOT EXISTS (SELECT 1 FROM %s b WHERE b.pk = m.pk AND b.col_name LIKE $2)"; - -const char * const SQL_META_INSERT_BLOCK_IGNORE = - "INSERT INTO %s (pk, col_name, col_version, db_version, seq, site_id) " - "VALUES ($1, $2, $3, $4, $5, 0) ON CONFLICT DO NOTHING"; diff --git a/src/sql.h b/src/sql.h index 68371218..36fbb8c6 100644 --- a/src/sql.h +++ b/src/sql.h @@ -80,8 +80,6 @@ extern const char * const SQL_BLOCKS_UPSERT; extern const char * const SQL_BLOCKS_SELECT; extern const char * const SQL_BLOCKS_DELETE; extern const char * const SQL_BLOCKS_LIST_ALIVE; -extern const char * const SQL_BLOCKS_INSERT_IGNORE; extern const char * const SQL_META_SCAN_COL_FOR_MIGRATION; -extern const char * const SQL_META_INSERT_BLOCK_IGNORE; #endif diff --git a/src/sqlite/sql_sqlite.c b/src/sqlite/sql_sqlite.c index a6b1d7ac..a2bb7d3e 100644 --- a/src/sqlite/sql_sqlite.c +++ b/src/sqlite/sql_sqlite.c @@ -337,14 +337,7 @@ const char * const SQL_BLOCKS_LIST_ALIVE = "AND m.pk = ?3 AND m.col_name LIKE ?4 AND m.col_version %% 2 = 1 " "ORDER BY b.col_name"; -const char * const SQL_BLOCKS_INSERT_IGNORE = - "INSERT OR IGNORE INTO %s (pk, col_name, col_value) VALUES (?1, ?2, ?3)"; - const char * const SQL_META_SCAN_COL_FOR_MIGRATION = "SELECT DISTINCT m.pk FROM %s m " "WHERE m.col_name = ?1 AND m.col_version %% 2 = 1 " "AND NOT EXISTS (SELECT 1 FROM %s b WHERE b.pk = m.pk AND b.col_name LIKE ?2)"; - -const char * const SQL_META_INSERT_BLOCK_IGNORE = - "INSERT OR IGNORE INTO %s (pk, col_name, col_version, db_version, seq, site_id) " - "VALUES (?1, ?2, ?3, ?4, ?5, 0)"; From 31c5d06d3e4fad2576e22ab9c4c67ac818d98a1a Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:20:47 +0200 Subject: [PATCH 17/28] test: drop make endian-unittest and the unused host-order helpers Review point 12. Once pk.c stopped using host-order conversions, compiling it with a forced __BYTE_ORDER__ produced a byte-identical object, so the target could not fail; it was also not run by make test or CI. make unittest-s390x now covers big-endian hosts. cloudsync_endian.h keeps only bswap64_u64, so a host-dependent encoding cannot be reintroduced by accident; the pk.c comments state the format is the same on every host. Co-Authored-By: Claude Opus 5 --- Makefile | 13 -------- docs/internal/audit-regressions.md | 11 +++++-- src/cloudsync_endian.h | 49 +++--------------------------- src/pk.c | 10 +++--- 4 files changed, 20 insertions(+), 63 deletions(-) diff --git a/Makefile b/Makefile index 003e94a7..c2e352d6 100644 --- a/Makefile +++ b/Makefile @@ -301,19 +301,6 @@ unittest: $(TARGET) $(DIST_DIR)/unit$(EXE) $(DIST_DIR)/review_regressions$(EXE) @./$(DIST_DIR)/unit$(EXE) @./$(DIST_DIR)/review_regressions$(EXE) -# Force pk.c's endian-conversion branch while preserving the real host ABI. -# This catches double conversion regressions even on a little-endian CI host. -$(BUILD_TEST)/pk_forced_big_endian.o: $(SRC_DIR)/pk.c $(SRC_DIR)/cloudsync_endian.h - @mkdir -p $(BUILD_TEST) - $(CC) $(T_CFLAGS) -U__BYTE_ORDER__ -D__BYTE_ORDER__=__ORDER_BIG_ENDIAN__ -c $< -o $@ - -$(DIST_DIR)/review_regressions_big_endian$(EXE): $(TEST_OBJ) $(BUILD_TEST)/pk_forced_big_endian.o - $(CC) $(filter-out $(BUILD_TEST)/pk.o $(patsubst %.c,$(BUILD_TEST)/%.o,$(notdir $(TEST_SRC))),$(TEST_OBJ)) $(BUILD_TEST)/review_regressions.o $(BUILD_TEST)/pk_forced_big_endian.o -o $@ $(T_LDFLAGS) - -.PHONY: endian-unittest -endian-unittest: $(DIST_DIR)/review_regressions_big_endian$(EXE) - @./$(DIST_DIR)/review_regressions_big_endian$(EXE) - # Run the SQLite unit and regression suites on a real big-endian host (s390x) under QEMU # emulation. The payload and primary-key encodings are byte-order sensitive; this is the # only build that executes them on big-endian hardware semantics. Needs Docker with diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md index 9311f067..f5e8b71d 100644 --- a/docs/internal/audit-regressions.md +++ b/docs/internal/audit-regressions.md @@ -67,7 +67,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. | Area | Focused coverage | | --- | --- | | 64-bit clocks | Incoming column/database versions, causal length and sequence above UINT32_MAX | -| Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; forced big-endian conversion build; the unit and regression suites on a real big-endian host (`make unittest-s390x`, s390x under QEMU) | +| Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; the unit and regression suites on a real big-endian host (`make unittest-s390x`, s390x under QEMU) | | Virtual-table planner | Unusable/unsupported constraints before an accepted constraint; no accepted constraints | | RLS denials | Block-column and GOS denials skipped; every column of a permitted block/GOS row written; a column hidden from UPDATE not recorded as applied; order-dependent denials applied on retry in both the batched and trigger paths; permanently denied rows skipped with checkpoint advanced | | Block materialization errors | Write failure via cloudsync_text_materialize keeps cause, code/SQLSTATE and names stage, column, table; single-shot allocation failure at every allocation never yields a blank error | @@ -83,7 +83,7 @@ The previously reported `MAX_PARAMS` issue is outside this change. | Fractional indexing | 4096-byte common prefix plus the existing module suite | | Test harness | Temporary directory cleanup, memory accounting and sanitizer-safe RowID generation | -Run `make unittest`, `make unittest-s390x`, `make endian-unittest`, `make network-unittest`, and +Run `make unittest`, `make unittest-s390x`, `make network-unittest`, and `make -C modules/fractional-indexing/test run`. The network test needs permission to bind a loopback socket; it does not contact an external service. Run Node checks from `packages/node` with `npm test -- --run`, `npm run typecheck`, and @@ -114,3 +114,10 @@ runtime suites were not executed. The x86_64 runtime attempt was unavailable on this host (`Bad CPU type in executable`, Rosetta not available). The endian test exercises conversion logic; it is not a substitute for real big-endian hardware testing. + +Update: `make endian-unittest` has since been removed. Once `pk.c` stopped using +host-order conversions, forcing `__BYTE_ORDER__` produced a byte-identical object, so +it could no longer fail; the host-order helpers it exercised are gone from +`cloudsync_endian.h`. Big-endian coverage is now `make unittest-s390x`: the SQLite unit +and regression suites run on s390x under QEMU, and with the pre-1.1.4 double encoding +restored they fail there on the golden bytes while passing on little-endian hosts. diff --git a/src/cloudsync_endian.h b/src/cloudsync_endian.h index 4109ea7f..affc394a 100644 --- a/src/cloudsync_endian.h +++ b/src/cloudsync_endian.h @@ -14,6 +14,11 @@ #include // _byteswap_uint64 #endif +// Only an unconditional byte swap is provided. Wire formats are defined on byte values, +// never on the host's byte order, so the same bytes are produced on every architecture; +// host-order conversions are deliberately absent. `make unittest-s390x` checks this on a +// big-endian host. + // ======================================================= // bswap64 - portable // ======================================================= @@ -51,49 +56,5 @@ static inline uint64_t bswap64_u64(uint64_t v) { #endif } -// ======================================================= -// Compile-time endianness detection -// ======================================================= - -#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && defined(__ORDER_BIG_ENDIAN__) - #if (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - #define HOST_IS_LITTLE_ENDIAN 1 - #elif (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - #define HOST_IS_LITTLE_ENDIAN 0 - #endif -#endif - -// WebAssembly is currently defined as little-endian in all major toolchains -#if !defined(HOST_IS_LITTLE_ENDIAN) && (defined(__wasm__) || defined(__EMSCRIPTEN__)) - #define HOST_IS_LITTLE_ENDIAN 1 -#endif - -// Runtime fallback if unknown at compile-time -static inline int host_is_little_endian_runtime (void) { - const uint16_t x = 1; - return *((const uint8_t*)&x) == 1; -} - -// ======================================================= -// Public API -// ======================================================= - -static inline uint64_t host_to_be64 (uint64_t v) { -#if defined(HOST_IS_LITTLE_ENDIAN) - #if HOST_IS_LITTLE_ENDIAN - return bswap64_u64(v); - #else - return v; - #endif -#else - return host_is_little_endian_runtime() ? bswap64_u64(v) : v; -#endif -} - -static inline uint64_t be64_to_host (uint64_t v) { - // same operation (bswap if little-endian) - return host_to_be64(v); -} - #endif diff --git a/src/pk.c b/src/pk.c index 40e88e9d..fe2ff7f1 100644 --- a/src/pk.c +++ b/src/pk.c @@ -195,8 +195,9 @@ static int pk_decode_data (const uint8_t *buffer, size_t blen, size_t *bseek, si } int pk_decode_double (const uint8_t *buffer, size_t blen, size_t *bseek, double *out) { - // Historical wire format is little-endian IEEE754, unlike integer fields. - // pk_decode_uint64 already constructs a host integer from big-endian bytes. + // Doubles travel as little-endian IEEE754 bytes on every host, unlike integer + // fields: pk_decode_uint64 reads the bytes as a big-endian integer, and swapping it + // yields the IEEE754 bit pattern whatever the host's byte order. uint64_t bits_be = 0; if (!pk_decode_uint64(buffer, blen, bseek, sizeof(uint64_t), &bits_be)) return 0; @@ -527,12 +528,13 @@ char *pk_encode (dbvalue_t **argv, int argc, char *b, bool is_prikey, size_t *bs } break; case DBTYPE_FLOAT: { - // Encode doubles as IEEE754 64-bit, little-endian. + // Encode doubles as IEEE754 64-bit little-endian bytes on every host. double value = database_value_double(argv[i]); if (value < 0) {value = -value; type = DATABASE_TYPE_NEGATIVE_FLOAT;} uint64_t bits; memcpy(&bits, &value, sizeof(bits)); - // Preserve the deployed little-endian double wire format on all hosts. + // pk_encode_uint64 writes big-endian bytes, so the swapped value lands as + // the little-endian bytes of the bit pattern, independent of the host. bits = bswap64_u64(bits); bseek = pk_encode_u8(buffer, bseek, (uint8_t)type); bseek = pk_encode_uint64(buffer, bseek, bits, sizeof(bits)); From 3e8d95d82d2d73ecc51b9b442653bfcf6d292360 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Thu, 17 Sep 2026 13:22:54 +0200 Subject: [PATCH 18/28] test: clean the private test directory whatever the file names, and say what is left Review point 14. remove_test_directory() deleted only files starting with cloudsync-test-, so a test creating any other file made the cleanup fail silently and left the directory in TMPDIR, although the directory is private to the run (mkdtemp). It now deletes every entry, names any it cannot delete, and prints the directory when it is kept (cleanup disabled or failed). The Windows length check gets the size_t cast the POSIX branch already had. review_regressions' on-disk BUSY test uses its own private directory instead of a file directly in TMPDIR. Co-Authored-By: Claude Opus 5 --- test/review_regressions.c | 47 ++++++++++++++++++++++++++++++++------- test/unit.c | 33 ++++++++++++++++++++------- 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/test/review_regressions.c b/test/review_regressions.c index 27cf66aa..21602c19 100644 --- a/test/review_regressions.c +++ b/test/review_regressions.c @@ -4,6 +4,11 @@ #include #include #include +#ifdef _WIN32 +#include +#else +#include +#endif #include "sqlite3.h" #include "cloudsync.h" #include "cloudsync_sqlite.h" @@ -63,6 +68,36 @@ static void test_best_index(void) { CHECK(strcmp(info.idxStr, " ORDER BY db_version, seq ASC") == 0); sqlite3_free(info.idxStr); } +// A private directory for this run's on-disk databases, removed with its content at the +// end; a failed run leaves nothing behind in a shared temporary directory. +static char scratch_dir[256]; +static bool scratch_create(void) { +#ifdef _WIN32 + char base[MAX_PATH]; + DWORD n = GetTempPathA(sizeof(base), base); + if (!n || n >= sizeof(base)) return false; + int len = snprintf(scratch_dir, sizeof(scratch_dir), "%scloudsync-rr-%lu-%llu", base, + (unsigned long)GetCurrentProcessId(), (unsigned long long)GetTickCount64()); + return len > 0 && (size_t)len < sizeof(scratch_dir) && CreateDirectoryA(scratch_dir, NULL); +#else + const char *base = getenv("TMPDIR"); + if (!base || !*base) base = "."; + int len = snprintf(scratch_dir, sizeof(scratch_dir), "%s/cloudsync-rr-XXXXXX", base); + return len > 0 && (size_t)len < sizeof(scratch_dir) && mkdtemp(scratch_dir) != NULL; +#endif +} +static void scratch_remove(const char *const *names, int count) { + char path[512]; + for (int i = 0; i < count; i++) { + snprintf(path, sizeof(path), "%s/%s", scratch_dir, names[i]); + remove(path); + } +#ifdef _WIN32 + RemoveDirectoryA(scratch_dir); +#else + rmdir(scratch_dir); +#endif +} static int skipped_warnings; static int skipped_changes; // sum of N over "skipped N received change(s) that failed to apply" static void log_callback(void *arg, int code, const char *message) { @@ -112,10 +147,8 @@ static void test_payload_errors(void) { // on a retry, so it must fail the apply and leave the cursor in place. { char path[512]; - const char *dir = getenv("TMPDIR"); - unsigned int nonce = 0; - sqlite3_randomness(sizeof(nonce), &nonce); - snprintf(path, sizeof(path), "%s/cloudsync-rr-busy-%08x.db", (dir && *dir) ? dir : ".", nonce); + CHECK(scratch_create()); + snprintf(path, sizeof(path), "%s/busy.db", scratch_dir); sqlite3 *source = open_db(), *target = NULL, *locker = NULL; CHECK(sqlite3_open(path, &target) == SQLITE_OK); CHECK(sqlite3_cloudsync_init(target, NULL, NULL) == SQLITE_OK); @@ -136,10 +169,8 @@ static void test_payload_errors(void) { CHECK(scalar(target, "SELECT count(*) FROM t") == 2); CHECK(close_db(source) == SQLITE_OK); CHECK(close_db(target) == SQLITE_OK); - char aux[600]; - remove(path); - snprintf(aux, sizeof(aux), "%s-journal", path); - remove(aux); + const char *const files[] = {"busy.db", "busy.db-journal"}; + scratch_remove(files, 2); } sqlite3 *db = open_db(); diff --git a/test/unit.c b/test/unit.c index 97b2ea2d..7cc0e4eb 100644 --- a/test/unit.c +++ b/test/unit.c @@ -4131,7 +4131,7 @@ static bool create_test_directory(void) { if (!n || n >= sizeof(base)) return false; int len = snprintf(test_directory, sizeof(test_directory), "%scloudsync-%lu-%llu", base, (unsigned long)GetCurrentProcessId(), (unsigned long long)GetTickCount64()); - return len > 0 && len < sizeof(test_directory) && CreateDirectoryA(test_directory, NULL); + return len > 0 && (size_t)len < sizeof(test_directory) && CreateDirectoryA(test_directory, NULL); #else const char *base = getenv("TMPDIR"); if (!base || !*base) { @@ -4146,31 +4146,42 @@ static bool create_test_directory(void) { #endif } +// Removes the private test directory and everything the tests left in it. The directory +// is created by this run alone (mkdtemp), so every entry is ours to delete whatever its +// name. An entry that cannot be deleted (a database still open on Windows, say) is named, +// so a failed cleanup says what was left behind. static bool remove_test_directory(void) { char path[512]; + bool removed_all = true; #ifdef _WIN32 WIN32_FIND_DATAA entry; snprintf(path, sizeof(path), "%s\\*", test_directory); HANDLE handle = FindFirstFileA(path, &entry); if (handle == INVALID_HANDLE_VALUE) return false; do { - if (strncmp(entry.cFileName, "cloudsync-test-", 15) != 0) continue; + if (strcmp(entry.cFileName, ".") == 0 || strcmp(entry.cFileName, "..") == 0) continue; snprintf(path, sizeof(path), "%s\\%s", test_directory, entry.cFileName); - DeleteFileA(path); + if (!DeleteFileA(path)) { + fprintf(stderr, "\tunable to delete test file %s\n", path); + removed_all = false; + } } while (FindNextFileA(handle, &entry)); FindClose(handle); - return RemoveDirectoryA(test_directory) != 0; + return RemoveDirectoryA(test_directory) != 0 && removed_all; #else DIR *dir = opendir(test_directory); if (!dir) return false; struct dirent *entry; while ((entry = readdir(dir)) != NULL) { - if (strncmp(entry->d_name, "cloudsync-test-", 15) != 0) continue; + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; snprintf(path, sizeof(path), "%s/%s", test_directory, entry->d_name); - unlink(path); + if (unlink(path) != 0) { + fprintf(stderr, "\tunable to delete test file %s\n", path); + removed_all = false; + } } closedir(dir); - return rmdir(test_directory) == 0; + return rmdir(test_directory) == 0 && removed_all; #endif } @@ -13627,7 +13638,13 @@ int main (int argc, const char * argv[]) { result++; } - if (cleanup_databases) result += test_report("Temporary Directory Cleanup:", remove_test_directory()); + if (cleanup_databases) { + bool cleaned = remove_test_directory(); + result += test_report("Temporary Directory Cleanup:", cleaned); + if (!cleaned) printf("\tTest databases kept in %s\n", test_directory); + } else { + printf("Test databases kept in %s\n", test_directory); + } return result; } From 5c980a1c6e9162e90cd8a8c6560da957db3730ac Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 19:54:04 -0600 Subject: [PATCH 19/28] fix(apply): stop at the first failed write instead of skipping it Skipping a failed or denied write and advancing the receive checkpoint could acknowledge changes that were never stored. cloudsync_payload_apply now stops at the first change it cannot write and returns that error, with its SQLSTATE on PostgreSQL. The failed PK group is rolled back, no later change is applied, and the checkpoint does not move, so redelivering after the cause is fixed applies the payload. - Remove the skip policy: transient/permanent classification, the in-payload RLS retry passes, skip warnings and the apply_failed/apply_failure counters. - Keep earlier PK groups where the enclosing transaction allows it (SQLite); a PostgreSQL statement failure still rolls the statement back. - Count the rows applied before a failure, so receive.rows and receive.tables report them together with receive.error; drop receive.failed/failedError. - A batched multi-column UPDATE that changes no row (row gone or hidden by a USING policy) falls back to the upsert instead of recording winner clocks. Update API.md, the RLS reference, the changelog and the SQLite, network and PostgreSQL regressions (27, 29, 57, 58, 59) to the fail-fast behavior. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 39 +- CHANGELOG.md | 10 +- docs/postgresql/reference/rls.md | 16 +- src/cloudsync.c | 413 ++++---------------- src/cloudsync.h | 11 +- src/network/network.c | 72 ++-- src/postgresql/cloudsync_postgresql.c | 5 +- src/postgresql/database_postgresql.c | 6 +- test/network_unit.c | 15 +- test/postgresql/27_rls_batch_merge.sql | 20 +- test/postgresql/29_rls_multicol.sql | 43 +- test/postgresql/57_audit_regressions.sql | 74 ++-- test/postgresql/58_v3_denied_checkpoint.sql | 48 ++- test/postgresql/59_rls_denial_retry.sql | 267 +++++++++---- test/review_regressions.c | 82 ++-- test/unit.c | 109 +++--- 16 files changed, 560 insertions(+), 670 deletions(-) diff --git a/API.md b/API.md index 22150cfd..9825c6a9 100644 --- a/API.md +++ b/API.md @@ -667,11 +667,11 @@ SELECT cloudsync_payload_apply(:payload); #### Failed writes -A change whose write fails on its data — a constraint, a trigger that raises, a type error — fails the same way every time it is delivered, so it is skipped: the rest of the payload is applied and the receive checkpoint still advances. Each skipped change is logged as a warning (`sqlite3_log` on SQLite, a `WARNING` on PostgreSQL) and counted in `receive.failed` by the network functions. A transient or configuration failure — a locked or busy database, a deadlock or serialization failure, a cancel, running out of memory or disk, an I/O error, a missing privilege, a read-only database — is never skipped: the function fails and the checkpoint stays where it was, so the change is retried. +The apply stops at the first change whose write fails and returns that error: a constraint, a trigger that raises, a type error, a row-level security policy, a missing privilege, a lock or serialization failure. On PostgreSQL the error keeps its original SQLSTATE. No later change in the payload is applied, the change that failed leaves neither data nor sync metadata behind, and the receive checkpoint does not move. -#### Row-level security denials +On SQLite the changes applied before the failure are kept (inside the caller's transaction, if there is one). On PostgreSQL the failing statement is rolled back as a whole. -On PostgreSQL, a change rejected by a row-level security `WITH CHECK` policy is skipped as well, and the checkpoint advances. Because a policy can depend on rows that arrive later in the same payload — a membership row that grants access to the rows before it — denied changes are retried once the rest of the payload has been applied, repeating while a retry lets more changes through. A change still denied after that is skipped for good, and the function raises one `WARNING` with the number of changes skipped. A change that only a later payload would authorize is not retried. The return value still counts denied changes as payload rows. +Fix the cause and deliver the payload again: the changes already applied merge again as no-ops. A row-level security policy that depends on rows later in the same payload is not retried within the payload; the apply fails until the rows it depends on have been applied. --- @@ -821,33 +821,31 @@ If the network is misconfigured or the remote server is unreachable, the functio **Returns:** A JSON string with the receive result: ```json -{"receive": {"rows": N, "failed": F, "failedError": "...", "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} +{"receive": {"rows": N, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}}} ``` -- `receive.rows`: The total number of rows received and applied to the local database, summed across all chunks drained this call. `0` when the receive phase failed, when nothing was available, or when only intermediate fragments were staged without completing a value. -- `receive.failed`: The number of entries skipped because their write failed on the data itself — a constraint, a trigger that raised, a type error — summed across all chunks drained this call. Such a failure repeats identically on every retry, so the entry is skipped and the receive cursor still advances past it instead of stalling every later change; each skip is also logged as a warning. Not counted in `receive.rows`. Row-level security denials are not part of this count: they can only occur where the changes are applied under a PostgreSQL policy (see [Row-level security](#row-level-security-denials) below). A transient or configuration failure (a lock or busy database, a deadlock or serialization failure, a cancel, out of memory or disk, an I/O error, a missing privilege, a read-only database) is never skipped: it is reported in `receive.error` and the cursor stays in place so the next call retries it. -- `receive.failedError` (optional): The error message of the first entry counted in `receive.failed`, present only when `failed` is non-zero. -- `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. -- `receive.chunks`: The number of payload chunks applied by this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a drained `N`-chunk stream (bounded by `max_chunks` if given). +- `receive.rows`: The total number of rows received and applied to the local database, summed across all chunks drained this call, including the rows applied before an error. `0` when nothing was available, or when only intermediate fragments were staged without completing a value. +- `receive.tables`: An array of table names that received changes (the union across all drained chunks, including changes applied before an error). Empty (`[]`) if no changes were applied. +- `receive.chunks`: The number of payload chunks fully applied by this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a drained `N`-chunk stream (bounded by `max_chunks` if given). - `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; transport-independent, not the compressed wire size). Useful for byte-budgeted draining together with `max_chunks`. - `receive.complete` (boolean): `true` when the receive stream is fully drained (nothing pending), `false` when more chunks remain — because `max_chunks` capped the drain, or it stopped early. When `false`, call this function again to continue. -- `receive.error` (optional, string): Present when client-side `cloudsync_payload_apply` failed. Contains a human-readable error message describing why the received payload could not be applied. +- `receive.error` (optional, string): Present when client-side `cloudsync_payload_apply` failed. Contains the error of the first change that could not be applied (see [Failed writes](#failed-writes)); `receive.complete` is then `false`, and the next call receives the changes again from the same checkpoint. - `receive.lastFailure` (optional, object): Present only when the server reports a failed check job. Forwarded verbatim from the server's `failures.check` and typically includes `jobId`, `dbVersion`, `seq`, `code`, `stage`, `message`, `retryable`, and `failedAt`. Distinct from `receive.error`: `receive.error` describes a client-side apply failure (string), while `receive.lastFailure` describes a server-side check-job failure (object). Both can coexist in the same response. This function is **check-scoped**: server-reported apply-job failures (`failures.apply`) are not surfaced here — see [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) and [`cloudsync_network_sync()`](#cloudsync_network_syncwait_ms-max_retries). **Example:** ```sql SELECT cloudsync_network_receive_changes(); --- '{"receive":{"rows":3,"failed":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' +-- '{"receive":{"rows":3,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Capped drain with more pending (call again to continue): --- '{"receive":{"rows":40,"failed":0,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' +-- '{"receive":{"rows":40,"tables":["docs"],"chunks":5,"bytes":1310720,"complete":false}}' -- With a client-side apply error: --- '{"receive":{"rows":0,"failed":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' +-- '{"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' -- With a server-reported check-job failure: --- '{"receive":{"rows":0,"failed":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' +-- '{"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":true,"lastFailure":{"jobId":456,"dbVersion":15,"seq":1,"code":"tenant_unreachable","stage":"encode_changes","message":"tenant check failed","retryable":true,"failedAt":"2026-04-24T10:22:00Z"}}}' ``` --- @@ -877,7 +875,7 @@ When the server delivers changes as a stream of chunks, this function drains the ```json { "send": {"status": "synced|syncing|out-of-sync|error", "localVersion": N, "serverVersion": N, "chunks": C, "bytes": B, "lastFailure": {...}}, - "receive": {"rows": N, "failed": F, "failedError": "...", "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} + "receive": {"rows": N, "tables": ["table1", "table2"], "chunks": C, "bytes": B, "complete": true, "error": "...", "lastFailure": {...}} } ``` @@ -886,9 +884,8 @@ When the server delivers changes as a stream of chunks, this function drains the - `send.serverVersion`: The latest version confirmed by the server. - `send.chunks` / `send.bytes`: Number of payload chunks sent and total serialized payload bytes sent during the send phase. Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes). - `send.lastFailure` (optional): Same semantics as in [`cloudsync_network_send_changes()`](#cloudsync_network_send_changes) — forwarded verbatim from the server's `failures.apply` whenever a failed apply job is reported, regardless of `status`. -- `receive.rows`: The **total** number of rows received and applied during the receive phase, summed across **all** chunks drained in this call. `0` when the receive phase failed. -- `receive.failed` / `receive.failedError`: The **total** number of entries skipped because their write failed, and the first such error — see [Receive Changes](#receive-changes). -- `receive.tables`: An array of table names that received changes (the union across all drained chunks). Empty (`[]`) if no changes were applied or the receive phase failed. +- `receive.rows`: The **total** number of rows received and applied during the receive phase, summed across **all** chunks drained in this call, including the rows applied before an error. +- `receive.tables`: An array of table names that received changes (the union across all drained chunks, including changes applied before an error). Empty (`[]`) if no changes were applied. - `receive.chunks`: The number of payload chunks applied in this call. `0` when nothing was ready, `1` for a single monolithic/inline page, and `N` for a fully drained `N`-chunk stream. `cloudsync_network_sync()` always drains the whole stream (it does not cap chunks). - `receive.bytes`: The total serialized payload bytes received this call (uncompressed cloudsync payload size, summed across chunks; not the compressed wire size). Same semantics as in [`cloudsync_network_receive_changes()`](#cloudsync_network_receive_changesmax_chunks). - `receive.complete` (boolean): `true` when the server stream was fully drained, `false` when the download stopped before the final chunk (an error occurred, or an internal safety bound was reached). When `false`, call `cloudsync_network_sync()` again to resume; re-delivered rows are idempotent. @@ -900,15 +897,15 @@ When the server delivers changes as a stream of chunks, this function drains the ```sql -- Perform a single synchronization cycle SELECT cloudsync_network_sync(); --- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"failed":0,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":2048},"receive":{"rows":3,"tables":["tasks"],"chunks":1,"bytes":820,"complete":true}}' -- Perform a synchronization cycle with custom retry settings SELECT cloudsync_network_sync(500, 3); -- A large download drained as a multi-chunk stream in a single call: --- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"failed":0,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' +-- '{"send":{"status":"synced","localVersion":42,"serverVersion":42,"chunks":0,"bytes":0},"receive":{"rows":1200,"tables":["docs"],"chunks":7,"bytes":1835008,"complete":true}}' -- Receive phase failed but send phase completed — the error is surfaced in JSON, not as a SQL error: --- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"failed":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' +-- '{"send":{"status":"synced","localVersion":5,"serverVersion":5,"chunks":1,"bytes":512},"receive":{"rows":0,"tables":[],"chunks":0,"bytes":0,"complete":false,"error":"Cannot apply the received payload because the schema hash is unknown 7218827471400075525."}}' ``` --- diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c0e9a9..be8c2147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,16 +8,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **PostgreSQL: rows rejected by a row-level security policy are retried within the payload, then skipped and reported.** A policy can depend on rows that arrive later in the same payload (a membership that grants access to the rows before it), so denied changes are retried once the rest of the payload is applied, repeating while a retry makes progress. A change still denied is skipped — it would be denied on every delivery — the receive cursor advances past it, and `cloudsync_payload_apply` raises one `WARNING` with the number of skipped changes. This now also covers block columns and grow-only-set tables, whose denial previously failed the whole apply. It does not yet extend to a value large enough to be sent in fragments: a denial there is still an error and the cursor does not advance, and a change authorized only by a later payload is not retried. - **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **A compressed payload can no longer make the extension allocate an arbitrary amount of memory.** The decompressed size is read from the payload header and allocated before decompressing, so a few forged bytes could previously claim gigabytes. A declared size larger than LZ4 can actually produce from the compressed bytes (255:1) is now rejected up front, as is one beyond LZ4's own `INT_MAX` limit, where the previous code handed LZ4 a negative capacity. There is no fixed size limit: any payload the library produces, however large, still applies. +### Changed + +- **`cloudsync_payload_apply` stops at the first change it cannot write and returns that error**: a constraint, a trigger that raises, a type error, a row-level security denial, a missing privilege, a lock or serialization failure. No later change in the payload is applied, the change that failed leaves no data or sync metadata behind, and the receive checkpoint does not move, so delivering the payload again once the cause is fixed applies it. On PostgreSQL the error keeps its SQLSTATE (`42501` for a policy denial) and the failing statement is rolled back; previously a row denied by a policy was skipped silently while the rest of the payload applied. Block columns, grow-only-set tables and fragmented values behave the same way. + ### Fixed -- **A received change whose write fails is no longer dropped silently.** Previously the error was discarded, so the change was lost without a trace. A failure of the data itself (a constraint, a raising trigger, a type error) repeats on every retry, so the change is still skipped and the receive cursor still advances — holding it back would stall every later change forever — but it is now logged as a warning and reported as `receive.failed` / `receive.failedError`. A transient or configuration failure (lock or busy database, deadlock, serialization failure, cancel, out of memory or disk, I/O error, missing privilege, read-only database) now fails the apply and leaves the cursor in place, so the change is retried instead of lost. On PostgreSQL, a failure that cannot be rolled back on its own fails the whole apply. +- **A received change whose write fails is no longer dropped silently.** Previously the error was discarded and the change was lost without a trace. The apply now fails with that error (see Changed), and the network functions report it as `receive.error`, together with the rows applied before it. +- **A received update to a row that is not there is no longer recorded as applied.** Several columns of an existing row are written with one UPDATE; when it changed nothing (the row was deleted while sync was disabled, or a policy's USING clause hides it), the changes were recorded as applied without being stored. The row is now written with the upsert instead, which inserts it or reports the policy. - **PostgreSQL: applying a payload read from a table no longer fails.** `SELECT cloudsync_payload_apply(data) FROM some_table` failed with `buffer pin ... is not owned by resource owner TopTransaction` (and could abort an assertion-enabled server): the internal savepoints left the caller's resource owner and memory context switched. Both are now restored when a savepoint ends. - **Block columns and grow-only-set tables merge under row-level security and NOT NULL constraints.** Both wrote a received column through an upsert holding only the primary key and that column: PostgreSQL checks an INSERT policy against that proposed row even when it becomes an update, and a NOT NULL column rejects it outright, so the column was never written. A block column's row now has its other received columns written first, and an existing row — of either kind — is updated in place. When the update finds no row to change (the row is gone, or an UPDATE policy hides it), the upsert is used as before, so the write is reported rather than recorded as applied without being stored. -- **A received row whose write fails leaves no partial state behind.** Applying a row wrote part of its metadata — for a deleted-and-recreated row, the new sentinel and the reset column clocks — before the row itself, outside the savepoint that protected the write. When the write then failed, the metadata claimed the row was there: delivered again, even after the cause was fixed (a lock that went away, a policy that now allows it), the row was silently never created, and an existing row kept zeroed clocks. Each primary key's changes are now applied under one savepoint that the failed write rolls back entirely. Skipped changes are also counted per payload entry, so a recreated row counts its sentinel too. +- **A received row whose write fails leaves no partial state behind.** Applying a row wrote part of its metadata — for a deleted-and-recreated row, the new sentinel and the reset column clocks — before the row itself, outside the savepoint that protected the write. When the write then failed, the metadata claimed the row was there: delivered again, even after the cause was fixed (a lock that went away, a policy that now allows it), the row was silently never created, and an existing row kept zeroed clocks. Each primary key's changes are now applied under one savepoint that the failed write rolls back entirely. - **PostgreSQL: savepoints opened by cloudsync no longer disturb the caller's snapshot.** Ending a nested savepoint swapped the active snapshot inside an enclosing one, so rolling that back could leave the calling statement without its snapshot (an assertion failure on assert-enabled builds). - **PostgreSQL: errors raised by cloudsync keep the SQLSTATE of the database error behind them.** A failure inside the tracking triggers or `cloudsync_payload_apply` — a serialization failure (`40001`), a deadlock (`40P01`), a unique violation (`23505`), a missing privilege (`42501`) — was re-raised as `XX000` internal_error, so application retry logic and error handling keyed on SQLSTATE could not react to it. cloudsync's own failures are still `XX000`. - **SQLite: a failed tracking write in the insert, update and delete triggers keeps its result code and context.** It was reported as a bare `SQLITE_ERROR` with SQLite's last message; it now carries the real code (`SQLITE_BUSY`, `SQLITE_CONSTRAINT`, ...) and cloudsync's message. diff --git a/docs/postgresql/reference/rls.md b/docs/postgresql/reference/rls.md index 8b6586d2..4de798b7 100644 --- a/docs/postgresql/reference/rls.md +++ b/docs/postgresql/reference/rls.md @@ -15,15 +15,11 @@ When flushing a batch, CloudSync chooses the statement type based on whether the - **New row**: `INSERT ... ON CONFLICT DO UPDATE` — all columns are present (including the ownership column), so the INSERT `WITH CHECK` policy can evaluate correctly. - **Existing row**: `UPDATE ... SET ... WHERE pk = ...` — only the changed columns are set. The UPDATE `USING` policy checks the existing row, which already has the correct ownership column value. -### Per-PK savepoint isolation +### Denied writes -Each primary key's flush is wrapped in its own savepoint. When RLS denies a write: +When RLS denies a write, `cloudsync_payload_apply` stops at that change and raises the policy's error (SQLSTATE `42501`). No later change in the payload is applied, and the failed statement is rolled back, so the payload leaves no rows or sync metadata behind and the receive checkpoint does not move. -1. The database raises an error inside the savepoint -2. CloudSync rolls back that savepoint, releasing all resources acquired during the failed statement -3. Processing continues with the next primary key - -This means a single payload can contain a mix of allowed and denied rows — allowed rows commit normally, denied rows are silently skipped. The caller receives the total number of column changes processed (including denied ones) rather than an error. +Once the policy allows the rows (or the rows it depends on have been applied), deliver the same payload again: it applies in full. A policy that depends on rows later in the same payload, such as a membership row that grants access to the rows before it, is not retried within the payload. ## Quick Setup @@ -119,7 +115,7 @@ SET ROLE authenticated; SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); ``` -The insert is denied by RLS. The row does not appear in DB B. No error is raised to the caller — CloudSync isolates the failure via a per-PK savepoint and continues processing the remaining payload. +The insert is denied by RLS: `cloudsync_payload_apply` raises `42501` and the row does not appear in DB B. ### Partial update sync @@ -140,7 +136,7 @@ The UPDATE policy checks the existing row (which has the correct `user_id`), so ### Mixed payload -When a single payload contains rows for multiple users, CloudSync handles each primary key independently: +When a single payload contains rows for multiple users, the first denied row fails the whole apply: ```sql -- On DB A @@ -151,7 +147,7 @@ INSERT INTO documents VALUES ('doc4', 'user2-uuid', 'Theirs', '...'); ```sql -- On DB B (running as user1) SELECT cloudsync_payload_apply(decode(:payload_hex, 'hex')); --- doc3 is inserted (allowed), doc4 is silently skipped (denied) +-- ERROR: doc4 is denied (42501); doc3 is rolled back with the statement ``` ## Supabase Notes diff --git a/src/cloudsync.c b/src/cloudsync.c index f948554f..141acff1 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -120,15 +120,6 @@ typedef struct { int cached_col_count; const char **cached_col_names; // array of pointers into table_context (not owned) - // Set by merge_flush_pending when a failure could not be contained by its savepoint - // (the savepoint could not be opened or rolled back), so the caller must not continue. - bool flush_uncontained; - - // Payload rows waiting in this batch (column rows and an explicit sentinel row), the - // count of changes lost if its flush fails. A sentinel implied by a column row is not - // a payload row of its own and is not counted. - int rows; - // True while cloudsync_payload_apply holds a savepoint around the whole PK group: the // flush then writes inside it instead of opening its own, so that a failed flush also // rolls back what the group's rows already wrote (a resurrected row's sentinel and @@ -209,15 +200,10 @@ struct cloudsync_context { int64_t apply_last_db_version; int64_t apply_last_seq; - // Entries applied, and entries skipped because their write failed, accumulated - // across a receive drain so an early chunk is still visible when a later chunk - // reports. Reset with cloudsync_apply_stats_reset. Kept here rather than derived - // from the apply return value, which reports payload entries (skipped ones - // included) and is a tested part of the SQL surface. Row-level security denials - // only occur on PostgreSQL, which has no receive drain: they are logged instead. + // Entries applied, accumulated across a receive drain so an early chunk is still + // visible when a later chunk reports, including the changes a failing payload + // applied before its error. Reset with cloudsync_apply_stats_reset. int apply_rows; - int apply_failed; - char apply_failure[512]; // first skipped failure since the last reset ("" = none) }; struct cloudsync_table_context { @@ -647,22 +633,13 @@ const char *cloudsync_errmsg (cloudsync_context *data) { } void cloudsync_apply_stats_reset (cloudsync_context *data) { - if (data) { data->apply_rows = 0; data->apply_failed = 0; data->apply_failure[0] = 0; } + if (data) data->apply_rows = 0; } -// Saturating: only a receive drain resets these, so on the direct-SQL apply path they -// accumulate for the life of the connection and signed overflow would be undefined. -static void cloudsync_apply_stats_add (cloudsync_context *data, int rows, int failed) { +// Saturating: only a receive drain resets it, so on the direct-SQL apply path it +// accumulates for the life of the connection and signed overflow would be undefined. +static void cloudsync_apply_stats_add (cloudsync_context *data, int rows) { if (rows > 0) data->apply_rows = (data->apply_rows > INT_MAX - rows) ? INT_MAX : data->apply_rows + rows; - if (failed > 0) data->apply_failed = (data->apply_failed > INT_MAX - failed) ? INT_MAX : data->apply_failed + failed; -} - -int cloudsync_apply_failed_count (cloudsync_context *data) { - return (data) ? data->apply_failed : 0; -} - -const char *cloudsync_apply_failure_message (cloudsync_context *data) { - return (data && data->apply_failure[0]) ? data->apply_failure : NULL; } int cloudsync_apply_rows_count (cloudsync_context *data) { @@ -1403,7 +1380,6 @@ static int merge_pending_add (cloudsync_context *data, cloudsync_table_context * e->seq = seq; batch->count++; - batch->rows++; return DBRES_OK; } @@ -1426,13 +1402,11 @@ static void merge_pending_free_entries (merge_pending_batch *batch) { batch->sentinel_pending = false; batch->row_exists = false; batch->count = 0; - batch->rows = 0; } static int merge_flush_pending (cloudsync_context *data) { merge_pending_batch *batch = data->pending_batch; if (!batch) return DBRES_OK; - batch->flush_uncontained = false; int rc = DBRES_OK; bool flush_savepoint = false; @@ -1450,7 +1424,7 @@ static int merge_flush_pending (cloudsync_context *data) { // Inside a payload apply's PK-group savepoint the caller rolls back instead. if (!batch->group_savepoint) { rc = database_begin_savepoint(data, "merge_flush"); - if (rc != DBRES_OK) { batch->flush_uncontained = true; goto cleanup; } + if (rc != DBRES_OK) goto cleanup; flush_savepoint = true; } @@ -1478,6 +1452,7 @@ static int merge_flush_pending (cloudsync_context *data) { // Check if cached prepared statement can be reused cloudsync_table_context *table = batch->table; dbvm_t *vm = NULL; +write_row:; bool cache_hit = false; if (batch->cached_vm && @@ -1578,6 +1553,7 @@ static int merge_flush_pending (cloudsync_context *data) { if (table->algo == table_algo_crdt_gos) table->enabled = 0; SYNCBIT_SET(data); rc = databasevm_step(vm); + bool update_missed = batch->row_exists && rc == DBRES_DONE && databasevm_changes(vm) == 0; dbvm_reset(vm); SYNCBIT_RESET(data); if (table->algo == table_algo_crdt_gos) table->enabled = 1; @@ -1586,6 +1562,14 @@ static int merge_flush_pending (cloudsync_context *data) { cloudsync_set_dberror(data); goto cleanup; } + if (update_missed) { + // The row is not there to update — gone, or hidden by a policy's USING clause, + // which an UPDATE skips silently. Write it through the upsert instead, which + // inserts a missing row or reports the policy, rather than record winner clocks + // for a write that never happened. + batch->row_exists = false; + goto write_row; + } rc = DBRES_OK; // Call merge_set_winner_clock for each buffered entry @@ -1620,7 +1604,7 @@ static int merge_flush_pending (cloudsync_context *data) { error_sqlstate = cloudsync_sqlstate(data); } } - if (rc != DBRES_OK && database_rollback_savepoint(data, "merge_flush") != DBRES_OK) batch->flush_uncontained = true; + if (rc != DBRES_OK) database_rollback_savepoint(data, "merge_flush"); } if (rc != DBRES_OK && flush_savepoint) { // the rollback cleared the error state: restore the failure's own message and SQLSTATE @@ -2206,7 +2190,6 @@ int merge_insert (cloudsync_context *data, cloudsync_table_context *table, const int rc = merge_sentinel_only_insert(data, table, insert_pk, insert_pk_len, insert_col_version, insert_db_version, insert_site_id, insert_site_id_len, insert_seq, rowid); if (rc != DBRES_OK) cloudsync_set_error(data, "Unable to perform merge_sentinel_only_insert", rc); - else if (data->pending_batch) data->pending_batch->rows++; // an explicit sentinel is a payload row return rc; } @@ -4206,93 +4189,29 @@ static void cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t } } -// A failed write is skipped only when retrying could not change the outcome. A lock, -// deadlock, serialization failure, cancel or resource shortage can succeed on the next -// attempt, and a missing privilege or a read-only database is fixed by configuration, -// not by the data: those still fail the apply and leave the receive cursor in place. -static bool cloudsync_apply_error_is_transient (int rc) { - // SQLite can report an extended code (SQLITE_BUSY_SNAPSHOT is 517): its low byte - // is the primary code. DBRES_POLICY_DENIED's low byte matches no case below. - if (rc <= 0) return false; - switch (rc & 0xFF) { - case DBRES_PERM: - case DBRES_READONLY: - case DBRES_AUTH: - case DBRES_BUSY: - case DBRES_LOCKED: - case DBRES_NOMEM: - case DBRES_INTERRUPT: - case DBRES_IOERR: - case DBRES_FULL: - return true; - } - return false; -} - -typedef struct { - int denied; // entries rejected by a row-level security policy - int failed; // entries skipped because their write failed - int fatal_rc; // first failure that stops the apply (DBRES_OK = none) - int fatal_sqlstate; - char fatal_message[1024]; -} cloudsync_apply_outcome; - -// Classifies one failed write of cloudsync_payload_apply. contained is true when the -// failure has already been rolled back, leaving the transaction usable. Returns true -// when the apply must stop. -static bool cloudsync_apply_note_failure (cloudsync_context *data, cloudsync_apply_outcome *out, int rc, int entries, bool contained) { - if (entries < 1) entries = 1; - if (rc == DBRES_POLICY_DENIED && contained) { - out->denied += entries; - return false; - } - if (contained && !cloudsync_apply_error_is_transient(rc)) { - // A write that fails on its data (a constraint, a trigger, a type error) fails - // the same way on every retry. Holding the cursor back would re-deliver it - // forever and stall every later change behind it, so the change is skipped — - // but reported, never dropped silently. - const char *msg = cloudsync_errmsg(data); - if (!msg || !msg[0]) msg = "Unable to apply a received change"; - out->failed += entries; - if (!data->apply_failure[0]) snprintf(data->apply_failure, sizeof(data->apply_failure), "%s", msg); - char warning[1200]; - snprintf(warning, sizeof(warning), "skipped %d received change%s that failed to apply: %s", entries, (entries == 1) ? "" : "s", msg); - database_log_warning(data, warning); - return false; - } - if (out->fatal_rc == DBRES_OK) { - out->fatal_rc = rc; - out->fatal_sqlstate = cloudsync_sqlstate(data); - snprintf(out->fatal_message, sizeof(out->fatal_message), "%s", cloudsync_errmsg(data)); - } - return true; -} - -// Steps one decoded payload row (an INSERT into cloudsync_changes). *contained reports -// whether a failure was rolled back and left the transaction usable. On SQLite a failed -// statement always is. On PostgreSQL the merge runs inside the cloudsync_changes trigger, -// so the row gets its own savepoint: a denial or a failed write then rolls back cleanly -// — metadata included — and the apply can go on to the next row. -static int cloudsync_payload_apply_row (cloudsync_context *data, dbvm_t *vm, bool *contained) { +// Steps one decoded payload row (an INSERT into cloudsync_changes). On PostgreSQL the merge +// runs inside the cloudsync_changes trigger, so the row gets its own savepoint: a failed +// write rolls back cleanly, metadata included, and leaves the transaction usable for the +// PK group's rollback. The failure's message and SQLSTATE survive the rollback. +static int cloudsync_payload_apply_row (cloudsync_context *data, dbvm_t *vm) { #ifdef CLOUDSYNC_POSTGRESQL_BUILD int rc = database_begin_savepoint(data, "cloudsync_apply_row"); - if (rc != DBRES_OK) { *contained = false; return rc; } + if (rc != DBRES_OK) return rc; rc = databasevm_step(vm); if (rc == DBRES_DONE) { int commit_rc = database_commit_savepoint(data, "cloudsync_apply_row"); - if (commit_rc == DBRES_OK) { *contained = true; return DBRES_DONE; } + if (commit_rc == DBRES_OK) return DBRES_DONE; rc = commit_rc; } char message[1024]; snprintf(message, sizeof(message), "%s", cloudsync_errmsg(data)); int sqlstate = cloudsync_sqlstate(data); - *contained = (database_rollback_savepoint(data, "cloudsync_apply_row") == DBRES_OK); + database_rollback_savepoint(data, "cloudsync_apply_row"); cloudsync_reset_error(data); cloudsync_set_error(data, message[0] ? message : "Unable to apply a received change", rc); cloudsync_set_sqlstate(data, sqlstate); return rc; #else - *contained = true; return databasevm_step(vm); #endif } @@ -4306,39 +4225,6 @@ static bool cloudsync_payload_row_is_block (const cloudsync_pk_decode_bind_conte memchr(row->col_name, BLOCK_SEPARATOR, (size_t)row->col_name_len) != NULL; } -// A run of payload rows rejected by a row-level security policy, kept so the apply can -// retry it once the rest of the payload is in: a policy can depend on rows that arrive -// later in the same payload (a membership row granting access to the rows before it). -typedef struct { - const char *start; // first row of the run inside the payload buffer - size_t avail; // bytes left in the buffer from start - uint32_t rows; // payload rows in the run - int entries; // entries counted as denied for the run -} cloudsync_denied_run; - -typedef struct { - cloudsync_denied_run *runs; - int count; - int capacity; -} cloudsync_denied_runs; - -// Records a denied run. A run for a whole PK group absorbs the single-row runs already -// recorded inside it, so no row is retried twice. Out of memory only loses the retry: -// the entries stay counted as denied. -static void cloudsync_denied_runs_add (cloudsync_denied_runs *list, const char *start, size_t avail, uint32_t rows, int entries) { - while (list->count > 0 && list->runs[list->count - 1].start >= start) { - entries += list->runs[--list->count].entries; - } - if (list->count == list->capacity) { - int capacity = list->capacity ? list->capacity * 2 : 8; - cloudsync_denied_run *runs = cloudsync_memory_realloc(list->runs, (uint64_t)capacity * sizeof(*runs)); - if (!runs) return; - list->runs = runs; - list->capacity = capacity; - } - list->runs[list->count++] = (cloudsync_denied_run){.start = start, .avail = avail, .rows = rows, .entries = entries}; -} - // Opens the savepoint around a PK group of payload rows (see merge_pending_batch). If it // cannot be opened the group runs without one and the flush falls back to its own. static void cloudsync_payload_group_open (cloudsync_context *data, merge_pending_batch *batch) { @@ -4348,14 +4234,11 @@ static void cloudsync_payload_group_open (cloudsync_context *data, merge_pending } // Flushes the pending PK group and closes its savepoint: released when the flush -// succeeds, rolled back when it fails, so a failed group leaves no trace behind — its -// rows can then be retried or delivered again and apply cleanly. *entries receives the -// payload rows the flush covered and *contained whether a failure left the transaction -// usable. The failure's message and SQLSTATE survive the rollback. -static int cloudsync_payload_group_flush (cloudsync_context *data, merge_pending_batch *batch, int *entries, bool *contained) { - *entries = batch->rows; +// succeeds, rolled back when it fails, so a failed group leaves no trace behind and +// applies cleanly when delivered again. The failure's message and SQLSTATE survive the +// rollback. +static int cloudsync_payload_group_flush (cloudsync_context *data, merge_pending_batch *batch) { int rc = merge_flush_pending(data); - *contained = !batch->flush_uncontained; if (!batch->group_savepoint) return rc; batch->group_savepoint = false; if (rc == DBRES_OK) { @@ -4365,7 +4248,7 @@ static int cloudsync_payload_group_flush (cloudsync_context *data, merge_pending char message[1024]; snprintf(message, sizeof(message), "%s", cloudsync_errmsg(data)); int sqlstate = cloudsync_sqlstate(data); - *contained = (database_rollback_savepoint(data, "cloudsync_merge_group") == DBRES_OK); + database_rollback_savepoint(data, "cloudsync_merge_group"); cloudsync_reset_error(data); cloudsync_set_error(data, message[0] ? message : "Unable to flush pending changes", rc); cloudsync_set_sqlstate(data, sqlstate); @@ -4495,19 +4378,14 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b } int n = 0; rc = cloudsync_payload_apply_fragment_row(data, &row, &n); - // A denial is NOT skipped here, unlike the row path. Continuing past one - // leaves PostgreSQL's transaction unusable — the next statement fails with - // "buffer pin is not owned by resource owner" — and neither a savepoint - // around this call nor dropping the staged-fragment delete recovers it. - // A denied oversize value is therefore still a hard receive error that - // stalls the cursor. See test 58_v3_denied_checkpoint.sql. + // stop at the first error, as the row path does if (rc != DBRES_OK) break; applied_rows += n; buffer += seek; buf_len -= seek; } if (clone) cloudsync_memory_free(clone); - cloudsync_apply_stats_add(data, (rc == DBRES_OK) ? applied_rows : 0, 0); + cloudsync_apply_stats_add(data, applied_rows); if (pnrows) *pnrows = applied_rows; // Advance the receive cursor only after the whole payload is applied, // gated on the caller-supplied checkpoint (a non-final chunk passes @@ -4528,12 +4406,11 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b uint16_t ncols = header.ncols; uint32_t nrows = header.nrows; int64_t last_payload_db_version = -1; - cloudsync_apply_outcome outcome = {0}; - bool stop = false; - cloudsync_denied_runs denied_runs = {0}; - const char *group_start = NULL; // first row of the current PK group - size_t group_avail = 0; - uint32_t group_rows = 0; + // The apply stops at the first failed write. Rows before the current PK group are + // written; the failed group is rolled back. applied counts the written rows. + int fail_rc = DBRES_OK; + int applied = 0; + int applied_at_savepoint = 0; // applied when the per-db_version savepoint began cloudsync_pk_decode_bind_context decoded_context = {.vm = vm}; // Initialize deferred column-batch merge @@ -4547,8 +4424,6 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b for (uint32_t i=0; iskip_decode_idx, cloudsync_payload_decode_callback, &decoded_context); if (res == -1) { cloudsync_payload_group_abandon(data, &batch); @@ -4572,29 +4447,17 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // Flush pending batch before any boundary change, closing the PK group if (pk_changed || tbl_changed || db_version_changed) { - int entries = 0; - bool contained = true; - int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); - if (flush_rc == DBRES_POLICY_DENIED && contained && group_start) { - cloudsync_denied_runs_add(&denied_runs, group_start, group_avail, group_rows, entries); - } - if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { - stop = true; - break; - } - group_start = row_start; - group_avail = row_avail; - group_rows = 0; + fail_rc = cloudsync_payload_group_flush(data, &batch); + if (fail_rc != DBRES_OK) break; + applied = (int)i; } - if (!group_start) { group_start = row_start; group_avail = row_avail; } - group_rows++; // Per-db_version savepoints group rows with the same source db_version // into one transaction. In SQLite autocommit mode, the RELEASE triggers // the commit hook which bumps data->db_version and resets seq, ensuring // unique (db_version, seq) tuples across groups. In PostgreSQL SPI, // database_in_transaction() is always true so this block is inactive — - // the per-PK group savepoint handles RLS and failed writes instead. + // the per-PK group savepoint protects each group instead. if (in_savepoint && db_version_changed) { rc = database_commit_savepoint(data, "cloudsync_payload_apply"); if (rc != DBRES_OK) { @@ -4615,6 +4478,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b goto cleanup; } in_savepoint = true; + applied_at_savepoint = applied; } // Track db_version for batch-flush boundary detection @@ -4629,175 +4493,69 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b last_tbl_len = decoded_context.tbl_len; if (batch.count > 0 && cloudsync_payload_row_is_block(&decoded_context)) { - int entries = 0; - bool contained = true; - int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); - if (flush_rc == DBRES_POLICY_DENIED && contained && group_start && group_rows > 1) { - // the rows of this PK group before the current one - cloudsync_denied_runs_add(&denied_runs, group_start, group_avail, group_rows - 1, entries); - } - if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { - stop = true; - break; - } + fail_rc = cloudsync_payload_group_flush(data, &batch); + if (fail_rc != DBRES_OK) break; + applied = (int)i; } cloudsync_payload_group_open(data, &batch); - bool contained = true; - rc = cloudsync_payload_apply_row(data, vm, &contained); - if (rc != DBRES_DONE) { - if (rc == DBRES_POLICY_DENIED && contained) { - cloudsync_denied_runs_add(&denied_runs, row_start, row_avail, 1, 1); - } - if (cloudsync_apply_note_failure(data, &outcome, rc, 1, contained)) { - stop = true; - buffer += seek; - buf_len -= seek; - dbvm_reset(vm); - break; - } - } - + int step_rc = cloudsync_payload_apply_row(data, vm); buffer += seek; buf_len -= seek; dbvm_reset(vm); + if (step_rc != DBRES_DONE) { + fail_rc = step_rc; + break; + } } - // Final flush after loop (a stopped apply discards the open group) - if (stop) { - cloudsync_payload_group_abandon(data, &batch); + // Close the last PK group: flushed on success, rolled back after a failure. + if (fail_rc == DBRES_OK) { + fail_rc = cloudsync_payload_group_flush(data, &batch); + if (fail_rc == DBRES_OK) applied = (int)nrows; } else { - int entries = 0; - bool contained = true; - int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); - if (flush_rc == DBRES_POLICY_DENIED && contained && group_start) { - cloudsync_denied_runs_add(&denied_runs, group_start, group_avail, group_rows, entries); - } - if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) stop = true; - } - - // The receive checkpoint is the last row of the payload, whatever a retry decodes. - int64_t payload_last_db_version = decoded_context.db_version; - int64_t payload_last_seq = decoded_context.seq; - - // Retry denied runs now that every other row is in: a policy that depends on rows - // later in the payload lets them through on a later pass. Repeat while a pass makes - // progress, so a chain of such dependencies resolves; a run still denied after the - // last pass is final. Rows that already applied are re-merged as no-ops. A denied - // group left no trace (its savepoint was rolled back), so it retries cleanly. - for (int pass = 0; !stop && denied_runs.count > 0 && pass < 32; pass++) { - cloudsync_denied_runs pending = denied_runs; - denied_runs = (cloudsync_denied_runs){0}; - bool progress = false; - for (int r = 0; r < pending.count && !stop; r++) { - cloudsync_denied_run *run = &pending.runs[r]; - const char *p = run->start; - size_t avail = run->avail; - int denied_now = 0; - bool decode_failed = false; - for (uint32_t k = 0; k < run->rows && !stop; k++) { - size_t seek = 0; - if (pk_decode((char *)p, avail, ncols, &seek, data->skip_decode_idx, cloudsync_payload_decode_callback, &decoded_context) == -1) { - decode_failed = true; - break; - } - if (batch.count > 0 && cloudsync_payload_row_is_block(&decoded_context)) { - int entries = 0; - bool contained = true; - int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); - if (flush_rc == DBRES_POLICY_DENIED && contained) { - denied_now += entries; - } else if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { - stop = true; - break; - } - } - cloudsync_payload_group_open(data, &batch); - bool contained = true; - int step_rc = cloudsync_payload_apply_row(data, vm, &contained); - if (step_rc == DBRES_POLICY_DENIED && contained) { - denied_now++; - } else if (step_rc != DBRES_DONE && cloudsync_apply_note_failure(data, &outcome, step_rc, 1, contained)) { - stop = true; - } - p += seek; - avail -= seek; - dbvm_reset(vm); - } - if (!stop && !decode_failed) { - int entries = 0; - bool contained = true; - int flush_rc = cloudsync_payload_group_flush(data, &batch, &entries, &contained); - if (flush_rc == DBRES_POLICY_DENIED && contained) { - denied_now += entries; - } else if (flush_rc != DBRES_OK && cloudsync_apply_note_failure(data, &outcome, flush_rc, entries, contained)) { - stop = true; - } - } else { - cloudsync_payload_group_abandon(data, &batch); - } - if (stop || decode_failed) { - // Keep the run counted as it was; nothing more can be retried. - cloudsync_denied_runs_add(&denied_runs, run->start, run->avail, run->rows, run->entries); - continue; - } - outcome.denied -= run->entries; - if (denied_now > 0) { - outcome.denied += denied_now; - cloudsync_denied_runs_add(&denied_runs, run->start, run->avail, run->rows, denied_now); - } - if (denied_now < run->entries) progress = true; - } - cloudsync_memory_free(pending.runs); - if (!progress) break; + cloudsync_payload_group_abandon(data, &batch); } - if (outcome.denied < 0) outcome.denied = 0; data->pending_batch = NULL; - rc = DBRES_OK; + char fail_message[1024] = {0}; + int fail_sqlstate = 0; + if (fail_rc != DBRES_OK) { + snprintf(fail_message, sizeof(fail_message), "%s", cloudsync_errmsg(data)); + fail_sqlstate = cloudsync_sqlstate(data); + } + + // Released after a failure too: the failed group is already rolled back, and the + // groups before it are kept. The receive checkpoint does not move, so they are + // delivered again and re-merge as no-ops. if (in_savepoint) { int rc1 = database_commit_savepoint(data, "cloudsync_payload_apply"); - if (rc1 != DBRES_OK && outcome.fatal_rc == DBRES_OK) { - outcome.fatal_rc = rc1; - outcome.fatal_sqlstate = cloudsync_sqlstate(data); - snprintf(outcome.fatal_message, sizeof(outcome.fatal_message), "%s", cloudsync_errmsg(data)); + if (rc1 != DBRES_OK) { + applied = applied_at_savepoint; + if (fail_rc == DBRES_OK) { + fail_rc = rc1; + snprintf(fail_message, sizeof(fail_message), "%s", cloudsync_errmsg(data)); + fail_sqlstate = cloudsync_sqlstate(data); + } } } - if (outcome.fatal_rc != DBRES_OK) { - rc = outcome.fatal_rc; + cloudsync_apply_stats_add(data, applied); + + rc = fail_rc; + if (rc != DBRES_OK) { // The captured message already carries the database error; clear it first so // it is not appended to itself a second time. cloudsync_reset_error(data); - cloudsync_set_error(data, outcome.fatal_message[0] ? outcome.fatal_message : "Unable to apply payload changes", rc); - cloudsync_set_sqlstate(data, outcome.fatal_sqlstate); - } - - { - int applied = (rc == DBRES_OK) ? (int)nrows - outcome.denied - outcome.failed : 0; - cloudsync_apply_stats_add(data, (applied > 0) ? applied : 0, outcome.failed); - } - - // Policy denials and failed writes that survive the retry are permanent: delivering - // them again fails the same way. So the cursor still advances past them — holding it - // back would re-deliver them forever and stall every later change behind them. They - // are reported instead: a warning per failed write (receive.failed on the network - // path) and one summary warning for denials. Only a transient or uncontained failure - // fails the apply and keeps the cursor in place. - if (rc == DBRES_OK && outcome.denied > 0) { - char warning[256]; - snprintf(warning, sizeof(warning), "skipped %d received change%s denied by a row-level security policy", - outcome.denied, (outcome.denied == 1) ? "" : "s"); - database_log_warning(data, warning); - } - - if (rc == DBRES_OK) { + cloudsync_set_error(data, fail_message[0] ? fail_message : "Unable to apply payload changes", rc); + cloudsync_set_sqlstate(data, fail_sqlstate); + } else { // Record the last applied (db_version, seq) and advance the receive cursor // once, gated on the caller-supplied checkpoint. A non-final chunk passes // CLOUDSYNC_CHECKPOINT_NONE so the cursor never lands mid-db_version. - if (payload_last_db_version > data->apply_last_db_version || - (payload_last_db_version == data->apply_last_db_version && payload_last_seq > data->apply_last_seq)) { - data->apply_last_db_version = payload_last_db_version; - data->apply_last_seq = payload_last_seq; + if (decoded_context.db_version > data->apply_last_db_version || + (decoded_context.db_version == data->apply_last_db_version && decoded_context.seq > data->apply_last_seq)) { + data->apply_last_db_version = decoded_context.db_version; + data->apply_last_seq = decoded_context.seq; } cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); } @@ -4813,7 +4571,6 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b // cleanup memory if (clone) cloudsync_memory_free(clone); - cloudsync_memory_free(denied_runs.runs); // error already saved in (save last error) if (rc != DBRES_OK) return rc; diff --git a/src/cloudsync.h b/src/cloudsync.h index f69ffa16..47ed5e38 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -117,16 +117,11 @@ void cloudsync_reset_error (cloudsync_context *data); void cloudsync_set_sqlstate (cloudsync_context *data, int sqlstate); int cloudsync_sqlstate (cloudsync_context *data); -// Entries applied and entries skipped because their write failed. Both counts accumulate -// across a receive drain (reset once before it) so an early chunk is still reported by -// the call that finishes the drain. The applied count is tracked here rather than -// derived from the apply return value, which reports the payload's entry count (skipped -// ones included) as part of the SQL surface. cloudsync_apply_failure_message returns the -// first skipped failure's message since the last reset, or NULL. +// Entries applied, accumulated across a receive drain (reset once before it) so an early +// chunk is still reported by the call that finishes the drain. A payload that fails +// still counts the changes it applied before its error, when they were kept. void cloudsync_apply_stats_reset (cloudsync_context *data); int cloudsync_apply_rows_count (cloudsync_context *data); -int cloudsync_apply_failed_count (cloudsync_context *data); -const char *cloudsync_apply_failure_message (cloudsync_context *data); int cloudsync_commit_hook (void *ctx); void cloudsync_rollback_hook (void *ctx); void cloudsync_set_schema (cloudsync_context *data, const char *schema); diff --git a/src/network/network.c b/src/network/network.c index d8fe74e4..0c900b55 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -2097,9 +2097,9 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync int64_t chunk_bytes = 0; rc = network_apply_check_chunk(context, chunk_json, chunk_json_len, final_chunk, &chunk_rows, err_out, &chunk_bytes); + bytes_total += chunk_bytes; if (rc == SQLITE_OK) { rows_total += chunk_rows; - bytes_total += chunk_bytes; chunks_total++; delivered = true; last_cursor = json_extract_int(chunk_json, chunk_json_len, "cursor", last_cursor); @@ -2134,6 +2134,7 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync int64_t chunk_bytes = 0; rc = network_apply_check_chunk(context, check_json, check_json_len, final_chunk, &chunk_rows, err_out, &chunk_bytes); + bytes_total = chunk_bytes; if (rc == SQLITE_OK && !final_chunk && next_cursor < 0) { // Symmetric with the chunks-array path: a non-final response // with no resumable cursor would otherwise silently drop the @@ -2142,7 +2143,6 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync rc = SQLITE_ERROR; } else if (rc == SQLITE_OK) { rows_total = chunk_rows; - bytes_total = chunk_bytes; chunks_total = 1; delivered = true; more_pending = !final_chunk && next_cursor >= 0; @@ -2167,12 +2167,16 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync if (out) { out->page_delivered = true; out->more_pending = more_pending; - out->bytes_received = bytes_total; - out->chunks_received = chunks_total; } } else { if (pnrows) *pnrows = 0; } + // Transfer accounting survives a failed chunk: the chunks applied before it and + // the bytes received, including the failing chunk's. + if (out) { + out->bytes_received = bytes_total; + out->chunks_received = chunks_total; + } if (data_json) cloudsync_memory_free(data_json); // failures.check may appear in either shape; extract opportunistically. if (out) { @@ -2210,9 +2214,7 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync // Result of a receive drain (see network_drain_changes). typedef struct { - int rows; // cumulative rows applied across the drain - int failed; // payload entries skipped because their write failed - char *failed_err; // owned; message of the first skipped failure, or NULL + int rows; // cumulative rows applied across the drain, a failed chunk's prefix included int chunks; // payload chunks applied this drain int64_t bytes; // serialized payload bytes received this drain bool complete; // true iff the receive stream is fully drained (nothing pending) @@ -2221,36 +2223,30 @@ typedef struct { } drain_result; // Builds the "receive":{...} member shared by cloudsync_network_sync and -// cloudsync_network_receive_changes. receive_err and check_failure_json are optional; -// "failedError" is emitted only with a skipped failure's message. Returns a malloc'd -// string, or NULL when out of memory. +// cloudsync_network_receive_changes. receive_err and check_failure_json are optional. +// Returns a malloc'd string, or NULL when out of memory. static char *network_receive_json (int rows, const drain_result *dr, const char *tables, const char *receive_err, const char *check_failure_json) { char *escaped_err = receive_err ? json_escape_string(receive_err) : NULL; - char *escaped_failed = dr->failed_err ? json_escape_string(dr->failed_err) : NULL; char *error_part = escaped_err ? cloudsync_memory_mprintf(",\"error\":\"%s\"", escaped_err) : NULL; - char *failed_error_part = escaped_failed ? cloudsync_memory_mprintf(",\"failedError\":\"%s\"", escaped_failed) : NULL; char *last_failure_part = check_failure_json ? cloudsync_memory_mprintf(",\"lastFailure\":%s", check_failure_json) : NULL; char *json = NULL; - if ((!receive_err || error_part) && (!dr->failed_err || failed_error_part) && (!check_failure_json || last_failure_part)) { + if ((!receive_err || error_part) && (!check_failure_json || last_failure_part)) { json = cloudsync_memory_mprintf( - "\"receive\":{\"rows\":%d,\"failed\":%d%s,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s%s%s}", - rows, dr->failed, failed_error_part ? failed_error_part : "", tables ? tables : "[]", + "\"receive\":{\"rows\":%d,\"tables\":%s,\"chunks\":%d,\"bytes\":%lld,\"complete\":%s%s%s}", + rows, tables ? tables : "[]", dr->chunks, (long long)dr->bytes, dr->complete ? "true" : "false", error_part ? error_part : "", last_failure_part ? last_failure_part : ""); } if (escaped_err) cloudsync_memory_free(escaped_err); - if (escaped_failed) cloudsync_memory_free(escaped_failed); if (error_part) cloudsync_memory_free(error_part); - if (failed_error_part) cloudsync_memory_free(failed_error_part); if (last_failure_part) cloudsync_memory_free(last_failure_part); return json; } #ifdef CLOUDSYNC_UNITTEST -char *network_test_receive_json (int rows, int failed, const char *failed_err, bool complete, - const char *receive_err, const char *check_failure_json) { - drain_result dr = {.rows = rows, .failed = failed, .failed_err = (char *)failed_err, .chunks = 1, .bytes = 10, .complete = complete}; +char *network_test_receive_json (int rows, bool complete, const char *receive_err, const char *check_failure_json) { + drain_result dr = {.rows = rows, .chunks = 1, .bytes = 10, .complete = complete}; return network_receive_json(rows, &dr, "[\"t\"]", receive_err, check_failure_json); } #endif @@ -2272,8 +2268,8 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, int64_t drain_prev_dbv = cloudsync_dbversion(data); sr->defer_tables = true; - // Apply counts accumulate on the context across every chunk of this drain, so a - // skipped write in an early chunk is still reported by the call that finishes it. + // Applied rows accumulate on the context across every chunk of this drain, including + // the changes a failing chunk applied before its error. cloudsync_apply_stats_reset(data); int ntries = 0; // counts only "nothing ready" (202) polls @@ -2302,12 +2298,12 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, int nrows = 0; // required out-param; the drain total comes from the context rc = cloudsync_network_check_internal(context, &nrows, sr, &receive_err, request_max_chunks); + bytes_total += sr->bytes_received; // a staged (incomplete) fragment applies 0 rows + nchunks += sr->chunks_received; // a receive error (network or apply) won't fix itself across retries if (rc != SQLITE_OK) { complete = false; break; } if (sr->page_delivered) { - bytes_total += sr->bytes_received; // a staged (incomplete) fragment applies 0 rows - nchunks += sr->chunks_received; complete = !sr->more_pending; // reflects whether the stream is finished if (!sr->more_pending) break; // final batch -> drained if (max_chunks > 0 && nchunks >= max_chunks) break; // caller cap: more pending @@ -2334,20 +2330,14 @@ static int network_drain_changes (sqlite3_context *context, sync_result *sr, sqlite3_sleep(wait_ms); } - // Compute the affected-tables union once, over the whole drain window. - // Report rows actually written, not payload entries: skipped writes are excluded. - // The apply return value still counts payload entries, which is a tested part of the - // SQL surface, so the accurate count is accumulated on the context instead. + // Compute the affected-tables union once, over the whole drain window. Changes + // applied before a failure are reported too: they were written. int applied_total = cloudsync_apply_rows_count(data); - - if (!receive_err && rc == SQLITE_OK && applied_total > 0) { + if (applied_total > 0) { sr->tables_json = network_get_affected_tables(db, drain_prev_dbv); } dr->rows = applied_total; - dr->failed = cloudsync_apply_failed_count(data); - const char *failure = cloudsync_apply_failure_message(data); - dr->failed_err = failure ? cloudsync_string_dup(failure) : NULL; dr->chunks = nchunks; dr->bytes = bytes_total; dr->complete = complete; @@ -2376,11 +2366,7 @@ void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retr if (rc != SQLITE_OK && !receive_err) { receive_err = cloudsync_string_dup("receive failed"); } - if (receive_err) { - rc = SQLITE_OK; - nrows_total = 0; - if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } - } + if (receive_err) rc = SQLITE_OK; const char *tables = sr.tables_json ? sr.tables_json : "[]"; const char *status = sr.status ? sr.status : "error"; @@ -2402,7 +2388,6 @@ void cloudsync_network_sync (sqlite3_context *context, int wait_ms, int max_retr if (buf) sqlite3_result_text(context, buf, -1, cloudsync_memory_free); else sqlite3_result_error_nomem(context); - if (dr.failed_err) cloudsync_memory_free(dr.failed_err); if (receive_err) cloudsync_memory_free(receive_err); if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.apply_failure_json) cloudsync_memory_free(sr.apply_failure_json); @@ -2445,7 +2430,6 @@ static void network_receive_changes_impl (sqlite3_context *context, int max_chun if (rc != SQLITE_OK && !receive_err) { if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); - if (dr.failed_err) cloudsync_memory_free(dr.failed_err); return; } @@ -2466,25 +2450,17 @@ static void network_receive_changes_impl (sqlite3_context *context, int max_chun if (code) cloudsync_memory_free(code); if (message) cloudsync_memory_free(message); if (receive_err) cloudsync_memory_free(receive_err); - if (dr.failed_err) cloudsync_memory_free(dr.failed_err); if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); return; } - // Apply errors → structured JSON with receive.error - if (receive_err) { - nrows = 0; - if (sr.tables_json) { cloudsync_memory_free(sr.tables_json); sr.tables_json = NULL; } - } - const char *tables = sr.tables_json ? sr.tables_json : "[]"; char *recv_part = network_receive_json(nrows, &dr, tables, receive_err, sr.check_failure_json); char *buf = recv_part ? cloudsync_memory_mprintf("{%s}", recv_part) : NULL; if (recv_part) cloudsync_memory_free(recv_part); if (buf) sqlite3_result_text(context, buf, -1, cloudsync_memory_free); else sqlite3_result_error_nomem(context); - if (dr.failed_err) cloudsync_memory_free(dr.failed_err); if (receive_err) cloudsync_memory_free(receive_err); if (sr.tables_json) cloudsync_memory_free(sr.tables_json); if (sr.check_failure_json) cloudsync_memory_free(sr.check_failure_json); diff --git a/src/postgresql/cloudsync_postgresql.c b/src/postgresql/cloudsync_postgresql.c index 1ed17f7b..7834561b 100644 --- a/src/postgresql/cloudsync_postgresql.c +++ b/src/postgresql/cloudsync_postgresql.c @@ -3476,9 +3476,8 @@ Datum cloudsync_changes_insert_trigger (PG_FUNCTION_ARGS) { rc = merge_insert (data, table, VARDATA_ANY(insert_pk), insert_pk_len, insert_cl, insert_name, col_value, insert_col_version, insert_db_version, VARDATA_ANY(insert_site_id), insert_site_id_len, insert_seq, &rowid); } if (rc == DBRES_POLICY_DENIED) { - // Keep a row-level security denial recognizable to the caller that inserted - // into cloudsync_changes (cloudsync_payload_apply): it is skipped and - // counted, unlike other merge failures. + // Keep a row-level security denial's SQLSTATE: cloudsync_payload_apply stops + // on it, as on every other failed write, and raises it to the caller. ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Error during merge_insert: %s", database_errmsg(data)))); } diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c index 436f17da..6582b310 100644 --- a/src/postgresql/database_postgresql.c +++ b/src/postgresql/database_postgresql.c @@ -562,10 +562,8 @@ char *sql_build_insert_missing_pks_query(const char *schema, const char *table_n // MARK: - HELPER FUNCTIONS - -// Map a PostgreSQL SQLSTATE to DBRES. Only the distinction between a transient failure -// (worth retrying later) and a failure of the data itself matters to callers: a payload -// apply skips a change whose write fails, but must never skip one that failed only -// because of a lock, a deadlock, a cancel or a resource shortage. +// Map a PostgreSQL SQLSTATE to the closest DBRES code. The SQLSTATE itself is kept +// separately (cloudsync_set_sqlstate) and is what the caller sees. static int map_sqlerrcode (int sqlerrcode) { switch (sqlerrcode) { case ERRCODE_INSUFFICIENT_PRIVILEGE: // a policy denial is recognized by the caller diff --git a/test/network_unit.c b/test/network_unit.c index 2758b1a2..4d98845c 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -198,7 +198,7 @@ static bool test_json_envelope(void) { cloudsync_memory_free(url); return ok; } -extern char *network_test_receive_json(int, int, const char *, bool, const char *, const char *); +extern char *network_test_receive_json(int, bool, const char *, const char *); static bool receive_json_is(char *json, const char *expected) { bool ok = json && strcmp(json, expected) == 0; if (!ok) printf("\n got: %s\n expected: %s\n", json ? json : "(null)", expected); @@ -206,12 +206,11 @@ static bool receive_json_is(char *json, const char *expected) { return ok; } static bool test_receive_json(void) { - bool ok = receive_json_is(network_test_receive_json(3, 0, NULL, true, NULL, NULL), - "\"receive\":{\"rows\":3,\"failed\":0,\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":true}"); - ok = receive_json_is(network_test_receive_json(2, 1, "rejected \"here\"", true, NULL, NULL), - "\"receive\":{\"rows\":2,\"failed\":1,\"failedError\":\"rejected \\\"here\\\"\",\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":true}") && ok; - ok = receive_json_is(network_test_receive_json(0, 0, NULL, false, "boom", "{\"code\":\"x\"}"), - "\"receive\":{\"rows\":0,\"failed\":0,\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":false,\"error\":\"boom\",\"lastFailure\":{\"code\":\"x\"}}") && ok; + bool ok = receive_json_is(network_test_receive_json(3, true, NULL, NULL), + "\"receive\":{\"rows\":3,\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":true}"); + // a failed receive still reports the changes applied before the error + ok = receive_json_is(network_test_receive_json(2, false, "rejected \"here\"", "{\"code\":\"x\"}"), + "\"receive\":{\"rows\":2,\"tables\":[\"t\"],\"chunks\":1,\"bytes\":10,\"complete\":false,\"error\":\"rejected \\\"here\\\"\",\"lastFailure\":{\"code\":\"x\"}}") && ok; return ok; } static bool test_unicode(void) { @@ -260,7 +259,7 @@ int main(void) { check("JSON keys only match root object members:", test_json_scope()); check("Gateway data envelope is unwrapped before scoped lookups:", test_json_envelope()); check("JSON Unicode, surrogate pairs and malformed escapes:", test_unicode()); - check("receive JSON members (failed, failedError, error, lastFailure):", test_receive_json()); + check("receive JSON members (rows, error, lastFailure):", test_receive_json()); printf("\nNetwork unit tests\n"); check("optimistic/confirmed version folds latest-valid (allows rollback):", test_optimistic_version_rollback()); check("non-buffer response is a no-op:", test_non_buffer_is_noop()); diff --git a/test/postgresql/27_rls_batch_merge.sql b/test/postgresql/27_rls_batch_merge.sql index 0e03d52b..aa692118 100644 --- a/test/postgresql/27_rls_batch_merge.sql +++ b/test/postgresql/27_rls_batch_merge.sql @@ -282,29 +282,29 @@ SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_d SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset +\set apply_5_state :SQLSTATE -- Reconnect for clean state after expected RLS denial \connect cloudsync_test_27_b \ir helper_psql_conn_setup.sql --- 1 row × 3 non-PK columns = 3 entries (returned even if denied) -SELECT (:apply_5::int = 3) AS apply_5_ok \gset +-- the denial raises 42501 and rolls the statement back +SELECT (:'apply_5_state' = '42501') AS apply_5_ok \gset \if :apply_5_ok -\echo [PASS] (:testid) RLS auth: denied apply returned :apply_5 +\echo [PASS] (:testid) RLS auth: denied apply raised 42501 \else -\echo [FAIL] (:testid) RLS auth: denied apply returned :apply_5 (expected 3) +\echo [FAIL] (:testid) RLS auth: denied apply ended with SQLSTATE :apply_5_state (expected 42501) SELECT (:fail::int + 1) AS fail \gset \endif --- A denial is permanent, so the cursor must still move past those rows: holding it --- back re-delivers them on every check forever, and in a chunked batch the final --- chunk would checkpoint past them anyway, dropping them with no report. +-- The denied rows were never applied, so the cursor must not move past them: a +-- redelivery once the policy allows them applies them. SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after_denied \gset -SELECT (:ckpt_after_denied::bigint > :ckpt_before_denied::bigint) AS ckpt_ok \gset +SELECT (:ckpt_after_denied::bigint = :ckpt_before_denied::bigint) AS ckpt_ok \gset \if :ckpt_ok -\echo [PASS] (:testid) RLS auth: denied apply still advanced the receive checkpoint +\echo [PASS] (:testid) RLS auth: denied apply left the receive checkpoint in place \else -\echo [FAIL] (:testid) RLS auth: denied apply left the checkpoint at :ckpt_after_denied (expected > :ckpt_before_denied) +\echo [FAIL] (:testid) RLS auth: denied apply moved the checkpoint to :ckpt_after_denied (expected :ckpt_before_denied) SELECT (:fail::int + 1) AS fail \gset \endif diff --git a/test/postgresql/29_rls_multicol.sql b/test/postgresql/29_rls_multicol.sql index de8f3047..d21e3dfe 100644 --- a/test/postgresql/29_rls_multicol.sql +++ b/test/postgresql/29_rls_multicol.sql @@ -229,17 +229,18 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_4 FROM cloudsync_changes \gset SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; SELECT cloudsync_payload_apply(decode(:'payload_hex_4', 'hex')) AS apply_4 \gset +\set apply_4_state :SQLSTATE -- Reconnect for clean state after expected RLS denial \connect cloudsync_test_29_b \ir helper_psql_conn_setup.sql --- 1 row × 5 columns = 5 entries in payload (returned even if denied) -SELECT (:apply_4::int = 5) AS apply_4_ok \gset +-- the denial raises 42501 and rolls the statement back +SELECT (:'apply_4_state' = '42501') AS apply_4_ok \gset \if :apply_4_ok -\echo [PASS] (:testid) RLS multicol auth: denied insert apply returned :apply_4 +\echo [PASS] (:testid) RLS multicol auth: denied insert apply raised 42501 \else -\echo [FAIL] (:testid) RLS multicol auth: denied insert apply returned :apply_4 (expected 5) +\echo [FAIL] (:testid) RLS multicol auth: denied insert apply ended with SQLSTATE :apply_4_state (expected 42501) SELECT (:fail::int + 1) AS fail \gset \endif @@ -314,17 +315,18 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_6 FROM cloudsync_changes \gset SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; SELECT cloudsync_payload_apply(decode(:'payload_hex_6', 'hex')) AS apply_6 \gset +\set apply_6_state :SQLSTATE -- Reconnect for clean state after expected RLS denial \connect cloudsync_test_29_b \ir helper_psql_conn_setup.sql --- 1 row × 2 changed columns (title, priority) = 2 entries in payload -SELECT (:apply_6::int = 2) AS apply_6_ok \gset +-- the denial raises 42501 and rolls the statement back +SELECT (:'apply_6_state' = '42501') AS apply_6_ok \gset \if :apply_6_ok -\echo [PASS] (:testid) RLS multicol auth: denied update apply returned :apply_6 +\echo [PASS] (:testid) RLS multicol auth: denied update apply raised 42501 \else -\echo [FAIL] (:testid) RLS multicol auth: denied update apply returned :apply_6 (expected 2) +\echo [FAIL] (:testid) RLS multicol auth: denied update apply ended with SQLSTATE :apply_6_state (expected 42501) SELECT (:fail::int + 1) AS fail \gset \endif @@ -339,7 +341,7 @@ SELECT (:fail::int + 1) AS fail \gset \endif -- ============================================================ --- Test 7: Mixed payload — own + other user's rows (per-PK savepoint) +-- Test 7: Mixed payload — own + other user's rows (stops at the denied row) -- ============================================================ \connect cloudsync_test_29_a \ir helper_psql_conn_setup.sql @@ -353,35 +355,36 @@ WHERE site_id = cloudsync_siteid() SELECT COALESCE(max(db_version), 0) AS max_dbv_7 FROM cloudsync_changes \gset --- Apply as test_rls_user with USER1 identity --- Per-PK savepoint: t6 (USER1) should succeed, t7 (USER2) should be denied +-- Apply as test_rls_user with USER1 identity: t7 (USER2) is denied, which fails the +-- statement, so t6 (USER1) before it is rolled back with it \connect cloudsync_test_29_b \ir helper_psql_conn_setup.sql SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; SELECT cloudsync_payload_apply(decode(:'payload_hex_7', 'hex')) AS apply_7 \gset +\set apply_7_state :SQLSTATE -- Reconnect for clean verification as superuser \connect cloudsync_test_29_b \ir helper_psql_conn_setup.sql --- 2 rows × 5 columns = 10 entries in payload -SELECT (:apply_7::int = 10) AS apply_7_ok \gset +-- the denial raises 42501 and rolls the statement back +SELECT (:'apply_7_state' = '42501') AS apply_7_ok \gset \if :apply_7_ok -\echo [PASS] (:testid) RLS multicol auth: mixed payload apply returned :apply_7 +\echo [PASS] (:testid) RLS multicol auth: mixed payload apply raised 42501 \else -\echo [FAIL] (:testid) RLS multicol auth: mixed payload apply returned :apply_7 (expected 10) +\echo [FAIL] (:testid) RLS multicol auth: mixed payload apply ended with SQLSTATE :apply_7_state (expected 42501) SELECT (:fail::int + 1) AS fail \gset \endif --- t6 (own row) should exist, t7 (other's row) should NOT -SELECT COUNT(*) AS t6_exists FROM tasks WHERE id = 't6' AND user_id = :'USER1'::UUID AND title = 'Task 6' \gset +-- neither row is kept: the failed statement rolled back as a whole +SELECT COUNT(*) AS t6_exists FROM tasks WHERE id = 't6' \gset SELECT COUNT(*) AS t7_exists FROM tasks WHERE id = 't7' \gset -SELECT (:t6_exists::int = 1 AND :t7_exists::int = 0) AS test7_ok \gset +SELECT (:t6_exists::int = 0 AND :t7_exists::int = 0) AS test7_ok \gset \if :test7_ok -\echo [PASS] (:testid) RLS multicol auth: mixed payload — per-PK savepoint isolation +\echo [PASS] (:testid) RLS multicol auth: mixed payload — failed statement kept nothing \else -\echo [FAIL] (:testid) RLS multicol auth: mixed payload — t6=:t6_exists (expect 1) t7=:t7_exists (expect 0) +\echo [FAIL] (:testid) RLS multicol auth: mixed payload — t6=:t6_exists (expect 0) t7=:t7_exists (expect 0) SELECT (:fail::int + 1) AS fail \gset \endif diff --git a/test/postgresql/57_audit_regressions.sql b/test/postgresql/57_audit_regressions.sql index 1c617ce6..bf73bea8 100644 --- a/test/postgresql/57_audit_regressions.sql +++ b/test/postgresql/57_audit_regressions.sql @@ -1,6 +1,6 @@ --- Audit: a change whose write fails on its data is skipped and reported, never --- silently dropped and never allowed to stall the cursor; transient failures are --- covered by 39_concurrent_write_apply.sql. +-- Audit: a change whose write fails stops the apply with its own SQLSTATE; nothing is +-- dropped silently and the checkpoint stays where it was until the same payload applies; +-- transient failures are covered by 39_concurrent_write_apply.sql. \set ON_ERROR_STOP on \connect postgres DROP DATABASE IF EXISTS cloudsync_audit_source; @@ -38,31 +38,38 @@ BEGIN RETURN NEW; END $$; CREATE TRIGGER deny_audit BEFORE INSERT ON t FOR EACH ROW EXECUTE FUNCTION deny_audit_row(); -SET client_min_messages = error; -- each skipped change raises a WARNING by design -- The payload is read from a table on purpose: the apply's internal savepoints must -- not disturb the resource owner of the scan feeding it. DO $$ -DECLARE denied INTEGER; +DECLARE denied INTEGER; state TEXT; msg TEXT; BEGIN FOR denied IN 1..3 LOOP DELETE FROM t; DELETE FROM t_cloudsync; DELETE FROM cloudsync_settings WHERE key IN ('check_dbversion','check_seq'); PERFORM set_config('audit.denied_id', denied::TEXT, false); - PERFORM cloudsync_payload_apply(data) FROM audit_payload; - IF (SELECT count(*) FROM t) <> 2 THEN - RAISE EXCEPTION 'Row % failing must not discard the other rows (found %)', denied, (SELECT count(*) FROM t); + state := NULL; + BEGIN PERFORM cloudsync_payload_apply(data) FROM audit_payload; + EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE, msg = MESSAGE_TEXT; END; + IF state IS DISTINCT FROM 'P0001' OR msg NOT LIKE '%audit write denied%' THEN + RAISE EXCEPTION 'Row % failing surfaced as SQLSTATE % (%)', denied, coalesce(state, 'none'), msg; END IF; - IF EXISTS (SELECT FROM t WHERE id = denied::TEXT) THEN - RAISE EXCEPTION 'Failed row % was written', denied; + -- the failed statement is rolled back whole, metadata and checkpoint included + IF EXISTS (SELECT FROM t) OR EXISTS (SELECT FROM t_cloudsync) THEN + RAISE EXCEPTION 'Row % failing left rows or metadata behind', denied; END IF; - IF coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) = 0 THEN - RAISE EXCEPTION 'A skipped failure at row % must still advance the checkpoint', denied; + IF coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) <> 0 THEN + RAISE EXCEPTION 'Row % failing moved the checkpoint', denied; + END IF; + -- once the write can succeed, the same payload applies in full + PERFORM set_config('audit.denied_id', '', false); + PERFORM cloudsync_payload_apply(data) FROM audit_payload; + IF (SELECT count(*) FROM t) <> 3 OR coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) = 0 THEN + RAISE EXCEPTION 'Row % redelivered: expected 3 rows and an advanced checkpoint', denied; END IF; END LOOP; END $$; -SET client_min_messages = warning; -\echo [PASS] (57-audit) first, middle and final write failures are skipped without discarding the rest or stalling the checkpoint +\echo [PASS] (57-audit) first, middle and final write failures stop the apply with their error and apply when redelivered CREATE FUNCTION raise_sqlstate() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION USING ERRCODE = TG_ARGV[0], MESSAGE = 'simulated ' || TG_ARGV[0]; END $$; @@ -71,9 +78,8 @@ SELECT cloudsync_init('plain_audit') \gset CREATE TABLE revived(id TEXT PRIMARY KEY NOT NULL, a TEXT, b TEXT); SELECT cloudsync_init('revived') \gset --- A payload apply keeps the SQLSTATE too, and does not skip these failures: a serialization --- failure is transient and a missing privilege is fixed by a GRANT, so both fail the --- apply and leave the checkpoint where it was. +-- A payload apply keeps the SQLSTATE of transient and privilege failures too, and leaves +-- the checkpoint where it was. CREATE TRIGGER fail_apply BEFORE INSERT ON plain_audit FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('40001'); CREATE TEMP TABLE plain_payload(data BYTEA); INSERT INTO plain_payload VALUES (decode(:'plain_payload_hex','hex')); @@ -88,7 +94,7 @@ BEGIN BEGIN PERFORM cloudsync_payload_apply(data) FROM plain_payload; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE, msg = MESSAGE_TEXT; END; IF state IS DISTINCT FROM code THEN - RAISE EXCEPTION 'Apply failing with % surfaced as SQLSTATE % (%)', code, coalesce(state, 'none: the failure was skipped'), msg; + RAISE EXCEPTION 'Apply failing with % surfaced as SQLSTATE % (%)', code, coalesce(state, 'none'), msg; END IF; IF EXISTS (SELECT FROM plain_audit) OR coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'),0) <> 0 THEN RAISE EXCEPTION 'Apply failing with % wrote rows or moved the checkpoint', code; @@ -96,16 +102,20 @@ BEGIN END LOOP; END $$; DROP TRIGGER fail_apply ON plain_audit; -\echo [PASS] (57-audit) apply keeps the SQLSTATE of transient and privilege failures and does not skip them +\echo [PASS] (57-audit) apply keeps the SQLSTATE of transient and privilege failures -- A resurrected row (sentinel plus columns) whose write fails leaves nothing behind — -- no sentinel, no zeroed clocks — so once the cause is gone the same payload creates it. CREATE TRIGGER fail_revived BEFORE INSERT ON revived FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('23514'); CREATE TEMP TABLE revived_payload(data BYTEA); INSERT INTO revived_payload VALUES (decode(:'revived_payload_hex','hex')); -SET client_min_messages = error; -SELECT cloudsync_payload_apply(data) FROM revived_payload \gset -SET client_min_messages = warning; +DO $$ +DECLARE state TEXT; +BEGIN + BEGIN PERFORM cloudsync_payload_apply(data) FROM revived_payload; + EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS state = RETURNED_SQLSTATE; END; + IF state IS DISTINCT FROM '23514' THEN RAISE EXCEPTION 'Rejected resurrected row surfaced as SQLSTATE %', coalesce(state, 'none'); END IF; +END $$; DROP TRIGGER fail_revived ON revived; DO $$ BEGIN IF EXISTS (SELECT FROM revived) THEN RAISE EXCEPTION 'Rejected resurrected row was written'; END IF; @@ -119,28 +129,28 @@ DO $$ BEGIN END $$; \echo [PASS] (57-audit) a failed resurrected row leaves no metadata and applies when delivered again --- Savepoints opened by the apply must leave the caller's snapshots intact, including --- when a group is rolled back and when the caller's own subtransaction is then aborted: --- statements after the apply keep running (an unbalanced active-snapshot stack trips an --- assertion in EnsurePortalSnapshotExists and takes the backend down). +-- Savepoints opened by the apply must leave the caller's snapshots intact when a group is +-- rolled back and the caller's own subtransaction then catches the error: statements +-- after the apply keep running (an unbalanced active-snapshot stack trips an assertion in +-- EnsurePortalSnapshotExists and takes the backend down). DELETE FROM revived; DELETE FROM revived_cloudsync; CREATE TRIGGER fail_revived BEFORE INSERT ON revived FOR EACH ROW EXECUTE FUNCTION raise_sqlstate('23514'); -SET client_min_messages = error; DO $$ DECLARE n INTEGER; BEGIN BEGIN PERFORM cloudsync_payload_apply(data) FROM revived_payload; - PERFORM count(*) FROM revived; - RAISE EXCEPTION 'abort the caller subtransaction'; - EXCEPTION WHEN raise_exception THEN NULL; + RAISE EXCEPTION 'the apply must fail'; + EXCEPTION WHEN check_violation THEN NULL; END; SELECT count(*) INTO n FROM revived_payload; - PERFORM cloudsync_payload_apply(data) FROM revived_payload; + BEGIN + PERFORM cloudsync_payload_apply(data) FROM revived_payload; + EXCEPTION WHEN check_violation THEN NULL; + END; SELECT count(*) INTO n FROM revived; IF n <> 0 THEN RAISE EXCEPTION 'Rejected resurrected row was written'; END IF; END $$; -SET client_min_messages = warning; DROP TRIGGER fail_revived ON revived; \echo [PASS] (57-audit) apply savepoints leave the snapshots of the caller intact across rollbacks diff --git a/test/postgresql/58_v3_denied_checkpoint.sql b/test/postgresql/58_v3_denied_checkpoint.sql index 1e0d7d34..1c05651d 100644 --- a/test/postgresql/58_v3_denied_checkpoint.sql +++ b/test/postgresql/58_v3_denied_checkpoint.sql @@ -1,18 +1,7 @@ --- A denied v3 (fragmented) value is a hard error, NOT a skipped entry. --- --- The row path treats a row-level security denial as permanent and skippable: it is --- counted, skipped, and the receive cursor advances past it. The v3 path cannot do --- the same today. Continuing past a denial leaves PostgreSQL's transaction unusable, --- and the next statement — the checkpoint write — fails with "buffer pin is not owned --- by resource owner TopTransaction". Neither a savepoint around the per-value apply --- nor skipping the staged-fragment delete recovers it. --- --- So this test pins the behaviour that actually holds: the denial surfaces as an --- error and the cursor does not move. That is a known gap, not a desired outcome — --- a denied oversize value is re-delivered on every drain. Closing it needs the v3 --- apply to leave a recoverable transaction state. --- --- Test 27 covers the skip-and-advance guarantee for the v2 row path. +-- A denied v3 (fragmented) value fails the apply like any other denied change: the +-- statement that completes the value raises the error, the receive checkpoint does not +-- move, and the pieces staged by the earlier statements stay, so redelivering after the +-- policy allows the row applies the value. \set testid '58-v3-denied' \ir helper_test_init.sql @@ -99,14 +88,35 @@ SELECT (:applied_count::int = 0) AS denied_ok \gset SELECT (:fail::int + 1) AS fail \gset \endif --- Known gap: the cursor does not advance, so this value is re-delivered every drain. --- Change this to expect an advance once the v3 apply leaves a recoverable state. SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset SELECT (:ckpt_after::bigint = :ckpt_before::bigint) AS ckpt_pinned \gset \if :ckpt_pinned -\echo [PASS] (:testid) known gap: a denied fragmented value leaves the checkpoint pinned at :ckpt_after +\echo [PASS] (:testid) a denied fragmented value leaves the checkpoint at :ckpt_after \else -\echo [FAIL] (:testid) checkpoint moved to :ckpt_after — the v3 denial gap may be closed; update this test +\echo [FAIL] (:testid) checkpoint moved to :ckpt_after after a denied fragmented value +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT (SELECT count(*) FROM cloudsync_payload_fragments) > 0 AS staged_kept \gset +\if :staged_kept +\echo [PASS] (:testid) the pieces staged before the denial are kept for a retry +\else +\echo [FAIL] (:testid) the staged pieces were discarded by the denial +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- Once the policy allows the row, redelivering the chunks applies the value. +ALTER POLICY frag_ins ON frag_rls WITH CHECK (true); +SET ROLE v3_denied_user; +SELECT format('SELECT cloudsync_payload_apply(payload) FROM chunk_transport WHERE ord = %s;', ord) +FROM chunk_transport ORDER BY ord \gexec +RESET ROLE; +SELECT (SELECT length(note) FROM frag_rls WHERE id = 'big') = 655360 + AND NOT EXISTS (SELECT FROM cloudsync_payload_fragments) AS redelivered_ok \gset +\if :redelivered_ok +\echo [PASS] (:testid) redelivery after the policy change applies the value and clears its pieces +\else +\echo [FAIL] (:testid) redelivery did not apply the fragmented value SELECT (:fail::int + 1) AS fail \gset \endif diff --git a/test/postgresql/59_rls_denial_retry.sql b/test/postgresql/59_rls_denial_retry.sql index 0d931106..94f34604 100644 --- a/test/postgresql/59_rls_denial_retry.sql +++ b/test/postgresql/59_rls_denial_retry.sql @@ -1,14 +1,15 @@ --- Row-level security denials outside the batched column path, and denials that depend --- on the order of the payload. +-- Row-level security denials stop cloudsync_payload_apply, and redelivery succeeds once +-- the authorization allows the rows. -- -- 1. A denial raised inside the cloudsync_changes trigger (a block column, a GOS table) --- is skipped like one from the batched path: the rest of the payload applies and the --- receive checkpoint advances, instead of the whole apply failing. --- 2. A row denied only because a row granting access arrives later in the same payload --- is retried once the payload is in, and is applied. --- 3. A row still denied after the retry stays skipped. --- Along the way, block columns and GOS tables must write every column of a permitted row --- although their policies reference another column. +-- raises 42501 like one from the batched path. The statement rolls back, so nothing +-- it wrote remains (no rows, blocks or metadata) and the receive checkpoint stays put. +-- 2. A row authorized only by a row later in the same payload is not reordered or +-- retried internally: the apply fails, and succeeds once the authorizing row is in. +-- 3. A column the policy does not let the session write fails the apply instead of being +-- recorded as applied. +-- Once allowed, block columns and GOS tables write every column of the row although +-- their policies reference another column. \set testid '59-rls-retry' \ir helper_test_init.sql @@ -66,8 +67,19 @@ INSERT INTO logs VALUES ('l_own', :'USER1', 'hello'); -- A task in a project USER1 never joins: denied for good. INSERT INTO tasks VALUES ('t_p2', 'p2', 'task in p2'); -SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS payload_hex -FROM cloudsync_changes WHERE site_id = cloudsync_siteid() \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS p_notes +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'notes' \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS p_events +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'events' \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS p_order +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl IN ('tasks', 'activity', 'members') + AND pk <> cloudsync_pk_encode('t_p2') \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS p_members +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'members' \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS p_logs +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'logs' \gset +SELECT encode(cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq), 'hex') AS p_denied +FROM cloudsync_changes WHERE site_id = cloudsync_siteid() AND tbl = 'tasks' AND pk = cloudsync_pk_encode('t_p2') \gset -- ------------------------------------------------------------------ target \connect cloudsync_test_59_dst @@ -126,85 +138,199 @@ GRANT ALL ON ALL TABLES IN SCHEMA public TO rls_retry_user; GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO rls_retry_user; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public, auth TO rls_retry_user; +-- 1. Trigger-path denial on a block column: nothing is kept, the apply fails. SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset - SET app.current_user_id = :'USER1'; SET ROLE rls_retry_user; \set ON_ERROR_STOP off -\unset apply_rows -SELECT cloudsync_payload_apply(decode(:'payload_hex', 'hex')) AS apply_rows \gset +SELECT cloudsync_payload_apply(decode(:'p_notes', 'hex')) AS _applied \gset +\set notes1_state :SQLSTATE \set ON_ERROR_STOP on RESET ROLE; - -\if :{?apply_rows} -\echo [PASS] (:testid) apply with trigger-path and order-dependent denials completed +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'notes1_state' = '42501' AND :ckpt_after::bigint = :ckpt_before::bigint) AS notes1_denied_ok \gset +\if :notes1_denied_ok +\echo [PASS] (:testid) block column denial: raised 42501 and left the checkpoint in place \else -\echo [FAIL] (:testid) apply failed instead of skipping the denied rows +\echo [FAIL] (:testid) block column denial: SQLSTATE :notes1_state, checkpoint :ckpt_before -> :ckpt_after SELECT (:fail::int + 1) AS fail \gset \endif - --- 1. Trigger-path denials are skipped, the permitted rows are applied. -SELECT (SELECT count(*) FROM notes WHERE id = 'n_own') = 1 - AND (SELECT count(*) FROM notes WHERE id = 'n_other') = 0 AS notes_ok \gset -\if :notes_ok -\echo [PASS] (:testid) block column: own row applied, other user row skipped +SELECT (SELECT count(*) FROM notes) = 0 + AND (SELECT count(*) FROM notes_cloudsync_blocks) = 0 + AND (SELECT count(*) FROM notes_cloudsync) = 0 AS notes_clean_ok \gset +\if :notes_clean_ok +\echo [PASS] (:testid) block column: the failed statement left no rows, blocks or metadata \else -\echo [FAIL] (:testid) block column: unexpected rows in notes +\echo [FAIL] (:testid) block column: the failed apply left state behind SELECT (:fail::int + 1) AS fail \gset \endif -SELECT (SELECT body FROM notes WHERE id = 'n_own') = E'line 1\nline 2' AS notes_body_ok \gset +-- Allowing the rows, the same payload applies in full. +ALTER POLICY own_select ON notes USING (true); +ALTER POLICY own_insert ON notes WITH CHECK (true); +ALTER POLICY own_update ON notes USING (true) WITH CHECK (true); +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_notes', 'hex')) AS _applied \gset +\set notes2_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'notes2_state' = '00000' AND :ckpt_after::bigint >= :ckpt_before::bigint) AS notes2_ok \gset +\if :notes2_ok +\echo [PASS] (:testid) block column redelivery: applied +\else +\echo [FAIL] (:testid) block column redelivery: SQLSTATE :notes2_state, checkpoint :ckpt_before -> :ckpt_after +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (SELECT body FROM notes WHERE id = 'n_own') = E'line 1\nline 2' + AND (SELECT body FROM notes WHERE id = 'n_other') = E'secret 1\nsecret 2' AS notes_body_ok \gset \if :notes_body_ok -\echo [PASS] (:testid) block column: own row materialized in full +\echo [PASS] (:testid) block column: both rows materialized in full \else -\echo [FAIL] (:testid) block column: own row body not materialized +\echo [FAIL] (:testid) block column: rows not materialized after redelivery SELECT (:fail::int + 1) AS fail \gset \endif -SELECT (SELECT count(*) FROM notes_cloudsync_blocks WHERE pk = cloudsync_pk_encode('n_other')) = 0 AS notes_blocks_ok \gset -\if :notes_blocks_ok -\echo [PASS] (:testid) block column: a denied row leaves no blocks behind +-- GOS table, whose merge runs inside the cloudsync_changes trigger. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_events', 'hex')) AS _applied \gset +\set events1_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'events1_state' = '42501' AND :ckpt_after::bigint = :ckpt_before::bigint) AS events1_denied_ok \gset +\if :events1_denied_ok +\echo [PASS] (:testid) GOS denial: raised 42501 and left the checkpoint in place \else -\echo [FAIL] (:testid) block column: denied row left blocks behind +\echo [FAIL] (:testid) GOS denial: SQLSTATE :events1_state, checkpoint :ckpt_before -> :ckpt_after SELECT (:fail::int + 1) AS fail \gset \endif - -SELECT (SELECT count(*) FROM events WHERE id = 'e_own' AND kind = 'login') = 1 - AND (SELECT count(*) FROM events WHERE id = 'e_other') = 0 AS events_ok \gset -\if :events_ok -\echo [PASS] (:testid) GOS table: own row applied with every column, other user row skipped +SELECT (SELECT count(*) FROM events) = 0 AND (SELECT count(*) FROM events_cloudsync) = 0 AS events_clean_ok \gset +\if :events_clean_ok +\echo [PASS] (:testid) GOS table: the failed statement left no rows or metadata \else -\echo [FAIL] (:testid) GOS table: unexpected rows in events +\echo [FAIL] (:testid) GOS table: the failed apply left state behind SELECT (:fail::int + 1) AS fail \gset \endif - --- 2. The task denied on the first pass is applied once its membership is in. -SELECT (SELECT count(*) FROM members WHERE project_id = 'p1') = 1 AS member_ok \gset -SELECT (SELECT title FROM tasks WHERE id = 't_p1') IS NOT DISTINCT FROM 'task in p1' AS retry_ok \gset -\if :member_ok -\if :retry_ok -\echo [PASS] (:testid) order-dependent denial: task applied after its membership on retry +ALTER POLICY own_select ON events USING (true); +ALTER POLICY own_insert ON events WITH CHECK (true); +ALTER POLICY own_update ON events USING (true) WITH CHECK (true); +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_events', 'hex')) AS _applied \gset +\set events2_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'events2_state' = '00000' AND :ckpt_after::bigint >= :ckpt_before::bigint) AS events2_ok \gset +\if :events2_ok +\echo [PASS] (:testid) GOS redelivery: applied \else -\echo [FAIL] (:testid) order-dependent denial: task authorized later in the payload was lost +\echo [FAIL] (:testid) GOS redelivery: SQLSTATE :events2_state, checkpoint :ckpt_before -> :ckpt_after SELECT (:fail::int + 1) AS fail \gset \endif +SELECT (SELECT count(*) FROM events WHERE kind = 'login') = 2 AS events_rows_ok \gset +\if :events_rows_ok +\echo [PASS] (:testid) GOS table: every column of both rows written \else -\echo [FAIL] (:testid) membership row was not applied +\echo [FAIL] (:testid) GOS table: rows incomplete after redelivery SELECT (:fail::int + 1) AS fail \gset \endif -SELECT (SELECT kind FROM activity WHERE id = 'a_p1') IS NOT DISTINCT FROM 'created' AS retry_gos_ok \gset -\if :retry_gos_ok -\echo [PASS] (:testid) order-dependent denial in the trigger path (GOS): applied on retry +-- 2. The tasks come before the membership that authorizes them: no internal retry. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_order', 'hex')) AS _applied \gset +\set order1_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'order1_state' = '42501' AND :ckpt_after::bigint = :ckpt_before::bigint) AS order1_denied_ok \gset +\if :order1_denied_ok +\echo [PASS] (:testid) order-dependent denial: raised 42501 and left the checkpoint in place +\else +\echo [FAIL] (:testid) order-dependent denial: SQLSTATE :order1_state, checkpoint :ckpt_before -> :ckpt_after +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (SELECT count(*) FROM members) = 0 AND (SELECT count(*) FROM tasks) = 0 AND (SELECT count(*) FROM activity) = 0 AS order_clean_ok \gset +\if :order_clean_ok +\echo [PASS] (:testid) order-dependent denial: nothing applied, membership included +\else +\echo [FAIL] (:testid) order-dependent denial: partial state left behind +SELECT (:fail::int + 1) AS fail \gset +\endif +-- Deliver the membership first, then the same payload again. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_members', 'hex')) AS _applied \gset +\set members_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'members_state' = '00000' AND :ckpt_after::bigint >= :ckpt_before::bigint) AS members_ok \gset +\if :members_ok +\echo [PASS] (:testid) membership: applied +\else +\echo [FAIL] (:testid) membership: SQLSTATE :members_state, checkpoint :ckpt_before -> :ckpt_after +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_order', 'hex')) AS _applied \gset +\set order2_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'order2_state' = '00000' AND :ckpt_after::bigint >= :ckpt_before::bigint) AS order2_ok \gset +\if :order2_ok +\echo [PASS] (:testid) order-dependent redelivery: applied \else -\echo [FAIL] (:testid) order-dependent denial in the trigger path (GOS): row was lost +\echo [FAIL] (:testid) order-dependent redelivery: SQLSTATE :order2_state, checkpoint :ckpt_before -> :ckpt_after +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT (SELECT title FROM tasks WHERE id = 't_p1') IS NOT DISTINCT FROM 'task in p1' + AND (SELECT kind FROM activity WHERE id = 'a_p1') IS NOT DISTINCT FROM 'created' + AND (SELECT title FROM tasks WHERE id = 't_rev') IS NOT DISTINCT FROM 'revived' AS order_rows_ok \gset +\if :order_rows_ok +\echo [PASS] (:testid) order-dependent rows (batched, GOS trigger path, resurrected) applied after the membership +\else +\echo [FAIL] (:testid) order-dependent rows missing after redelivery SELECT (:fail::int + 1) AS fail \gset \endif --- A column the policy does not let this session write is reported as denied, never --- recorded as applied while the table does not hold it. -SELECT NOT EXISTS (SELECT FROM logs_cloudsync WHERE col_name = 'msg') - AND (SELECT msg FROM logs WHERE id = 'l_own') IS NULL AS no_silent_loss_ok \gset +-- 3. logs has no UPDATE policy: a GOS table writes its second column as an update, +-- which that policy hides. The write fails instead of being recorded as applied. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_logs', 'hex')) AS _applied \gset +\set logs_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'logs_state' = '42501' AND :ckpt_after::bigint = :ckpt_before::bigint) AS logs_denied_ok \gset +\if :logs_denied_ok +\echo [PASS] (:testid) column hidden from UPDATE: raised 42501 and left the checkpoint in place +\else +\echo [FAIL] (:testid) column hidden from UPDATE: SQLSTATE :logs_state, checkpoint :ckpt_before -> :ckpt_after +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT NOT EXISTS (SELECT FROM logs_cloudsync) AND NOT EXISTS (SELECT FROM logs) AS no_silent_loss_ok \gset \if :no_silent_loss_ok \echo [PASS] (:testid) a column hidden from UPDATE is not recorded as applied \else @@ -212,32 +338,31 @@ SELECT NOT EXISTS (SELECT FROM logs_cloudsync WHERE col_name = 'msg') SELECT (:fail::int + 1) AS fail \gset \endif -SELECT (SELECT title FROM tasks WHERE id = 't_rev') IS NOT DISTINCT FROM 'revived' AS retry_revived_ok \gset -\if :retry_revived_ok -\echo [PASS] (:testid) order-dependent denial of a resurrected row: applied on retry +-- A task in a project USER1 never joins stays denied on every delivery. +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before \gset +SET app.current_user_id = :'USER1'; +SET ROLE rls_retry_user; +\set ON_ERROR_STOP off +SELECT cloudsync_payload_apply(decode(:'p_denied', 'hex')) AS _applied \gset +\set denied_state :SQLSTATE +\set ON_ERROR_STOP on +RESET ROLE; +SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset +SELECT (:'denied_state' = '42501' AND :ckpt_after::bigint = :ckpt_before::bigint) AS denied_denied_ok \gset +\if :denied_denied_ok +\echo [PASS] (:testid) permanent denial: raised 42501 and left the checkpoint in place \else -\echo [FAIL] (:testid) order-dependent denial of a resurrected row: row was lost +\echo [FAIL] (:testid) permanent denial: SQLSTATE :denied_state, checkpoint :ckpt_before -> :ckpt_after SELECT (:fail::int + 1) AS fail \gset \endif - --- 3. A row still denied after the retry stays out, and the cursor moves on. SELECT (SELECT count(*) FROM tasks WHERE id = 't_p2') = 0 AS still_denied_ok \gset \if :still_denied_ok -\echo [PASS] (:testid) a row still denied after the retry is skipped +\echo [PASS] (:testid) a permanently denied row is not applied \else \echo [FAIL] (:testid) a permanently denied row was applied SELECT (:fail::int + 1) AS fail \gset \endif -SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_after \gset -SELECT (:ckpt_after::bigint > :ckpt_before::bigint) AS ckpt_ok \gset -\if :ckpt_ok -\echo [PASS] (:testid) receive checkpoint advanced past the skipped rows -\else -\echo [FAIL] (:testid) receive checkpoint stayed at :ckpt_after -SELECT (:fail::int + 1) AS fail \gset -\endif - \connect postgres \ir helper_psql_conn_setup.sql DROP DATABASE IF EXISTS cloudsync_test_59_src; diff --git a/test/review_regressions.c b/test/review_regressions.c index 21602c19..b551502d 100644 --- a/test/review_regressions.c +++ b/test/review_regressions.c @@ -98,16 +98,6 @@ static void scratch_remove(const char *const *names, int count) { rmdir(scratch_dir); #endif } -static int skipped_warnings; -static int skipped_changes; // sum of N over "skipped N received change(s) that failed to apply" -static void log_callback(void *arg, int code, const char *message) { - (void)arg; - if (code == SQLITE_WARNING && message && strstr(message, "failed to apply")) { - skipped_warnings++; - const char *n = strstr(message, "skipped "); - if (n) skipped_changes += atoi(n + 8); - } -} static int apply_payload(sqlite3 *source, sqlite3 *target) { sqlite3_stmt *read = NULL, *write = NULL; CHECK(sqlite3_prepare_v2(source, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes", -1, &read, NULL) == SQLITE_OK); @@ -120,8 +110,10 @@ static int apply_payload(sqlite3 *source, sqlite3 *target) { return rc; } static void test_payload_errors(void) { - // A write that fails on its data fails the same way on every retry: it is skipped - // and reported, and the cursor still advances past it. + // The apply stops at the first failed write (first, middle or last row) with the + // database's own error. The rows before it are kept, the failed row leaves neither + // data nor metadata, and the cursor does not move: once the cause is fixed, the same + // payload applies in full. for (int denied = 1; denied <= 3; denied++) { sqlite3 *source = open_db(), *target = open_db(); const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL,value TEXT); SELECT cloudsync_init('t');"; @@ -130,21 +122,27 @@ static void test_payload_errors(void) { char trigger[256]; snprintf(trigger, sizeof(trigger), "CREATE TRIGGER deny BEFORE INSERT ON t WHEN NEW.id='%d' BEGIN SELECT RAISE(ABORT,'denied'); END", denied); CHECK(sql(target, trigger) == SQLITE_OK); - skipped_warnings = 0; - CHECK(apply_payload(source, target) == SQLITE_ROW); - CHECK(skipped_warnings == 1); - CHECK(scalar(target, "SELECT count(*) FROM t") == 2); - char query[128]; - snprintf(query, sizeof(query), "SELECT count(*) FROM t WHERE id='%d'", denied); + CHECK(apply_payload(source, target) != SQLITE_ROW); + CHECK(strstr(sqlite3_errmsg(target), "denied") != NULL); + CHECK((sqlite3_extended_errcode(target) & 0xFF) == SQLITE_CONSTRAINT); + char query[160]; + snprintf(query, sizeof(query), "SELECT count(*) FROM t WHERE id<'%d'", denied); + CHECK(scalar(target, query) == denied - 1); + CHECK(scalar(target, "SELECT count(*) FROM t") == denied - 1); + snprintf(query, sizeof(query), "SELECT count(*) FROM t_cloudsync WHERE pk=cloudsync_pk_encode('%d')", denied); CHECK(scalar(target, query) == 0); - CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") > 0); + CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") == 0); CHECK(sqlite3_get_autocommit(target)); + CHECK(sql(target, "DROP TRIGGER deny") == SQLITE_OK); + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(scalar(target, "SELECT count(*) FROM t") == 3); + CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") > 0); CHECK(close_db(source) == SQLITE_OK); CHECK(close_db(target) == SQLITE_OK); } - // A transient failure (the database is locked by another connection) could succeed - // on a retry, so it must fail the apply and leave the cursor in place. + // A transient failure (the database is locked by another connection) fails the apply + // the same way and leaves the cursor in place. { char path[512]; CHECK(scratch_create()); @@ -157,9 +155,7 @@ static void test_payload_errors(void) { CHECK(sql(source, "INSERT INTO t VALUES('1','a'),('2','b');") == SQLITE_OK); CHECK(sqlite3_open(path, &locker) == SQLITE_OK); CHECK(sql(locker, "BEGIN IMMEDIATE") == SQLITE_OK); - skipped_warnings = 0; CHECK(apply_payload(source, target) != SQLITE_ROW); - CHECK(skipped_warnings == 0); CHECK(sql(locker, "ROLLBACK") == SQLITE_OK); CHECK(sqlite3_close(locker) == SQLITE_OK); CHECK(scalar(target, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)") == 0); @@ -212,8 +208,8 @@ static void test_resurrected_group_rollback(void) { const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, a TEXT, b TEXT); SELECT cloudsync_init('t');"; // A resurrected row arrives as a sentinel plus its columns. When its write fails the - // whole group is skipped and counted (3 changes, not 2), and nothing it wrote remains: - // delivered again once the cause is gone, the row is created. + // apply stops and nothing the group wrote remains: delivered again once the cause is + // gone, the row is created. for (int transient = 0; transient < 2; transient++) { sqlite3 *source = open_db(), *target = open_db(); CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); @@ -222,16 +218,8 @@ static void test_resurrected_group_rollback(void) { CHECK(sql(target, transient ? "CREATE TRIGGER rej BEFORE INSERT ON t WHEN NEW.id='r1' BEGIN SELECT fail_busy(); END" : "CREATE TRIGGER rej BEFORE INSERT ON t WHEN NEW.id='r1' BEGIN SELECT RAISE(ABORT,'rejected r1'); END") == SQLITE_OK); - skipped_warnings = 0; skipped_changes = 0; - int rc = apply_payload(source, target); - if (transient) { - CHECK(rc != SQLITE_ROW); // transient: the apply fails and is retried - CHECK(skipped_warnings == 0); - } else { - CHECK(rc == SQLITE_ROW); // data failure: skipped and reported - CHECK(skipped_warnings == 1); - CHECK(skipped_changes == 3); - } + CHECK(apply_payload(source, target) != SQLITE_ROW); + CHECK(strstr(sqlite3_errmsg(target), transient ? "simulated busy" : "rejected r1") != NULL); CHECK(scalar(target, "SELECT count(*) FROM t_cloudsync") == 0); CHECK(sql(target, "DROP TRIGGER rej") == SQLITE_OK); CHECK(apply_payload(source, target) == SQLITE_ROW); @@ -250,14 +238,28 @@ static void test_resurrected_group_rollback(void) { CHECK(sql(target, "CREATE TRIGGER rej BEFORE UPDATE ON t BEGIN SELECT RAISE(ABORT,'rejected update'); END;" "CREATE TRIGGER rej2 BEFORE INSERT ON t BEGIN SELECT RAISE(ABORT,'rejected insert'); END;") == SQLITE_OK); CHECK(sql(target, "CREATE TEMP TABLE before_clocks AS SELECT col_name, col_version FROM t_cloudsync") == SQLITE_OK); - skipped_warnings = 0; - CHECK(apply_payload(source, target) == SQLITE_ROW); - CHECK(skipped_warnings >= 1); // the resurrection was attempted and failed + CHECK(apply_payload(source, target) != SQLITE_ROW); // the resurrection was attempted and failed CHECK(scalar(target, "SELECT count(*) FROM (SELECT col_name, col_version FROM t_cloudsync EXCEPT SELECT col_name, col_version FROM before_clocks)") == 0); CHECK(scalar(target, "SELECT count(*) FROM t WHERE id='r1' AND a='x' AND b='y'") == 1); CHECK(close_db(source) == SQLITE_OK); CHECK(close_db(target) == SQLITE_OK); } +static void test_batched_update_missing_row(void) { + // Metadata says the row exists but the base row is gone (deleted while sync was + // disabled). A received multi-column update must write the row, not record winner + // clocks for an UPDATE that changed nothing. + const char *schema = "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, a TEXT, b TEXT); SELECT cloudsync_init('t');"; + sqlite3 *source = open_db(), *target = open_db(); + CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); + CHECK(sql(source, "INSERT INTO t VALUES('r1','x','y')") == SQLITE_OK); + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(sql(target, "SELECT cloudsync_disable('t'); DELETE FROM t WHERE id='r1'; SELECT cloudsync_enable('t');") == SQLITE_OK); + CHECK(sql(source, "UPDATE t SET a='x2', b='y2' WHERE id='r1'") == SQLITE_OK); + CHECK(apply_payload(source, target) == SQLITE_ROW); + CHECK(scalar(target, "SELECT count(*) FROM t WHERE id='r1' AND a='x2' AND b='y2'") == 1); + CHECK(close_db(source) == SQLITE_OK); + CHECK(close_db(target) == SQLITE_OK); +} static void test_block_write_errors(void) { for (int update = 0; update < 2; update++) { sqlite3 *db = open_db(); @@ -299,9 +301,7 @@ static void test_block_not_null_payload(void) { CHECK(sql(source, schema) == SQLITE_OK && sql(target, schema) == SQLITE_OK); CHECK(sql(target, "CREATE TRIGGER no_null_owner BEFORE INSERT ON docs WHEN NEW.owner = 'x' BEGIN SELECT RAISE(ABORT,'owner required'); END") == SQLITE_OK); CHECK(sql(source, "INSERT INTO docs VALUES('1','alice','line 1' || char(10) || 'line 2')") == SQLITE_OK); - skipped_warnings = 0; CHECK(apply_payload(source, target) == SQLITE_ROW); - CHECK(skipped_warnings == 0); CHECK(scalar(target, "SELECT count(*) FROM docs WHERE id='1' AND owner='alice' AND body='line 1' || char(10) || 'line 2'") == 1); CHECK(close_db(source) == SQLITE_OK); CHECK(close_db(target) == SQLITE_OK); @@ -405,13 +405,13 @@ int main(void) { faults.xMalloc = fault_malloc; faults.xRealloc = fault_realloc; CHECK(sqlite3_config(SQLITE_CONFIG_MALLOC, &faults) == SQLITE_OK); - CHECK(sqlite3_config(SQLITE_CONFIG_LOG, log_callback, NULL) == SQLITE_OK); CHECK(sqlite3_initialize() == SQLITE_OK); test_clocks_and_double(); test_best_index(); test_payload_errors(); test_payload_high_compression(); test_resurrected_group_rollback(); + test_batched_update_missing_row(); test_block_write_errors(); test_block_materialize_errors(); test_block_migration_orphan(); diff --git a/test/unit.c b/test/unit.c index 7cc0e4eb..83fe06e7 100644 --- a/test/unit.c +++ b/test/unit.c @@ -9045,8 +9045,26 @@ bool do_test_row_filter_prefill(int nclients, bool print_result, bool cleanup_da return result; } -// Test that BEFORE triggers with RAISE(ABORT) simulate RLS denial: -// per-PK savepoints isolate failures so allowed rows commit and denied rows roll back. +// Sends every change of source to target in one payload; returns the apply's step code. +static int rls_merge_step (sqlite3 *source, sqlite3 *target, bool only_locals) { + sqlite3_stmt *sel = NULL, *ins = NULL; + const char *sel_sql = only_locals + ? "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();" + : "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; + int rc = sqlite3_prepare_v2(source, sel_sql, -1, &sel, NULL); + if (rc == SQLITE_OK) rc = sqlite3_prepare_v2(target, "SELECT cloudsync_payload_decode(?);", -1, &ins, NULL); + if (rc == SQLITE_OK && sqlite3_step(sel) == SQLITE_ROW) { + sqlite3_bind_value(ins, 1, sqlite3_column_value(sel, 0)); + rc = sqlite3_step(ins); + } + sqlite3_finalize(sel); + sqlite3_finalize(ins); + return rc; +} + +// Test that BEFORE triggers with RAISE(ABORT) simulate RLS denial: the apply stops at the +// denied row, the rows before it are kept, the denied PK rolls back, and redelivering +// after the policy allows it applies the rest. bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_databases, bool only_locals) { sqlite3 *db[MAX_SIMULATED_CLIENTS] = {NULL}; bool result = false; @@ -9116,27 +9134,10 @@ bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_d rc = sqlite3_exec(db[0], "INSERT INTO tasks VALUES ('t5', 'user2', 'Task 5', 7);", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - // Merge with partial-failure tolerance: cloudsync_payload_decode returns error - // when any PK is denied, but allowed PKs are already committed via per-PK savepoints. - { - sqlite3_stmt *sel = NULL, *ins = NULL; - const char *sel_sql = only_locals - ? "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();" - : "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; - rc = sqlite3_prepare_v2(db[0], sel_sql, -1, &sel, NULL); - if (rc != SQLITE_OK) { sqlite3_finalize(sel); goto finalize; } - rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &ins, NULL); - if (rc != SQLITE_OK) { sqlite3_finalize(sel); sqlite3_finalize(ins); goto finalize; } - - while (sqlite3_step(sel) == SQLITE_ROW) { - sqlite3_value *v = sqlite3_column_value(sel, 0); - if (sqlite3_value_type(v) == SQLITE_NULL) continue; - sqlite3_bind_value(ins, 1, v); - sqlite3_step(ins); // partial failure expected — ignore rc - sqlite3_reset(ins); - } - sqlite3_finalize(sel); - sqlite3_finalize(ins); + // The payload stops at the denied row (t5, last): t4 before it is kept. + if (rls_merge_step(db[0], db[1], only_locals) == SQLITE_ROW) { + printf("Phase 2: the denied insert must fail the apply\n"); + goto finalize; } // Verify: t4 present (user1 → allowed) @@ -9187,26 +9188,44 @@ bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_d rc = sqlite3_exec(db[0], "UPDATE tasks SET title='Task 2 Hacked', priority=99 WHERE id='t2';", NULL, NULL, NULL); if (rc != SQLITE_OK) goto finalize; - // Merge with partial-failure tolerance (same pattern as phase 2) + // The payload still carries the denied t5 insert ahead of the updates: the apply + // stops there again and t1's update is not applied. + if (rls_merge_step(db[0], db[1], only_locals) == SQLITE_ROW) { + printf("Phase 3: the denied insert must fail the apply again\n"); + goto finalize; + } + { + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT priority FROM tasks WHERE id='t1';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + int priority = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (priority != 3) { + printf("Phase 3: t1 must not be updated past the failure (priority=%d)\n", priority); + goto finalize; + } + } + + // Allow inserts again and redeliver: t5 and t1's update apply, and the apply stops + // at t2's denied update (the last change). + rc = sqlite3_exec(db[1], "DROP TRIGGER rls_deny_insert;", NULL, NULL, NULL); + if (rc != SQLITE_OK) goto finalize; + if (rls_merge_step(db[0], db[1], only_locals) == SQLITE_ROW) { + printf("Phase 3: the denied update must fail the apply\n"); + goto finalize; + } { - sqlite3_stmt *sel = NULL, *ins = NULL; - const char *sel_sql = only_locals - ? "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes WHERE site_id=cloudsync_siteid();" - : "SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) FROM cloudsync_changes;"; - rc = sqlite3_prepare_v2(db[0], sel_sql, -1, &sel, NULL); - if (rc != SQLITE_OK) { sqlite3_finalize(sel); goto finalize; } - rc = sqlite3_prepare_v2(db[1], "SELECT cloudsync_payload_decode(?);", -1, &ins, NULL); - if (rc != SQLITE_OK) { sqlite3_finalize(sel); sqlite3_finalize(ins); goto finalize; } - - while (sqlite3_step(sel) == SQLITE_ROW) { - sqlite3_value *v = sqlite3_column_value(sel, 0); - if (sqlite3_value_type(v) == SQLITE_NULL) continue; - sqlite3_bind_value(ins, 1, v); - sqlite3_step(ins); // partial failure expected — ignore rc - sqlite3_reset(ins); - } - sqlite3_finalize(sel); - sqlite3_finalize(ins); + sqlite3_stmt *stmt = NULL; + rc = sqlite3_prepare_v2(db[1], "SELECT COUNT(*) FROM tasks WHERE id='t5';", -1, &stmt, NULL); + if (rc != SQLITE_OK) goto finalize; + if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } + int count = sqlite3_column_int(stmt, 0); + sqlite3_finalize(stmt); + if (count != 1) { + printf("Phase 3: t5 expected once inserts are allowed, got %d\n", count); + goto finalize; + } } // Verify: t1 updated (user1 → allowed) @@ -9215,7 +9234,8 @@ bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_d rc = sqlite3_prepare_v2(db[1], "SELECT title, priority FROM tasks WHERE id='t1';", -1, &stmt, NULL); if (rc != SQLITE_OK) goto finalize; if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } - const char *title = (const char *)sqlite3_column_text(stmt, 0); + char title[64]; + snprintf(title, sizeof(title), "%s", (const char *)sqlite3_column_text(stmt, 0)); int priority = sqlite3_column_int(stmt, 1); bool ok = (strcmp(title, "Task 1 Updated") == 0) && (priority == 10); sqlite3_finalize(stmt); @@ -9231,7 +9251,8 @@ bool do_test_rls_trigger_denial (int nclients, bool print_result, bool cleanup_d rc = sqlite3_prepare_v2(db[1], "SELECT title, priority FROM tasks WHERE id='t2';", -1, &stmt, NULL); if (rc != SQLITE_OK) goto finalize; if (sqlite3_step(stmt) != SQLITE_ROW) { sqlite3_finalize(stmt); goto finalize; } - const char *title = (const char *)sqlite3_column_text(stmt, 0); + char title[64]; + snprintf(title, sizeof(title), "%s", (const char *)sqlite3_column_text(stmt, 0)); int priority = sqlite3_column_int(stmt, 1); bool ok = (strcmp(title, "Task 2") == 0) && (priority == 5); sqlite3_finalize(stmt); From 05d42de6e7ff5a52b6d528431f0980034fe33b5c Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 20:08:20 -0600 Subject: [PATCH 20/28] fix(receive): keep one checkpoint per stream and check its fragments The receive checkpoint also identifies the server's prepared pages, so it stays fixed for the whole stream and moves once, after the final chunk. - A fragmented value the stream delivered must be complete when its final chunk applies; otherwise the call fails and the checkpoint stays. Only values this stream staged are tracked, so staging left by other streams or direct calls never blocks progress. Later pieces of a value the stream already applied are ignored instead of being staged again as an incomplete group. - Any failed chunk makes the next call replay the window from page 0 with fresh observations; calls capped by max_chunks keep both. - Checkpoint write failures are returned instead of ignored, and a direct v3 fragment call no longer moves the checkpoint. Add a test-only /check responder and network tests for capped paging, failure replay, checkpoint write errors, incomplete/out-of-order/duplicate fragments, abandoned staging and completion by another caller. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 4 +- CHANGELOG.md | 3 + src/cloudsync.c | 116 ++++++-- src/cloudsync.h | 15 +- src/network/network.c | 27 +- src/postgresql/sql_postgresql.c | 3 + src/sql.h | 1 + src/sqlite/sql_sqlite.c | 3 + test/network_unit.c | 281 ++++++++++++++++++++ test/postgresql/58_v3_denied_checkpoint.sql | 4 +- 10 files changed, 430 insertions(+), 27 deletions(-) diff --git a/API.md b/API.md index 9825c6a9..c098458b 100644 --- a/API.md +++ b/API.md @@ -651,7 +651,7 @@ On PostgreSQL, apply chunks as individual statements from the transport/client l - Monolithic payloads generated by [`cloudsync_payload_encode()`](#cloudsync_payload_encodetbl-pk-col_name-col_value-col_version-db_version-site_id-cl-seq). - Chunk-fragment payloads generated by [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id). -When a v3 fragment payload is received, CloudSync stores the fragment in an internal table and returns after applying zero or more completed values. Once the final fragment for a value is received, the completed value is validated and applied. Duplicate fragment delivery is idempotent. +When a v3 fragment payload is received, CloudSync stores the fragment in an internal table and returns after applying zero or more completed values. Once the final fragment for a value is received, the completed value is validated and applied. Fragments can arrive in any order, and duplicate fragment delivery is idempotent. Applying a fragment never moves the receive checkpoint. **Parameters:** @@ -810,7 +810,7 @@ By default this function **drains all currently-available chunks** in one call. SELECT cloudsync_network_receive_changes(5) ->> '$.receive.complete'; ``` -The drain position (the per-stream page cursor) is held **in memory** on the network context, so a capped drain resumes where it left off on the next call — the caller does not manage any cursor; it just loops while `receive.complete` is `false`. If the connection is closed or the process restarts mid-drain, the cursor is lost and the next call safely restarts the drain from the beginning of the stream: already-applied chunks are re-downloaded and re-applied idempotently, so **no rows are skipped** — only redundant download is incurred. This is safe because the durable receive checkpoint (`check_dbversion`/`check_seq`) only advances after a stream has been **fully** applied, never in the middle of a source `db_version`. +The drain position (the per-stream page cursor) is held **in memory** on the network context, so a capped drain resumes where it left off on the next call — the caller does not manage any cursor; it just loops while `receive.complete` is `false`. If the connection is closed or the process restarts mid-drain, or a call fails, the next call safely restarts the drain from the beginning of the stream: already-applied chunks are re-downloaded and re-applied idempotently, so **no rows are skipped** — only redundant download is incurred. This is safe because the durable receive checkpoint (`check_dbversion`/`check_seq`) stays fixed for the whole stream and only advances once the stream has been **fully** applied. A stream whose final chunk arrives while a fragmented value it delivered is still incomplete fails with an error instead of advancing. If the network is misconfigured or the remote server is unreachable, the function raises a SQL error. If the received payload cannot be applied locally (for example because of an unknown schema hash), the error is returned as a `receive.error` field in the JSON response. If the server reports an unresolved failed check job (e.g. an `encode_changes` failure), that failure is forwarded as a `receive.lastFailure` object. diff --git a/CHANGELOG.md b/CHANGELOG.md index be8c2147..99240fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **A received change whose write fails is no longer dropped silently.** Previously the error was discarded and the change was lost without a trace. The apply now fails with that error (see Changed), and the network functions report it as `receive.error`, together with the rows applied before it. - **A received update to a row that is not there is no longer recorded as applied.** Several columns of an existing row are written with one UPDATE; when it changed nothing (the row was deleted while sync was disabled, or a policy's USING clause hides it), the changes were recorded as applied without being stored. The row is now written with the upsert instead, which inserts it or reports the policy. +- **A receive stream no longer checkpoints past a fragmented value it delivered incompletely.** When the stream's final chunk arrives while a value it delivered is still missing pieces, the call now fails with an error and the checkpoint stays in place. Pieces staged by other streams or direct calls are ignored, so they never block a receive. After any failed receive, the next call downloads the stream again from its first page, which re-checks every value against the whole stream; the checkpoint still moves only when the stream completes. +- **A receive checkpoint that cannot be written is reported** as a receive error instead of being ignored. +- **Applying a fragment directly with `cloudsync_payload_apply` no longer moves the receive checkpoint.** A fragment carries no end-of-stream marker, so completing a value in a direct call could advance the cursor past changes not received yet. - **PostgreSQL: applying a payload read from a table no longer fails.** `SELECT cloudsync_payload_apply(data) FROM some_table` failed with `buffer pin ... is not owned by resource owner TopTransaction` (and could abort an assertion-enabled server): the internal savepoints left the caller's resource owner and memory context switched. Both are now restored when a savepoint ends. - **Block columns and grow-only-set tables merge under row-level security and NOT NULL constraints.** Both wrote a received column through an upsert holding only the primary key and that column: PostgreSQL checks an INSERT policy against that proposed row even when it becomes an update, and a NOT NULL column rejects it outright, so the column was never written. A block column's row now has its other received columns written first, and an existing row — of either kind — is updated in place. When the update finds no row to change (the row is gone, or an UPDATE policy hides it), the upsert is used as before, so the write is reported rather than recorded as applied without being stored. - **A received row whose write fails leaves no partial state behind.** Applying a row wrote part of its metadata — for a deleted-and-recreated row, the new sentinel and the reset column clocks — before the row itself, outside the savepoint that protected the write. When the write then failed, the metadata claimed the row was there: delivered again, even after the cause was fixed (a lock that went away, a policy that now allows it), the row was silently never created, and an existing row kept zeroed clocks. Each primary key's changes are now applied under one savepoint that the failed write rolls back entirely. diff --git a/src/cloudsync.c b/src/cloudsync.c index 141acff1..91223709 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -145,6 +145,11 @@ struct cloudsync_pk_decode_bind_context { int64_t seq; }; +typedef struct { + char value_id[33]; + bool applied; // applied by this stream: later pieces are redundant +} cloudsync_stream_value; + struct cloudsync_context { void *db; char errmsg[1024]; @@ -204,6 +209,12 @@ struct cloudsync_context { // visible when a later chunk reports, including the changes a failing payload // applied before its error. Reset with cloudsync_apply_stats_reset. int apply_rows; + + // v3 values the current receive stream delivered: pending ones are checked before + // the stream's final checkpoint. See cloudsync_receive_stream_reset. + cloudsync_stream_value *stream_values; + int stream_values_count; + int stream_values_cap; }; struct cloudsync_table_context { @@ -642,6 +653,35 @@ static void cloudsync_apply_stats_add (cloudsync_context *data, int rows) { if (rows > 0) data->apply_rows = (data->apply_rows > INT_MAX - rows) ? INT_MAX : data->apply_rows + rows; } +void cloudsync_receive_stream_reset (cloudsync_context *data) { + if (data) data->stream_values_count = 0; +} + +static cloudsync_stream_value *cloudsync_stream_value_find (cloudsync_context *data, const char *value_id) { + for (int i = 0; i < data->stream_values_count; i++) { + if (strcmp(data->stream_values[i].value_id, value_id) == 0) return &data->stream_values[i]; + } + return NULL; +} + +// Records whether the current receive stream has applied value_id or still has it pending. +static int cloudsync_stream_value_track (cloudsync_context *data, const char *value_id, bool applied) { + cloudsync_stream_value *v = cloudsync_stream_value_find(data, value_id); + if (!v) { + if (data->stream_values_count == data->stream_values_cap) { + int cap = data->stream_values_cap ? data->stream_values_cap * 2 : 8; + cloudsync_stream_value *values = cloudsync_memory_realloc(data->stream_values, (uint64_t)cap * sizeof(*values)); + if (!values) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: out of memory", DBRES_NOMEM); + data->stream_values = values; + data->stream_values_cap = cap; + } + v = &data->stream_values[data->stream_values_count++]; + snprintf(v->value_id, sizeof(v->value_id), "%s", value_id); + } + v->applied = applied; + return DBRES_OK; +} + int cloudsync_apply_rows_count (cloudsync_context *data) { return (data) ? data->apply_rows : 0; } @@ -2551,6 +2591,7 @@ void cloudsync_context_free (void *ctx) { cloudsync_terminate(data); cloudsync_memory_free(data->tables); + cloudsync_memory_free(data->stream_values); cloudsync_memory_free(data); } @@ -4093,7 +4134,7 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data return rc; } -static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, cloudsync_payload_fragment_row *row, int *pnrows) { +static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, cloudsync_payload_fragment_row *row, bool track, int *pnrows) { char value_id[64]; char checksum_hex[17]; int part_index = 0, part_count = 0; @@ -4121,6 +4162,12 @@ static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, clouds return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 fragment identity", DBRES_MISUSE); } + // A piece of a value this stream already applied is redundant: staging it again + // would leave it behind as an incomplete group. It happens when pieces left by an + // interrupted attempt complete the value before the replay delivers its last piece. + cloudsync_stream_value *seen = track ? cloudsync_stream_value_find(data, value_id) : NULL; + if (seen && seen->applied) return DBRES_OK; + // the fragments table is guaranteed by dbutils_settings_init; no DDL here // because the apply path runs under sync-only credentials on server nodes int rc = cloudsync_payload_fragments_cleanup_stale(data); @@ -4149,7 +4196,11 @@ static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, clouds if (rc == DBRES_DONE) rc = DBRES_OK; if (rc != DBRES_OK) return rc; - return cloudsync_payload_apply_reassembled_fragment(data, value_id, checksum_hex, pnrows); + int applied = 0; + rc = cloudsync_payload_apply_reassembled_fragment(data, value_id, checksum_hex, &applied); + if (rc != DBRES_OK) return rc; + if (pnrows) *pnrows += applied; + return track ? cloudsync_stream_value_track(data, value_id, applied > 0) : DBRES_OK; } // #ifndef CLOUDSYNC_OMIT_RLS_VALIDATION @@ -4158,17 +4209,38 @@ static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, clouds // (or a fully-applied chunk stream) has been applied. See the checkpoint-mode // documentation on cloudsync_payload_apply in cloudsync.h. The advance is // strictly monotonic so re-delivered rows never regress the cursor. -static void cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t checkpoint_db_version, int64_t checkpoint_seq) { +static int cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t checkpoint_db_version, int64_t checkpoint_seq) { int64_t target_db_version; int64_t target_seq; - if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_NONE) return; + if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_NONE) return DBRES_OK; if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_LAST_APPLIED) { // Nothing applied -> nothing to checkpoint. - if (data->apply_last_db_version < 0) return; + if (data->apply_last_db_version < 0) return DBRES_OK; target_db_version = data->apply_last_db_version; target_seq = data->apply_last_seq; } else { + // The final chunk of a receive stream. A fragmented value the stream delivered + // must be complete by now: its pieces are removed when it is applied, here or + // by another connection. Only values this stream staged are checked. Staging + // left by other streams or direct calls is ignored on purpose: a value_id + // identifies a value, not the stream that delivered it, and a replay can + // legitimately omit an old value (replaced, lost a conflict, filtered out, + // untracked), so requiring the whole staging table to be empty could stop the + // cursor for good. The age cleanup bounds such leftovers. A value another + // connection completes while this stream is still delivering it leaves this + // stream's later pieces staged again: the check fails once, and the replay + // from the first page applies the value again as a no-op and passes. + for (int i = 0; i < data->stream_values_count; i++) { + if (data->stream_values[i].applied) continue; + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_EXISTS, &vm, 0); + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 1, data->stream_values[i].value_id, -1); + if (rc == DBRES_OK) rc = databasevm_step(vm); + if (vm) databasevm_finalize(vm); + if (rc == DBRES_ROW) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: the receive stream ended with an incomplete fragmented value", DBRES_ERROR); + if (rc != DBRES_DONE) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to check staged fragments", rc); + } target_db_version = checkpoint_db_version; target_seq = checkpoint_seq; } @@ -4177,16 +4249,20 @@ static void cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t int64_t cur_seq = dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_CHECK_SEQ); // monotonic guard: never move the cursor backwards - if (target_db_version < cur_db_version) return; - if (target_db_version == cur_db_version && target_seq <= cur_seq) return; - - char buf[256]; - snprintf(buf, sizeof(buf), "%" PRId64, target_db_version); - dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); - if (target_seq != cur_seq) { - snprintf(buf, sizeof(buf), "%" PRId64, target_seq); - dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_SEQ, buf); + if (target_db_version > cur_db_version || (target_db_version == cur_db_version && target_seq > cur_seq)) { + // db_version first: failing between the two writes then re-delivers rows + // instead of skipping them + char buf[256]; + snprintf(buf, sizeof(buf), "%" PRId64, target_db_version); + int rc = dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_DBVERSION, buf); + if (rc == DBRES_OK && target_seq != cur_seq) { + snprintf(buf, sizeof(buf), "%" PRId64, target_seq); + rc = dbutils_settings_set_key_value(data, CLOUDSYNC_KEY_CHECK_SEQ, buf); + } + if (rc != DBRES_OK) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to write the receive checkpoint", rc); } + if (checkpoint_db_version >= 0) cloudsync_receive_stream_reset(data); + return DBRES_OK; } // Steps one decoded payload row (an INSERT into cloudsync_changes). On PostgreSQL the merge @@ -4377,7 +4453,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b break; } int n = 0; - rc = cloudsync_payload_apply_fragment_row(data, &row, &n); + rc = cloudsync_payload_apply_fragment_row(data, &row, checkpoint_db_version != CLOUDSYNC_CHECKPOINT_LAST_APPLIED, &n); // stop at the first error, as the row path does if (rc != DBRES_OK) break; applied_rows += n; @@ -4389,8 +4465,12 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b if (pnrows) *pnrows = applied_rows; // Advance the receive cursor only after the whole payload is applied, // gated on the caller-supplied checkpoint (a non-final chunk passes - // CLOUDSYNC_CHECKPOINT_NONE and leaves the cursor untouched). - if (rc == DBRES_OK) cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); + // CLOUDSYNC_CHECKPOINT_NONE and leaves the cursor untouched). A fragment + // carries no end-of-stream marker, so LAST_APPLIED (a direct call) never + // moves the cursor. + if (rc == DBRES_OK && checkpoint_db_version != CLOUDSYNC_CHECKPOINT_LAST_APPLIED) { + rc = cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); + } return rc; } @@ -4557,7 +4637,7 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b data->apply_last_db_version = decoded_context.db_version; data->apply_last_seq = decoded_context.seq; } - cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); + rc = cloudsync_payload_apply_checkpoint(data, checkpoint_db_version, checkpoint_seq); } cleanup: diff --git a/src/cloudsync.h b/src/cloudsync.h index 47ed5e38..0d9ed6ab 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -134,17 +134,26 @@ const char *cloudsync_table_schema (cloudsync_context *data, const char *table_n // on a complete db_version boundary, otherwise a stop between chunks of a single // source db_version silently skips the unapplied rows on the next /check (the // server's cloudsync_payload_chunks uses db_version > since with no seq cursor). +// The cursor also identifies the server's prepared pages, so it stays fixed for a +// whole receive stream and moves once, when the stream's final chunk has applied. // >= 0 advance the cursor to exactly this // (watermark_db_version), with checkpoint_seq. -// Used once a chunk stream is fully applied. +// Used for the final chunk of a stream: fails +// if a fragmented value the stream delivered +// is still incomplete. // CLOUDSYNC_CHECKPOINT_NONE do not advance the cursor. Used for a -// non-final chunk of a multi-chunk stream. +// non-final chunk of a stream. // CLOUDSYNC_CHECKPOINT_LAST_APPLIED advance to this artifact's last applied // (db_version, seq). Legacy/monolithic // behavior: safe only for a complete payload -// that ends on a db_version boundary. +// that ends on a db_version boundary. A v3 +// fragment never moves the cursor this way. +// NONE and an explicit watermark mark the call as part of a receive stream: the +// fragmented values it stages are tracked until applied. Reset that tracking with +// cloudsync_receive_stream_reset whenever a stream starts from its first page. #define CLOUDSYNC_CHECKPOINT_NONE (-1) #define CLOUDSYNC_CHECKPOINT_LAST_APPLIED (-2) +void cloudsync_receive_stream_reset (cloudsync_context *data); int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *nrows, int64_t checkpoint_db_version, int64_t checkpoint_seq); int cloudsync_payload_encode_step (cloudsync_payload_context *payload, cloudsync_context *data, int argc, dbvalue_t **argv); int cloudsync_payload_encode_final (cloudsync_payload_context *payload, cloudsync_context *data); diff --git a/src/network/network.c b/src/network/network.c index 0c900b55..88c3b135 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -95,6 +95,7 @@ struct network_data { // drain, so the server (which is stateless across /check calls) needs the client // to echo which spool page to serve next. In-memory only: losing it just // restarts the drain from page 0, which is safe because apply is idempotent. + // A failed chunk also restarts from page 0 on the next call. int64_t check_cursor; // next page index to request (0 = fresh drain) int64_t check_cursor_since; // the check_dbversion check_cursor belongs to #ifndef CLOUDSYNC_OMIT_CURL @@ -519,7 +520,17 @@ static size_t network_header_callback(char *buffer, size_t size, size_t nitems, return len; } +#ifdef CLOUDSYNC_UNITTEST +static NETWORK_RESULT (*network_test_responder)(const char *endpoint, const char *json_payload); +void network_test_set_responder (NETWORK_RESULT (*responder)(const char *, const char *)) { + network_test_responder = responder; +} +#endif + NETWORK_RESULT network_receive_buffer (network_data *data, const char *endpoint, const char *authentication, bool zero_terminated, bool is_post_request, char *json_payload, const char **extra_headers, int nextra_headers) { +#ifdef CLOUDSYNC_UNITTEST + if (network_test_responder) return network_test_responder(endpoint, json_payload); +#endif char *buffer = NULL; size_t blen = 0; struct curl_slist* headers = NULL; @@ -1478,6 +1489,12 @@ static char *network_base64_encode(const unsigned char *src, size_t len) { return out; } +#ifdef CLOUDSYNC_UNITTEST +char *network_test_base64_encode (const unsigned char *src, size_t len) { + return network_base64_encode(src, len); +} +#endif + static int network_base64_value(char c) { if (c >= 'A' && c <= 'Z') return c - 'A'; if (c >= 'a' && c <= 'z') return c - 'a' + 26; @@ -2018,6 +2035,8 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync netdata->check_cursor = 0; netdata->check_cursor_since = db_version; } + // Page 0 starts a fresh stream: forget what an earlier stream staged. + if (netdata->check_cursor == 0) cloudsync_receive_stream_reset(data); // Capture local db_version before download so we can query cloudsync_changes afterwards int64_t prev_dbv = cloudsync_dbversion(data); @@ -2158,10 +2177,12 @@ int cloudsync_network_check_internal(sqlite3_context *context, int *pnrows, sync if (tokens) cloudsync_memory_free(tokens); - if (rc == SQLITE_OK && delivered) { + if (rc != SQLITE_OK) { + // The next call replays the window from its first page, as a fresh stream. + netdata->check_cursor = 0; + if (pnrows) *pnrows = 0; + } else if (delivered) { // Finalize cursor state after the returned batch is applied/staged. - // Batched responses advance the in-memory spool cursor after each - // successful chunk, so a later failure retries from the failed chunk. netdata->check_cursor = more_pending ? next_cursor : 0; if (pnrows) *pnrows = rows_total; if (out) { diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c index 2f2dd433..0718ee4f 100644 --- a/src/postgresql/sql_postgresql.c +++ b/src/postgresql/sql_postgresql.c @@ -133,6 +133,9 @@ const char * const SQL_PAYLOAD_FRAGMENTS_SELECT = "SELECT fragment, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, checksum " "FROM cloudsync_payload_fragments WHERE value_id=$1 ORDER BY part_index ASC;"; +const char * const SQL_PAYLOAD_FRAGMENTS_EXISTS = + "SELECT 1 FROM cloudsync_payload_fragments WHERE value_id=$1 LIMIT 1;"; + const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = "DELETE FROM cloudsync_payload_fragments WHERE value_id=$1;"; diff --git a/src/sql.h b/src/sql.h index 36fbb8c6..fc337c84 100644 --- a/src/sql.h +++ b/src/sql.h @@ -71,6 +71,7 @@ extern const char * const SQL_PAYLOAD_FRAGMENTS_CREATE_TABLE; extern const char * const SQL_PAYLOAD_FRAGMENTS_UPSERT; extern const char * const SQL_PAYLOAD_FRAGMENTS_COUNT; extern const char * const SQL_PAYLOAD_FRAGMENTS_SELECT; +extern const char * const SQL_PAYLOAD_FRAGMENTS_EXISTS; extern const char * const SQL_PAYLOAD_FRAGMENTS_DELETE; extern const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE; diff --git a/src/sqlite/sql_sqlite.c b/src/sqlite/sql_sqlite.c index a2bb7d3e..06780f34 100644 --- a/src/sqlite/sql_sqlite.c +++ b/src/sqlite/sql_sqlite.c @@ -303,6 +303,9 @@ const char * const SQL_PAYLOAD_FRAGMENTS_SELECT = "SELECT fragment, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, checksum " "FROM cloudsync_payload_fragments WHERE value_id=? ORDER BY part_index ASC;"; +const char * const SQL_PAYLOAD_FRAGMENTS_EXISTS = + "SELECT 1 FROM cloudsync_payload_fragments WHERE value_id=? LIMIT 1;"; + const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = "DELETE FROM cloudsync_payload_fragments WHERE value_id=?;"; diff --git a/test/network_unit.c b/test/network_unit.c index 4d98845c..ba3b019e 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -252,6 +252,285 @@ static bool test_stalled_http_timeout(void) { } #endif + +// MARK: - Receive stream (canned /check responses, real apply) + +#include +#include "sqlite3.h" +#include "cloudsync.h" +#include "cloudsync_sqlite.h" +extern void network_test_set_responder(NETWORK_RESULT (*)(const char *, const char *)); +extern char *network_test_base64_encode(const unsigned char *, size_t); + +// The server spool of one receive window: its pages, and the watermark every chunk +// announces. Each request records the page cursor and the dbVersion it was sent with. +#define SPOOL_MAX 16 +static const char *spool[SPOOL_MAX]; +static int spool_pages; +static int64_t spool_watermark; +static int64_t req_cursor[64], req_since[64]; +static int nreq; + +static int64_t json_int_after(const char *json, const char *key, int64_t fallback) { + const char *p = strstr(json, key); + return p ? strtoll(p + strlen(key), NULL, 10) : fallback; +} + +static NETWORK_RESULT spool_responder(const char *endpoint, const char *request) { + NETWORK_RESULT r = {0}; + size_t n = endpoint ? strlen(endpoint) : 0; + if (n < 6 || strcmp(endpoint + n - 6, "/check") != 0 || !request) { r.code = CLOUDSYNC_NETWORK_ERROR; return r; } + int64_t cursor = json_int_after(request, "\"cursor\":", 0); + if (nreq < 64) { req_cursor[nreq] = cursor; req_since[nreq] = json_int_after(request, "\"dbVersion\":", -1); nreq++; } + int64_t max = json_int_after(request, "\"maxChunks\":", 1); + size_t cap = 256; + for (int i = 0; i < spool_pages; i++) cap += strlen(spool[i]) + 96; + char *json = cloudsync_memory_alloc(cap); + size_t len = (size_t)snprintf(json, cap, "{\"data\":{\"chunks\":["); + int64_t k = cursor; + for (; k < spool_pages && k < cursor + max; k++) { + len += (size_t)snprintf(json + len, cap - len, "%s{\"cursor\":%lld,\"payload\":\"%s\",\"watermark\":%lld}", + k > cursor ? "," : "", (long long)k, spool[k], (long long)spool_watermark); + } + bool final = k >= spool_pages; + snprintf(json + len, cap - len, "],\"final\":%s,\"nextCursor\":%lld}}", final ? "true" : "false", (long long)(final ? -1 : k)); + r.code = CLOUDSYNC_NETWORK_BUFFER; + r.buffer = json; + r.blen = strlen(json); + return r; +} + +static int db_exec(sqlite3 *db, const char *sql) { return sqlite3_exec(db, sql, NULL, NULL, NULL); } + +static int64_t db_int(sqlite3 *db, const char *sql) { + sqlite3_stmt *vm = NULL; + int64_t v = INT64_MIN; + if (sqlite3_prepare_v2(db, sql, -1, &vm, NULL) == SQLITE_OK && sqlite3_step(vm) == SQLITE_ROW) v = sqlite3_column_int64(vm, 0); + sqlite3_finalize(vm); + return v; +} + +static int64_t db_checkpoint(sqlite3 *db) { + return db_int(db, "SELECT coalesce((SELECT value FROM cloudsync_settings WHERE key='check_dbversion'),0)"); +} + +static sqlite3 *stream_db(bool network) { + sqlite3 *db = NULL; + if (sqlite3_open(":memory:", &db) != SQLITE_OK || sqlite3_cloudsync_init(db, NULL, NULL) != SQLITE_OK) return NULL; + if (db_exec(db, "CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, v BLOB); SELECT cloudsync_init('t');") != SQLITE_OK) return NULL; + if (network && db_exec(db, "SELECT cloudsync_network_init('test-managed-database-id');") != SQLITE_OK) return NULL; + return db; +} + +static void stream_close(sqlite3 *db) { + if (!db) return; + db_exec(db, "SELECT cloudsync_terminate();"); + sqlite3_close(db); +} + +// Base64 payloads of every row of src matching where (one monolithic payload each), +// or of the chunks cloudsync_payload_chunks produces. The caller frees them. +static int stream_payloads(sqlite3 *src, const char *query, char **out, int max) { + sqlite3_stmt *vm = NULL; + int n = 0; + if (sqlite3_prepare_v2(src, query, -1, &vm, NULL) != SQLITE_OK) return 0; + while (n < max && sqlite3_step(vm) == SQLITE_ROW) { + out[n++] = network_test_base64_encode(sqlite3_column_blob(vm, 0), (size_t)sqlite3_column_bytes(vm, 0)); + } + sqlite3_finalize(vm); + return n; +} + +// Applies every payload the query returns on src directly to dst, as a SQL caller would. +static bool direct_apply(sqlite3 *dst, sqlite3 *src, const char *query) { + sqlite3_stmt *read = NULL, *write = NULL; + bool ok = sqlite3_prepare_v2(src, query, -1, &read, NULL) == SQLITE_OK && + sqlite3_prepare_v2(dst, "SELECT cloudsync_payload_apply(?1)", -1, &write, NULL) == SQLITE_OK; + while (ok && sqlite3_step(read) == SQLITE_ROW) { + sqlite3_bind_value(write, 1, sqlite3_column_value(read, 0)); + ok = sqlite3_step(write) == SQLITE_ROW; + sqlite3_reset(write); + } + sqlite3_finalize(read); + sqlite3_finalize(write); + return ok; +} + +// Runs cloudsync_network_receive_changes(max_chunks); returns its JSON or "ERROR: ...". +static char *stream_receive(sqlite3 *db, int max_chunks) { + static char result[1024]; + sqlite3_stmt *vm = NULL; + snprintf(result, sizeof(result), "ERROR: prepare"); + if (sqlite3_prepare_v2(db, "SELECT cloudsync_network_receive_changes(?1)", -1, &vm, NULL) != SQLITE_OK) return result; + sqlite3_bind_int(vm, 1, max_chunks); + if (sqlite3_step(vm) == SQLITE_ROW) snprintf(result, sizeof(result), "%s", (const char *)sqlite3_column_text(vm, 0)); + else snprintf(result, sizeof(result), "ERROR: %s", sqlite3_errmsg(db)); + sqlite3_finalize(vm); + return result; +} + +static bool expect(bool ok, const char *what, const char *detail) { + if (!ok) printf("\n %s%s%s\n", what, detail ? ": " : "", detail ? detail : ""); + return ok; +} + +// Three monolithic pages. A call capped by max_chunks keeps the same window (dbVersion) +// and asks for the next page; the checkpoint moves only when the final page applied. +// A failed page restarts the next call from page 0, keeping the rows applied before it. +static bool test_stream_paging(void) { + bool ok = true; + sqlite3 *src = stream_db(false), *dst = stream_db(true); + char *pages[SPOOL_MAX] = {0}; + ok = ok && src && dst; + ok = ok && db_exec(src, "INSERT INTO t VALUES('k1',x'01'); INSERT INTO t VALUES('k2',x'02'); INSERT INTO t VALUES('k3',x'03');") == SQLITE_OK; + int n = ok ? stream_payloads(src, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes GROUP BY pk ORDER BY pk", pages, SPOOL_MAX) : 0; + ok = ok && expect(n == 3, "three pages", NULL); + for (int i = 0; i < n; i++) spool[i] = pages[i]; + spool_pages = n; + spool_watermark = ok ? db_int(src, "SELECT max(db_version) FROM cloudsync_changes") : 0; + nreq = 0; + network_test_set_responder(spool_responder); + + for (int call = 0; ok && call < 3; call++) { + char *json = stream_receive(dst, 1); + bool last = call == 2; + ok = expect(strstr(json, last ? "\"complete\":true" : "\"complete\":false") != NULL, "capped call", json) && ok; + ok = expect(nreq == call + 1 && req_cursor[call] == call && req_since[call] == 0, "capped call requests the next page of the same window", NULL) && ok; + ok = expect(db_checkpoint(dst) == (last ? spool_watermark : 0), "checkpoint moves only after the final page", json) && ok; + } + ok = ok && expect(db_int(dst, "SELECT count(*) FROM t") == 3, "all rows applied", NULL); + + // failure in the middle page: the first row stays, nothing is checkpointed, and the + // next call starts again from page 0 of the same window + stream_close(dst); + dst = stream_db(true); + ok = ok && dst && db_exec(dst, "CREATE TRIGGER deny BEFORE INSERT ON t WHEN NEW.id='k2' BEGIN SELECT RAISE(ABORT,'k2 denied'); END") == SQLITE_OK; + nreq = 0; + char *json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "k2 denied") && strstr(json, "\"complete\":false") && strstr(json, "\"rows\":1,\"tables\":[\"t\"]"), "failure reports the error and the rows applied before it", json); + ok = ok && expect(db_checkpoint(dst) == 0, "no checkpoint after a failure", NULL); + ok = ok && db_exec(dst, "DROP TRIGGER deny") == SQLITE_OK; + int before = nreq; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(nreq > before && req_cursor[before] == 0 && req_since[before] == 0, "the call after a failure starts from page 0", NULL); + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark, "replay completes the window", json); + ok = ok && expect(db_int(dst, "SELECT count(*) FROM t") == 3, "all rows applied after the replay", NULL); + + // a checkpoint that cannot be written is reported, not hidden + stream_close(dst); + dst = stream_db(true); + ok = ok && dst && db_exec(dst, "CREATE TRIGGER deny_ckpt BEFORE INSERT ON cloudsync_settings WHEN NEW.key='check_dbversion' BEGIN SELECT RAISE(ABORT,'checkpoint denied'); END") == SQLITE_OK; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "receive checkpoint") && strstr(json, "\"complete\":false") && db_checkpoint(dst) == 0, "checkpoint write failure is reported", json); + + network_test_set_responder(NULL); + for (int i = 0; i < n; i++) cloudsync_memory_free(pages[i]); + stream_close(src); + stream_close(dst); + return ok; +} + +// One value too large for a chunk, sent as fragments. +#define FRAGMENT_CHUNKS "SELECT payload FROM cloudsync_payload_chunks() WHERE substr(payload,5,1)=x'03' ORDER BY chunk_index" +static bool test_stream_fragments(void) { + bool ok = true; + sqlite3 *src = stream_db(false), *dst = NULL; + char *frags[SPOOL_MAX] = {0}, *other[1] = {0}; + ok = ok && src && db_exec(src, "SELECT cloudsync_set('payload_max_chunk_size','262144'); INSERT INTO t VALUES('big', randomblob(700000));") == SQLITE_OK; + int nf = ok ? stream_payloads(src, "SELECT payload FROM cloudsync_payload_chunks() ORDER BY chunk_index", frags, SPOOL_MAX) : 0; + ok = ok && expect(nf >= 3, "value split into three or more fragments", NULL); + ok = ok && db_exec(src, "INSERT INTO t VALUES('small', x'05')") == SQLITE_OK; + int no = ok ? stream_payloads(src, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes WHERE pk=cloudsync_pk_encode('small')", other, 1) : 0; + ok = ok && expect(no == 1, "monolithic page", NULL); + spool_watermark = ok ? db_int(src, "SELECT max(db_version) FROM cloudsync_changes") : 0; + network_test_set_responder(spool_responder); + const char *big_ok = "SELECT count(*) FROM t WHERE id='big' AND length(v)=700000"; + char *json = ""; + + // a stream that ends with a delivered value incomplete fails before checkpointing; + // the fresh replay from page 0 then completes it + dst = stream_db(true); + spool_pages = 0; + for (int i = 0; i < nf; i++) if (i != 1) spool[spool_pages++] = frags[i]; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "incomplete fragmented value") && strstr(json, "\"complete\":false") && db_checkpoint(dst) == 0, "incomplete value at the final chunk", json); + spool_pages = 0; + for (int i = 0; i < nf; i++) spool[spool_pages++] = frags[i]; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark && db_int(dst, big_ok) == 1, "fresh replay completes the value", json); + ok = ok && expect(db_int(dst, "SELECT count(*) FROM cloudsync_payload_fragments") == 0, "pieces removed once applied", NULL); + stream_close(dst); + + // out-of-order and duplicate pieces, across calls capped to one page + dst = stream_db(true); + spool_pages = 0; + spool[spool_pages++] = frags[nf - 1]; + spool[spool_pages++] = frags[0]; + spool[spool_pages++] = frags[0]; + for (int i = 1; i < nf - 1; i++) spool[spool_pages++] = frags[i]; + for (int call = 0; ok && call < spool_pages; call++) json = stream_receive(dst, 1); + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark && db_int(dst, big_ok) == 1, "out-of-order and duplicate pieces", json); + stream_close(dst); + + // a direct fragment call never moves the checkpoint, and the staging it leaves blocks + // neither a direct monolithic call nor a stream that does not deliver that value + dst = stream_db(true); + ok = ok && expect(direct_apply(dst, src, FRAGMENT_CHUNKS " LIMIT 1") && db_checkpoint(dst) == 0, "direct fragment call does not checkpoint", NULL); + ok = ok && expect(direct_apply(dst, src, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes WHERE pk=cloudsync_pk_encode('small')") && db_checkpoint(dst) == spool_watermark, "staged fragments do not block a direct monolithic call", NULL); + stream_close(dst); + dst = stream_db(true); + ok = ok && direct_apply(dst, src, FRAGMENT_CHUNKS " LIMIT 1"); + spool[0] = other[0]; + spool_pages = 1; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark && db_int(dst, "SELECT count(*) FROM cloudsync_payload_fragments") > 0, "staging from a direct call does not block a stream", json); + stream_close(dst); + + // a value abandoned by an interrupted stream: the replay after the failure no longer + // delivers it (replaced upstream), and completes + dst = stream_db(true); + spool[0] = frags[0]; + spool[1] = other[0]; + spool_pages = 2; + json = ok ? stream_receive(dst, 1) : ""; + ok = ok && db_exec(dst, "CREATE TRIGGER deny BEFORE INSERT ON t WHEN NEW.id='small' BEGIN SELECT RAISE(ABORT,'small denied'); END") == SQLITE_OK; + json = ok ? stream_receive(dst, 1) : ""; + ok = ok && expect(strstr(json, "small denied") != NULL, "interrupting failure", json); + ok = ok && db_exec(dst, "DROP TRIGGER deny") == SQLITE_OK; + spool[0] = other[0]; + spool_pages = 1; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark, "abandoned group does not block the replay", json); + stream_close(dst); + + // completed by another caller before the stream delivers it: the stream stages the + // pieces again and re-applies the value as a no-op + dst = stream_db(true); + ok = ok && direct_apply(dst, src, FRAGMENT_CHUNKS) && db_int(dst, big_ok) == 1; + spool_pages = 0; + for (int i = 0; i < nf; i++) spool[spool_pages++] = frags[i]; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark, "value completed by another caller first", json); + stream_close(dst); + + // completed by another caller while the stream is delivering it: the stream's later + // pieces are staged again, the final check fails once, and the replay passes + dst = stream_db(true); + json = ok ? stream_receive(dst, 1) : ""; + ok = ok && direct_apply(dst, src, FRAGMENT_CHUNKS) && db_int(dst, big_ok) == 1; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "incomplete fragmented value") && db_checkpoint(dst) == 0, "value completed by another caller mid-stream fails once", json); + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == spool_watermark && db_int(dst, "SELECT count(*) FROM cloudsync_payload_fragments") == 0, "the replay passes", json); + stream_close(dst); + + network_test_set_responder(NULL); + for (int i = 0; i < nf; i++) cloudsync_memory_free(frags[i]); + if (other[0]) cloudsync_memory_free(other[0]); + stream_close(src); + return ok; +} + int main(void) { #if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) check("HTTP deadlines: API elapsed cap and artifact stall cap:", test_stalled_http_timeout()); @@ -265,6 +544,8 @@ int main(void) { check("non-buffer response is a no-op:", test_non_buffer_is_noop()); check("send batch /apply payload (window / batchId / chunkIndex / isFinal):", test_apply_json_payload_batch()); check("network_compute_status:", test_compute_status()); + check("receive stream: capped paging, failure replay, checkpoint errors:", test_stream_paging()); + check("receive stream: fragmented values:", test_stream_fragments()); if (failures) { printf("\n%d test(s) FAILED\n", failures); return 1; } printf("\nAll network unit tests passed\n"); return 0; diff --git a/test/postgresql/58_v3_denied_checkpoint.sql b/test/postgresql/58_v3_denied_checkpoint.sql index 1c05651d..f8d03009 100644 --- a/test/postgresql/58_v3_denied_checkpoint.sql +++ b/test/postgresql/58_v3_denied_checkpoint.sql @@ -111,8 +111,10 @@ SET ROLE v3_denied_user; SELECT format('SELECT cloudsync_payload_apply(payload) FROM chunk_transport WHERE ord = %s;', ord) FROM chunk_transport ORDER BY ord \gexec RESET ROLE; +-- a direct fragment call has no end-of-stream marker: it never moves the checkpoint SELECT (SELECT length(note) FROM frag_rls WHERE id = 'big') = 655360 - AND NOT EXISTS (SELECT FROM cloudsync_payload_fragments) AS redelivered_ok \gset + AND NOT EXISTS (SELECT FROM cloudsync_payload_fragments) + AND coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) = :ckpt_before::bigint AS redelivered_ok \gset \if :redelivered_ok \echo [PASS] (:testid) redelivery after the policy change applies the value and clears its pieces \else From 684aae36c52dd545e928858f02741e7d7fbb3575 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 20:12:41 -0600 Subject: [PATCH 21/28] fix(fragments): keep resumed groups and make each fragment call one unit - Stale cleanup removes a group only when all its pieces are older than the retention window, and runs after the incoming piece is staged, so a value resumed after a long pause keeps its earlier pieces. Retention and throttle are unchanged. - The cleanup runs in its own savepoint: a failure is rolled back and logged instead of failing the piece being applied. - Staging a piece, reassembling, applying the value and removing its pieces run under one savepoint; a failure to remove the pieces is now returned instead of ignored, and pieces staged by earlier calls stay for a retry. - A failed staging count query is reported instead of read as "incomplete". Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 1 + src/cloudsync.c | 128 +++++++++++++++++--------------- src/postgresql/sql_postgresql.c | 5 +- src/sqlite/sql_sqlite.c | 5 +- test/review_regressions.c | 105 ++++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99240fa3..465a5dca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **A receive stream no longer checkpoints past a fragmented value it delivered incompletely.** When the stream's final chunk arrives while a value it delivered is still missing pieces, the call now fails with an error and the checkpoint stays in place. Pieces staged by other streams or direct calls are ignored, so they never block a receive. After any failed receive, the next call downloads the stream again from its first page, which re-checks every value against the whole stream; the checkpoint still moves only when the stream completes. - **A receive checkpoint that cannot be written is reported** as a receive error instead of being ignored. - **Applying a fragment directly with `cloudsync_payload_apply` no longer moves the receive checkpoint.** A fragment carries no end-of-stream marker, so completing a value in a direct call could advance the cursor past changes not received yet. +- **Staged fragments are cleaned up without breaking a value in progress.** The cleanup of fragment groups left incomplete for 24 hours removed the old pieces of a value whose delivery had just resumed, so that value could never complete; it now removes a group only when all of its pieces are old. A failing cleanup no longer fails the fragment being applied: it is rolled back and logged. Applying a completed value and removing its pieces now happen together, so a failure to remove them is reported and the value is applied again on the next delivery, instead of the failure being ignored. - **PostgreSQL: applying a payload read from a table no longer fails.** `SELECT cloudsync_payload_apply(data) FROM some_table` failed with `buffer pin ... is not owned by resource owner TopTransaction` (and could abort an assertion-enabled server): the internal savepoints left the caller's resource owner and memory context switched. Both are now restored when a savepoint ends. - **Block columns and grow-only-set tables merge under row-level security and NOT NULL constraints.** Both wrote a received column through an upsert holding only the primary key and that column: PostgreSQL checks an INSERT policy against that proposed row even when it becomes an update, and a NOT NULL column rejects it outright, so the column was never written. A block column's row now has its other received columns written first, and an existing row — of either kind — is updated in place. When the update finds no row to change (the row is gone, or an UPDATE policy hides it), the upsert is used as before, so the write is reported rather than recorded as applied without being stored. - **A received row whose write fails leaves no partial state behind.** Applying a row wrote part of its metadata — for a deleted-and-recreated row, the new sentinel and the reset column clocks — before the row itself, outside the savepoint that protected the write. When the write then failed, the metadata claimed the row was there: delivered again, even after the cause was fixed (a lock that went away, a policy that now allows it), the row was silently never created, and an existing row kept zeroed clocks. Each primary key's changes are now applied under one savepoint that the failed write rolls back entirely. diff --git a/src/cloudsync.c b/src/cloudsync.c index 91223709..fa06c4ed 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -3892,6 +3892,8 @@ static int cloudsync_payload_fragments_cleanup_stale (cloudsync_context *data) { // every applied fragment would be O(n^2) for a heavily-fragmented value, since // each fragment arrives as its own apply call. Throttle it to at most once per // CLOUDSYNC_PAYLOAD_FRAGMENT_CLEANUP_MIN_INTERVAL per connection. + // Its own savepoint keeps a failed cleanup from failing the piece being applied: + // it is rolled back and logged. Only a savepoint that cannot be managed is an error. int64_t now = (int64_t)time(NULL); if (data->last_fragment_cleanup != 0 && now - data->last_fragment_cleanup < CLOUDSYNC_PAYLOAD_FRAGMENT_CLEANUP_MIN_INTERVAL) { @@ -3899,14 +3901,22 @@ static int cloudsync_payload_fragments_cleanup_stale (cloudsync_context *data) { } data->last_fragment_cleanup = now; - dbvm_t *vm = NULL; - int rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE, &vm, 0); + int rc = database_begin_savepoint(data, "cloudsync_fragment_cleanup"); if (rc != DBRES_OK) return rc; - int64_t cutoff = now - CLOUDSYNC_PAYLOAD_FRAGMENT_STALE_SECONDS; - rc = databasevm_bind_int(vm, 1, cutoff); + dbvm_t *vm = NULL; + rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE, &vm, 0); + if (rc == DBRES_OK) rc = databasevm_bind_int(vm, 1, now - CLOUDSYNC_PAYLOAD_FRAGMENT_STALE_SECONDS); if (rc == DBRES_OK) rc = databasevm_step(vm); - databasevm_finalize(vm); - return (rc == DBRES_DONE) ? DBRES_OK : rc; + if (vm) databasevm_finalize(vm); + if (rc == DBRES_DONE) return database_commit_savepoint(data, "cloudsync_fragment_cleanup"); + + char warning[1100]; + snprintf(warning, sizeof(warning), "stale fragment cleanup failed: %s", cloudsync_errmsg(data)); + rc = database_rollback_savepoint(data, "cloudsync_fragment_cleanup"); + if (rc != DBRES_OK) return rc; + cloudsync_reset_error(data); + database_log_warning(data, warning); + return DBRES_OK; } static int cloudsync_payload_apply_single_decoded_row (cloudsync_context *data, @@ -3920,7 +3930,6 @@ static int cloudsync_payload_apply_single_decoded_row (cloudsync_context *data, int *pnrows) { int rc = DBRES_OK; dbvm_t *vm = NULL; - bool in_savepoint = false; merge_pending_batch batch = {0}; rc = databasevm_prepare(data, SQL_CHANGES_INSERT_ROW, &vm, 0); @@ -3946,12 +3955,6 @@ static int cloudsync_payload_apply_single_decoded_row (cloudsync_context *data, if (rc == DBRES_OK) rc = databasevm_bind_int(vm, 9, seq); if (rc != DBRES_OK) goto cleanup; - if (!database_in_transaction(data)) { - rc = database_begin_savepoint(data, "cloudsync_payload_apply"); - if (rc != DBRES_OK) goto cleanup; - in_savepoint = true; - } - data->pending_batch = &batch; rc = databasevm_step(vm); if (rc == DBRES_DONE) rc = DBRES_OK; @@ -3964,12 +3967,6 @@ static int cloudsync_payload_apply_single_decoded_row (cloudsync_context *data, if (rc != DBRES_OK) goto cleanup; data->pending_batch = NULL; - if (in_savepoint) { - rc = database_commit_savepoint(data, "cloudsync_payload_apply"); - in_savepoint = false; - if (rc != DBRES_OK) goto cleanup; - } - // Do NOT advance the receive cursor here: a v3 value carries a single // (db_version, seq) that can be in the middle of its source db_version, and a // db_version's chunks can span multiple /check artifacts. Advancing per value @@ -3986,7 +3983,6 @@ static int cloudsync_payload_apply_single_decoded_row (cloudsync_context *data, if (pnrows) *pnrows += 1; cleanup: - if (rc != DBRES_OK && in_savepoint) database_rollback_savepoint(data, "cloudsync_payload_apply"); data->pending_batch = NULL; merge_pending_free_entries(&batch); if (batch.cached_vm) databasevm_finalize(batch.cached_vm); @@ -4011,7 +4007,11 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data rc = databasevm_bind_text(vm, 1, value_id, -1); if (rc != DBRES_OK) { databasevm_finalize(vm); return rc; } rc = databasevm_step(vm); - if (rc != DBRES_ROW) { databasevm_finalize(vm); return DBRES_OK; } + if (rc != DBRES_ROW) { + cloudsync_set_dberror(data); + databasevm_finalize(vm); + return (rc == DBRES_DONE) ? DBRES_OK : rc; + } int64_t have = database_column_int(vm, 0); int64_t part_count_min = database_column_int(vm, 1); int64_t part_count_max = database_column_int(vm, 2); @@ -4106,23 +4106,14 @@ static int cloudsync_payload_apply_reassembled_fragment (cloudsync_context *data rc = cloudsync_payload_apply_single_decoded_row(data, tbl, tbl_len, pk, pk_len, col_name, col_name_len, value, (size_t)total_size, col_version, db_version, site_id, site_id_len, cl, seq, pnrows); - // A denial leaves the transaction unusable until the caller's savepoint rolls it - // back, so the staged fragments cannot be dropped here. They are bounded by the - // stale-fragment cleanup instead. if (rc != DBRES_OK) goto cleanup; + // the caller's savepoint makes applying the value and removing its pieces one unit rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_DELETE, &vm, 0); - if (rc == DBRES_OK) { - databasevm_bind_text(vm, 1, value_id, -1); - // A failed delete is deliberately tolerated rather than propagated: the value - // is already applied, so failing here would stall the cursor and re-deliver - // it, and a delete that fails once fails again on every retry. The leftover - // rows are bounded by the stale-fragment cleanup. - // (The former `if (step_rc == DBRES_DONE) rc = DBRES_OK;` only looked like a - // check: rc was already DBRES_OK from the prepare.) - databasevm_step(vm); - } - + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 1, value_id, -1); + if (rc == DBRES_OK) rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + else cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to remove applied fragments", rc); cleanup: if (vm) databasevm_finalize(vm); @@ -4168,37 +4159,54 @@ static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, clouds cloudsync_stream_value *seen = track ? cloudsync_stream_value_find(data, value_id) : NULL; if (seen && seen->applied) return DBRES_OK; - // the fragments table is guaranteed by dbutils_settings_init; no DDL here - // because the apply path runs under sync-only credentials on server nodes - int rc = cloudsync_payload_fragments_cleanup_stale(data); + // Stage the piece, then reassemble, apply and remove the value's pieces as one + // unit: a failure rolls back only this call and leaves the pieces staged by + // earlier calls, so the value stays retryable. + int rc = database_begin_savepoint(data, "cloudsync_fragment"); if (rc != DBRES_OK) return rc; + // the fragments table is guaranteed by dbutils_settings_init; no DDL here + // because the apply path runs under sync-only credentials on server nodes dbvm_t *vm = NULL; rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_UPSERT, &vm, 0); - if (rc != DBRES_OK) return rc; - databasevm_bind_text(vm, 1, value_id, -1); - databasevm_bind_int(vm, 2, part_index); - databasevm_bind_int(vm, 3, part_count); - databasevm_bind_int(vm, 4, total_size); - databasevm_bind_text(vm, 5, checksum_hex, -1); - databasevm_bind_int(vm, 6, (int64_t)time(NULL)); - databasevm_bind_text(vm, 7, row->tbl, (int)row->tbl_len); - databasevm_bind_blob(vm, 8, row->pk, (uint64_t)row->pk_len); - databasevm_bind_text(vm, 9, base_col, (int)base_col_len); - databasevm_bind_int(vm, 10, row->col_version); - databasevm_bind_int(vm, 11, row->db_version); - databasevm_bind_blob(vm, 12, row->site_id, (uint64_t)row->site_id_len); - databasevm_bind_int(vm, 13, row->cl); - databasevm_bind_int(vm, 14, row->seq); - databasevm_bind_blob(vm, 15, row->col_value, (uint64_t)row->col_value_len); - rc = databasevm_step(vm); - databasevm_finalize(vm); - if (rc == DBRES_DONE) rc = DBRES_OK; - if (rc != DBRES_OK) return rc; + if (rc == DBRES_OK) { + databasevm_bind_text(vm, 1, value_id, -1); + databasevm_bind_int(vm, 2, part_index); + databasevm_bind_int(vm, 3, part_count); + databasevm_bind_int(vm, 4, total_size); + databasevm_bind_text(vm, 5, checksum_hex, -1); + databasevm_bind_int(vm, 6, (int64_t)time(NULL)); + databasevm_bind_text(vm, 7, row->tbl, (int)row->tbl_len); + databasevm_bind_blob(vm, 8, row->pk, (uint64_t)row->pk_len); + databasevm_bind_text(vm, 9, base_col, (int)base_col_len); + databasevm_bind_int(vm, 10, row->col_version); + databasevm_bind_int(vm, 11, row->db_version); + databasevm_bind_blob(vm, 12, row->site_id, (uint64_t)row->site_id_len); + databasevm_bind_int(vm, 13, row->cl); + databasevm_bind_int(vm, 14, row->seq); + databasevm_bind_blob(vm, 15, row->col_value, (uint64_t)row->col_value_len); + rc = databasevm_step(vm); + if (rc == DBRES_DONE) rc = DBRES_OK; + else cloudsync_set_dberror(data); + } + if (vm) databasevm_finalize(vm); + + // After staging, so a group resumed after a long pause is recent again and is kept. + if (rc == DBRES_OK) rc = cloudsync_payload_fragments_cleanup_stale(data); int applied = 0; - rc = cloudsync_payload_apply_reassembled_fragment(data, value_id, checksum_hex, &applied); - if (rc != DBRES_OK) return rc; + if (rc == DBRES_OK) rc = cloudsync_payload_apply_reassembled_fragment(data, value_id, checksum_hex, &applied); + if (rc == DBRES_OK) rc = database_commit_savepoint(data, "cloudsync_fragment"); + if (rc != DBRES_OK) { + char message[1024]; + snprintf(message, sizeof(message), "%s", cloudsync_errmsg(data)); + int sqlstate = cloudsync_sqlstate(data); + database_rollback_savepoint(data, "cloudsync_fragment"); + cloudsync_reset_error(data); + cloudsync_set_error(data, message[0] ? message : "Unable to apply a fragment", rc); + cloudsync_set_sqlstate(data, sqlstate); + return rc; + } if (pnrows) *pnrows += applied; return track ? cloudsync_stream_value_track(data, value_id, applied > 0) : DBRES_OK; } diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c index 0718ee4f..1508073a 100644 --- a/src/postgresql/sql_postgresql.c +++ b/src/postgresql/sql_postgresql.c @@ -140,10 +140,9 @@ const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = "DELETE FROM cloudsync_payload_fragments WHERE value_id=$1;"; const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE = - "DELETE FROM cloudsync_payload_fragments " - "WHERE created_at < $1 AND value_id IN (" + "DELETE FROM cloudsync_payload_fragments WHERE value_id IN (" "SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " - "HAVING COUNT(*) < MAX(part_count));"; + "HAVING MAX(created_at) < $1 AND COUNT(*) < MAX(part_count));"; // MARK: Additional SQL constants for PostgreSQL diff --git a/src/sqlite/sql_sqlite.c b/src/sqlite/sql_sqlite.c index 06780f34..229fe2b9 100644 --- a/src/sqlite/sql_sqlite.c +++ b/src/sqlite/sql_sqlite.c @@ -310,10 +310,9 @@ const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = "DELETE FROM cloudsync_payload_fragments WHERE value_id=?;"; const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE = - "DELETE FROM cloudsync_payload_fragments " - "WHERE created_at < ? AND value_id IN (" + "DELETE FROM cloudsync_payload_fragments WHERE value_id IN (" "SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " - "HAVING COUNT(*) < MAX(part_count));"; + "HAVING MAX(created_at) < ? AND COUNT(*) < MAX(part_count));"; // MARK: Blocks (block-level LWW) diff --git a/test/review_regressions.c b/test/review_regressions.c index b551502d..5618a5df 100644 --- a/test/review_regressions.c +++ b/test/review_regressions.c @@ -260,6 +260,110 @@ static void test_batched_update_missing_row(void) { CHECK(close_db(source) == SQLITE_OK); CHECK(close_db(target) == SQLITE_OK); } +// The v3 fragments of one value too large for a chunk, read from a source database. +#define MAX_FRAGS 8 +static int frag_count; +static void *frag_data[MAX_FRAGS]; +static int frag_size[MAX_FRAGS]; +static const char *frag_schema = "CREATE TABLE f(id TEXT PRIMARY KEY NOT NULL, v BLOB); SELECT cloudsync_init('f');"; +static void frags_load(void) { + sqlite3 *src = open_db(); + sqlite3_stmt *vm = NULL; + CHECK(sql(src, frag_schema) == SQLITE_OK); + CHECK(sql(src, "SELECT cloudsync_set('payload_max_chunk_size','262144'); INSERT INTO f VALUES('big', randomblob(700000));") == SQLITE_OK); + CHECK(sqlite3_prepare_v2(src, "SELECT payload FROM cloudsync_payload_chunks() WHERE substr(payload,5,1)=x'03' ORDER BY chunk_index", -1, &vm, NULL) == SQLITE_OK); + while (frag_count < MAX_FRAGS && vm && sqlite3_step(vm) == SQLITE_ROW) { + frag_size[frag_count] = sqlite3_column_bytes(vm, 0); + frag_data[frag_count] = malloc((size_t)frag_size[frag_count]); + memcpy(frag_data[frag_count], sqlite3_column_blob(vm, 0), (size_t)frag_size[frag_count]); + frag_count++; + } + sqlite3_finalize(vm); + CHECK(frag_count >= 3); + CHECK(close_db(src) == SQLITE_OK); +} +static int frag_apply(sqlite3 *db, int i) { + sqlite3_stmt *vm = NULL; + CHECK(sqlite3_prepare_v2(db, "SELECT cloudsync_payload_apply(?1)", -1, &vm, NULL) == SQLITE_OK); + sqlite3_bind_blob(vm, 1, frag_data[i], frag_size[i], SQLITE_STATIC); + int rc = sqlite3_step(vm); + sqlite3_finalize(vm); + return rc; +} +static sqlite3 *frag_target(const char *path) { + sqlite3 *db = NULL; + CHECK(sqlite3_open(path ? path : ":memory:", &db) == SQLITE_OK); + CHECK(sqlite3_cloudsync_init(db, NULL, NULL) == SQLITE_OK); + CHECK(sql(db, "CREATE TABLE IF NOT EXISTS f(id TEXT PRIMARY KEY NOT NULL, v BLOB); SELECT cloudsync_init('f');") == SQLITE_OK); + return db; +} +// an incomplete group whose only piece is two days old +static const char *stale_group = "INSERT INTO cloudsync_payload_fragments (value_id, part_index, part_count, total_size, checksum, created_at, tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) " + "VALUES ('00000000000000000000000000000000', 0, 2, 2, '0000000000000000', strftime('%s','now') - 172800, 'f', x'00', 'v', 1, 1, x'00', 1, 0, x'00')"; +static void test_fragment_retention(void) { + frags_load(); + const char *big = "SELECT count(*) FROM f WHERE id='big' AND length(v)=700000"; + char path[512]; + CHECK(scratch_create()); + snprintf(path, sizeof(path), "%s/frags.db", scratch_dir); + + // A group resumed after two days keeps its old pieces: the cleanup runs after the + // new piece is staged, when the group is recent again. A fully stale group goes. + sqlite3 *db = frag_target(path); + CHECK(frag_apply(db, 0) == SQLITE_ROW); + CHECK(sql(db, "UPDATE cloudsync_payload_fragments SET created_at = created_at - 172800") == SQLITE_OK); + CHECK(sql(db, stale_group) == SQLITE_OK); + CHECK(close_db(db) == SQLITE_OK); + db = frag_target(path); // a new connection: its first fragment runs the cleanup + CHECK(frag_apply(db, 1) == SQLITE_ROW); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_payload_fragments WHERE value_id <> '00000000000000000000000000000000'") == 2); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_payload_fragments WHERE value_id = '00000000000000000000000000000000'") == 0); + for (int i = 2; i < frag_count; i++) CHECK(frag_apply(db, i) == SQLITE_ROW); + CHECK(scalar(db, big) == 1); + CHECK(close_db(db) == SQLITE_OK); + const char *const files[] = {"frags.db", "frags.db-journal"}; + scratch_remove(files, 2); + + // A cleanup that fails is rolled back and logged; the value still applies. + db = frag_target(NULL); + CHECK(sql(db, stale_group) == SQLITE_OK); + CHECK(sql(db, "CREATE TRIGGER no_cleanup BEFORE DELETE ON cloudsync_payload_fragments WHEN OLD.tbl='f' AND OLD.pk=x'00' BEGIN SELECT RAISE(ABORT,'cleanup denied'); END") == SQLITE_OK); + for (int i = 0; i < frag_count; i++) CHECK(frag_apply(db, i) == SQLITE_ROW); + CHECK(scalar(db, big) == 1); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_payload_fragments") == 1); + CHECK(close_db(db) == SQLITE_OK); + + // A value whose write fails keeps the pieces staged by earlier calls, and applies + // once the last piece is delivered again. + db = frag_target(NULL); + CHECK(sql(db, "CREATE TRIGGER deny BEFORE INSERT ON f WHEN NEW.id='big' BEGIN SELECT RAISE(ABORT,'big denied'); END") == SQLITE_OK); + for (int i = 0; i < frag_count - 1; i++) CHECK(frag_apply(db, i) == SQLITE_ROW); + CHECK(frag_apply(db, frag_count - 1) != SQLITE_ROW); + CHECK(strstr(sqlite3_errmsg(db), "big denied") != NULL); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_payload_fragments") == frag_count - 1); + CHECK(sql(db, "DROP TRIGGER deny") == SQLITE_OK); + CHECK(frag_apply(db, frag_count - 1) == SQLITE_ROW); + CHECK(scalar(db, big) == 1); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_payload_fragments") == 0); + CHECK(close_db(db) == SQLITE_OK); + + // Applying the value and removing its pieces is one unit: when the pieces cannot be + // removed, the value is not applied either, and the call fails. + db = frag_target(NULL); + CHECK(sql(db, "CREATE TRIGGER keep BEFORE DELETE ON cloudsync_payload_fragments BEGIN SELECT RAISE(ABORT,'delete denied'); END") == SQLITE_OK); + for (int i = 0; i < frag_count - 1; i++) CHECK(frag_apply(db, i) == SQLITE_ROW); + CHECK(frag_apply(db, frag_count - 1) != SQLITE_ROW); + CHECK(strstr(sqlite3_errmsg(db), "delete denied") != NULL); + CHECK(scalar(db, "SELECT count(*) FROM f") == 0); + CHECK(scalar(db, "SELECT count(*) FROM f_cloudsync") == 0); + CHECK(scalar(db, "SELECT count(*) FROM cloudsync_payload_fragments") == frag_count - 1); + CHECK(sql(db, "DROP TRIGGER keep") == SQLITE_OK); + CHECK(frag_apply(db, frag_count - 1) == SQLITE_ROW); + CHECK(scalar(db, big) == 1); + CHECK(close_db(db) == SQLITE_OK); + + for (int i = 0; i < frag_count; i++) free(frag_data[i]); +} static void test_block_write_errors(void) { for (int update = 0; update < 2; update++) { sqlite3 *db = open_db(); @@ -412,6 +516,7 @@ int main(void) { test_payload_high_compression(); test_resurrected_group_rollback(); test_batched_update_missing_row(); + test_fragment_retention(); test_block_write_errors(); test_block_materialize_errors(); test_block_migration_orphan(); From ce0557fdf52838e2fcd8617a4bdde31cc20f2ab2 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 20:25:48 -0600 Subject: [PATCH 22/28] fix(postgres): serialize concurrent applies of one fragmented value The server applies each uploaded chunk as its own job, so pieces of one value can reach concurrent transactions. Under READ COMMITTED each could stage its piece, see only its own, and commit without ever applying the value, and the stale cleanup could remove the old pieces of a value another transaction was resuming (both reproduced by the new test before the fix). - READ COMMITTED: a transaction-level advisory lock per value_id makes the pieces wait for each other; after the wait the next statement sees the committed pieces. The stale cleanup try-locks the same key and skips a value being applied. - SERIALIZABLE takes no lock: the lost-value outcome matches no serial order, so one transaction fails with a retryable serialization error. - REPEATABLE READ is refused for fragments (0A000): a waiter would keep the snapshot taken before the wait and miss the other piece. - SQLite already serializes writers; its lock is a no-op. Add 60_fragment_concurrency.sql (dblink) for the three isolation levels and the cleanup race. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 2 +- CHANGELOG.md | 1 + src/cloudsync.c | 6 +- src/database.h | 1 + src/postgresql/database_postgresql.c | 23 +++ src/postgresql/sql_postgresql.c | 6 +- src/sqlite/database_sqlite.c | 5 + test/postgresql/60_fragment_concurrency.sql | 173 ++++++++++++++++++++ test/postgresql/full_test.sql | 1 + 9 files changed, 214 insertions(+), 4 deletions(-) create mode 100644 test/postgresql/60_fragment_concurrency.sql diff --git a/API.md b/API.md index c098458b..9a66edcd 100644 --- a/API.md +++ b/API.md @@ -651,7 +651,7 @@ On PostgreSQL, apply chunks as individual statements from the transport/client l - Monolithic payloads generated by [`cloudsync_payload_encode()`](#cloudsync_payload_encodetbl-pk-col_name-col_value-col_version-db_version-site_id-cl-seq). - Chunk-fragment payloads generated by [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id). -When a v3 fragment payload is received, CloudSync stores the fragment in an internal table and returns after applying zero or more completed values. Once the final fragment for a value is received, the completed value is validated and applied. Fragments can arrive in any order, and duplicate fragment delivery is idempotent. Applying a fragment never moves the receive checkpoint. +When a v3 fragment payload is received, CloudSync stores the fragment in an internal table and returns after applying zero or more completed values. Once the final fragment for a value is received, the completed value is validated and applied. Fragments can arrive in any order, and duplicate fragment delivery is idempotent. Applying a fragment never moves the receive checkpoint. On PostgreSQL, pieces of one value applied by concurrent transactions wait for each other under `READ COMMITTED`, and fail with a retryable serialization error under `SERIALIZABLE` when they conflict; a fragment is refused under `REPEATABLE READ`, where a transaction could miss a piece committed while it waited. **Parameters:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 465a5dca..3442fb74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **A receive checkpoint that cannot be written is reported** as a receive error instead of being ignored. - **Applying a fragment directly with `cloudsync_payload_apply` no longer moves the receive checkpoint.** A fragment carries no end-of-stream marker, so completing a value in a direct call could advance the cursor past changes not received yet. - **Staged fragments are cleaned up without breaking a value in progress.** The cleanup of fragment groups left incomplete for 24 hours removed the old pieces of a value whose delivery had just resumed, so that value could never complete; it now removes a group only when all of its pieces are old. A failing cleanup no longer fails the fragment being applied: it is rolled back and logged. Applying a completed value and removing its pieces now happen together, so a failure to remove them is reported and the value is applied again on the next delivery, instead of the failure being ignored. +- **PostgreSQL: pieces of one fragmented value applied at the same time no longer leave the value unapplied.** The server applies each uploaded chunk as its own job, so two transactions could each stage one piece, see only their own, and both succeed without ever applying the value. Under `READ COMMITTED` they now wait for each other, and the stale cleanup skips a value another transaction is applying. Under `SERIALIZABLE` the conflict fails one transaction with a retryable serialization error. A fragment applied under `REPEATABLE READ` is refused with SQLSTATE `0A000`. - **PostgreSQL: applying a payload read from a table no longer fails.** `SELECT cloudsync_payload_apply(data) FROM some_table` failed with `buffer pin ... is not owned by resource owner TopTransaction` (and could abort an assertion-enabled server): the internal savepoints left the caller's resource owner and memory context switched. Both are now restored when a savepoint ends. - **Block columns and grow-only-set tables merge under row-level security and NOT NULL constraints.** Both wrote a received column through an upsert holding only the primary key and that column: PostgreSQL checks an INSERT policy against that proposed row even when it becomes an update, and a NOT NULL column rejects it outright, so the column was never written. A block column's row now has its other received columns written first, and an existing row — of either kind — is updated in place. When the update finds no row to change (the row is gone, or an UPDATE policy hides it), the upsert is used as before, so the write is reported rather than recorded as applied without being stored. - **A received row whose write fails leaves no partial state behind.** Applying a row wrote part of its metadata — for a deleted-and-recreated row, the new sentinel and the reset column clocks — before the row itself, outside the savepoint that protected the write. When the write then failed, the metadata claimed the row was there: delivered again, even after the cause was fixed (a lock that went away, a policy that now allows it), the row was silently never created, and an existing row kept zeroed clocks. Each primary key's changes are now applied under one savepoint that the failed write rolls back entirely. diff --git a/src/cloudsync.c b/src/cloudsync.c index fa06c4ed..a966bfe7 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -4159,10 +4159,14 @@ static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, clouds cloudsync_stream_value *seen = track ? cloudsync_stream_value_find(data, value_id) : NULL; if (seen && seen->applied) return DBRES_OK; + // concurrent transactions applying pieces of this value wait for each other + int rc = database_fragment_lock(data, value_id); + if (rc != DBRES_OK) return rc; + // Stage the piece, then reassemble, apply and remove the value's pieces as one // unit: a failure rolls back only this call and leaves the pieces staged by // earlier calls, so the value stays retryable. - int rc = database_begin_savepoint(data, "cloudsync_fragment"); + rc = database_begin_savepoint(data, "cloudsync_fragment"); if (rc != DBRES_OK) return rc; // the fragments table is guaranteed by dbutils_settings_init; no DDL here diff --git a/src/database.h b/src/database.h index 127ea1a9..50cb621d 100644 --- a/src/database.h +++ b/src/database.h @@ -97,6 +97,7 @@ int database_begin_savepoint (cloudsync_context *data, const char *savepoint_nam int database_commit_savepoint (cloudsync_context *data, const char *savepoint_name); int database_rollback_savepoint (cloudsync_context *data, const char *savepoint_name); bool database_in_transaction (cloudsync_context *data); +int database_fragment_lock (cloudsync_context *data, const char *value_id); int database_errcode (cloudsync_context *data); const char *database_errmsg (cloudsync_context *data); void database_log_warning (cloudsync_context *data, const char *message); diff --git a/src/postgresql/database_postgresql.c b/src/postgresql/database_postgresql.c index 6582b310..0701f674 100644 --- a/src/postgresql/database_postgresql.c +++ b/src/postgresql/database_postgresql.c @@ -1189,6 +1189,29 @@ bool database_in_transaction (cloudsync_context *data) { return IsTransactionState(); } +// Pieces of one value can be applied by concurrent transactions (the server runs one +// apply job per uploaded chunk), each seeing only its own piece and leaving the value +// unapplied. Under READ COMMITTED they are serialized per value until commit: the +// statements after the wait take a new snapshot, so the last one sees every piece. +// SERIALIZABLE needs no lock: that outcome matches no serial order, so one of the +// transactions fails with a retryable serialization failure. REPEATABLE READ gets +// neither guarantee (a waiter would keep its old snapshot), so it is refused. +// The stale cleanup (SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE) uses the same lock key. +int database_fragment_lock (cloudsync_context *data, const char *value_id) { + if (IsolationIsSerializable()) return DBRES_OK; + if (IsolationUsesXactSnapshot()) { + int rc = cloudsync_set_error(data, "cloudsync_payload_apply: a fragmented value cannot be applied under REPEATABLE READ, use READ COMMITTED or SERIALIZABLE", DBRES_MISUSE); + cloudsync_set_sqlstate(data, ERRCODE_FEATURE_NOT_SUPPORTED); + return rc; + } + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, "SELECT pg_advisory_xact_lock(1129530962, hashtext($1));", &vm, 0); + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 1, value_id, -1); + if (rc == DBRES_OK) rc = databasevm_step(vm); + if (vm) databasevm_finalize(vm); + return (rc == DBRES_ROW) ? DBRES_OK : cloudsync_set_error(data, "cloudsync_payload_apply: unable to lock a fragmented value", rc); +} + bool database_table_exists (cloudsync_context *data, const char *name, const char *schema) { return database_system_exists(data, name, "table", false, schema); } diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c index 1508073a..13a76702 100644 --- a/src/postgresql/sql_postgresql.c +++ b/src/postgresql/sql_postgresql.c @@ -141,8 +141,10 @@ const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE = "DELETE FROM cloudsync_payload_fragments WHERE value_id IN (" - "SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " - "HAVING MAX(created_at) < $1 AND COUNT(*) < MAX(part_count));"; + "SELECT value_id FROM (SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " + "HAVING MAX(created_at) < $1 AND COUNT(*) < MAX(part_count)) stale " + // skip a value another transaction is applying (see database_fragment_lock) + "WHERE pg_try_advisory_xact_lock(1129530962, hashtext(value_id)));"; // MARK: Additional SQL constants for PostgreSQL diff --git a/src/sqlite/database_sqlite.c b/src/sqlite/database_sqlite.c index c0f1c3d9..63ac9f82 100644 --- a/src/sqlite/database_sqlite.c +++ b/src/sqlite/database_sqlite.c @@ -598,6 +598,11 @@ bool database_in_transaction (cloudsync_context *data) { return in_transaction; } +int database_fragment_lock (cloudsync_context *data, const char *value_id) { + // writers are already serialized + return DBRES_OK; +} + bool database_table_exists (cloudsync_context *data, const char *name, const char *schema) { UNUSED_PARAMETER(schema); return database_system_exists(data, name, "table"); diff --git a/test/postgresql/60_fragment_concurrency.sql b/test/postgresql/60_fragment_concurrency.sql new file mode 100644 index 00000000..26437a32 --- /dev/null +++ b/test/postgresql/60_fragment_concurrency.sql @@ -0,0 +1,173 @@ +-- Pieces of one fragmented value applied by concurrent transactions (the server runs +-- one apply job per uploaded chunk). Each transaction must not see only its own piece +-- and succeed, leaving a complete value unapplied; and the stale cleanup must not +-- remove the pieces of a value another transaction is reconstructing. + +\set testid '60-fragment-concurrency' +\ir helper_test_init.sql + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_60_src; +DROP DATABASE IF EXISTS cloudsync_test_60_dst; +CREATE DATABASE cloudsync_test_60_src; +CREATE DATABASE cloudsync_test_60_dst; + +-- Source: 'two' splits into two pieces, 'three' into three. +\connect cloudsync_test_60_src +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('t') AS _init \gset +-- a size of 1 clamps to the minimum chunk size +SELECT cloudsync_set('payload_max_chunk_size', '1') AS _size \gset +INSERT INTO t VALUES ('two', repeat('A', 300000)); +SELECT string_agg(encode(payload, 'hex'), ',' ORDER BY chunk_index) AS two_hex FROM cloudsync_payload_chunks() \gset +DELETE FROM t; +SELECT max(db_version) AS dbv FROM cloudsync_changes \gset +INSERT INTO t VALUES ('three', repeat('B', 600000)); +SELECT string_agg(encode(payload, 'hex'), ',' ORDER BY chunk_index) AS three_hex +FROM cloudsync_payload_chunks(:dbv) WHERE get_byte(payload, 4) = 3 \gset + +\connect cloudsync_test_60_dst +\ir helper_psql_conn_setup.sql +CREATE EXTENSION IF NOT EXISTS cloudsync; +CREATE EXTENSION IF NOT EXISTS dblink; +CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('t') AS _init \gset +CREATE TABLE transport(name TEXT, part INT, payload BYTEA, PRIMARY KEY (name, part)); +INSERT INTO transport SELECT 'two', ord, decode(h, 'hex') FROM unnest(string_to_array(:'two_hex', ',')) WITH ORDINALITY AS x(h, ord); +INSERT INTO transport SELECT 'three', ord, decode(h, 'hex') FROM unnest(string_to_array(:'three_hex', ',')) WITH ORDINALITY AS x(h, ord); +CREATE FUNCTION apply_part(n TEXT, p INT) RETURNS INT LANGUAGE plpgsql AS $$ +DECLARE data BYTEA; BEGIN + SELECT payload INTO STRICT data FROM transport WHERE name = n AND part = p; + RETURN cloudsync_payload_apply(data); +END $$; +-- Returns once session y has finished or is waiting on a lock, so the test never +-- relies on a sleep to make the two transactions overlap. +CREATE FUNCTION wait_for_y() RETURNS TEXT LANGUAGE plpgsql AS $$ +DECLARE state TEXT; BEGIN + FOR i IN 1..500 LOOP + IF dblink_is_busy('y') = 0 THEN RETURN 'finished'; END IF; + PERFORM pg_stat_clear_snapshot(); + SELECT wait_event_type INTO state FROM pg_stat_activity WHERE application_name = 'cloudsync_60_y'; + IF state = 'Lock' THEN RETURN 'waiting'; END IF; + PERFORM pg_sleep(0.01); + END LOOP; + RETURN 'timeout'; +END $$; + +SELECT (SELECT count(*) FROM transport WHERE name = 'two') = 2 + AND (SELECT count(*) FROM transport WHERE name = 'three') = 3 AS parts_ok \gset +\if :parts_ok +\echo [PASS] (:testid) values split into 2 and 3 fragments +\else +\echo [FAIL] (:testid) unexpected fragment counts +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT dblink_connect('x', format('dbname=%s user=%s application_name=cloudsync_60_x', current_database(), current_user)) AS _cx \gset +SELECT dblink_connect('y', format('dbname=%s user=%s application_name=cloudsync_60_y', current_database(), current_user)) AS _cy \gset +-- Warm both sessions as a long-running worker would be: their first cloudsync call +-- initializes the context, and an open transaction holding those writes would make +-- the other session wait for reasons unrelated to the fragments. +SELECT n AS _n FROM dblink('x', 'SELECT apply_part(''two'', 1)') AS r(n INT) \gset +SELECT n AS _n FROM dblink('y', 'SELECT apply_part(''two'', 1)') AS r(n INT) \gset +DELETE FROM cloudsync_payload_fragments; + +-- 1. READ COMMITTED: x stages piece 1 and stays open while y applies piece 2. +SELECT dblink_exec('x', 'BEGIN') AS _b \gset +SELECT n AS _n FROM dblink('x', 'SELECT apply_part(''two'', 1)') AS r(n INT) \gset +SELECT dblink_send_query('y', 'SELECT apply_part(''two'', 2)') AS _s \gset +SELECT wait_for_y() AS y_state \gset +SELECT dblink_exec('x', 'COMMIT') AS _c \gset +SELECT count(*) AS _n FROM dblink_get_result('y', false) AS r(n INT) \gset +SELECT count(*) AS _drain FROM dblink_get_result('y', false) AS r(n INT) \gset +SELECT (SELECT length(value) FROM t WHERE id = 'two') IS NOT DISTINCT FROM 300000 + AND NOT EXISTS (SELECT FROM cloudsync_payload_fragments) AS rc_ok \gset +\if :rc_ok +\echo [PASS] (:testid) READ COMMITTED: the value applies once both pieces commit (y :y_state) +\else +\echo [FAIL] (:testid) READ COMMITTED: both pieces committed but the value was not applied (y :y_state) +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 2. SERIALIZABLE: the same overlap either applies the value or fails one transaction +-- with a retryable serialization failure. Never both succeed without the value. +DELETE FROM t; DELETE FROM t_cloudsync; DELETE FROM cloudsync_payload_fragments; +SELECT dblink_exec('x', 'BEGIN ISOLATION LEVEL SERIALIZABLE') AS _b \gset +SELECT n AS _n FROM dblink('x', 'SELECT apply_part(''two'', 1)') AS r(n INT) \gset +SELECT dblink_exec('y', 'BEGIN ISOLATION LEVEL SERIALIZABLE') AS _b \gset +SELECT dblink_send_query('y', 'SELECT apply_part(''two'', 2)') AS _s \gset +SELECT wait_for_y() AS y_state \gset +SELECT dblink_exec('x', 'COMMIT', false) AS x_commit \gset +SELECT count(*) AS _n FROM dblink_get_result('y', false) AS r(n INT) \gset +SELECT count(*) AS _drain FROM dblink_get_result('y', false) AS r(n INT) \gset +SELECT dblink_error_message('y') AS y_apply_error \gset +SELECT dblink_exec('y', 'COMMIT', false) AS y_commit \gset +SELECT dblink_error_message('y') AS y_commit_error \gset +SELECT dblink_exec('y', 'ROLLBACK', false) AS _r \gset +SELECT (SELECT length(value) FROM t WHERE id = 'two') IS NOT DISTINCT FROM 300000 AS ser_applied \gset +SELECT (:'y_apply_error' LIKE '%could not serialize%' OR :'y_commit_error' LIKE '%could not serialize%') AS ser_failed \gset +SELECT (:'ser_applied' OR :'ser_failed') AS ser_ok \gset +\if :ser_ok +\echo [PASS] (:testid) SERIALIZABLE: the overlap applies the value or fails with a serialization error +\else +\echo [FAIL] (:testid) SERIALIZABLE: both transactions succeeded without applying the value (apply: :y_apply_error, commit: :y_commit_error) +SELECT (:fail::int + 1) AS fail \gset +\endif +-- a retry of the failed piece completes the value +SELECT n AS _n FROM dblink('y', 'SELECT apply_part(''two'', 2)') AS r(n INT) \gset +SELECT (SELECT length(value) FROM t WHERE id = 'two') IS NOT DISTINCT FROM 300000 + AND NOT EXISTS (SELECT FROM cloudsync_payload_fragments) AS ser_retry_ok \gset +\if :ser_retry_ok +\echo [PASS] (:testid) SERIALIZABLE: redelivering the piece completes the value +\else +\echo [FAIL] (:testid) SERIALIZABLE: the value is still missing after redelivery +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 3. REPEATABLE READ keeps the snapshot taken before the wait, so a fragment apply +-- there is refused instead of risking a value that is never applied. +DELETE FROM t; DELETE FROM t_cloudsync; DELETE FROM cloudsync_payload_fragments; +SELECT dblink_exec('y', 'BEGIN ISOLATION LEVEL REPEATABLE READ') AS _b \gset +SELECT count(*) AS _n FROM dblink('y', 'SELECT apply_part(''two'', 1)', false) AS r(n INT) \gset +SELECT dblink_error_message('y') AS rr_error \gset +SELECT dblink_exec('y', 'ROLLBACK', false) AS _r \gset +SELECT (:'rr_error' LIKE '%REPEATABLE READ%') AS rr_ok \gset +\if :rr_ok +\echo [PASS] (:testid) REPEATABLE READ: a fragment apply is refused +\else +\echo [FAIL] (:testid) REPEATABLE READ: expected a refusal, got: :rr_error +SELECT (:fail::int + 1) AS fail \gset +\endif + +-- 4. The stale cleanup must not remove the old pieces of a value that another +-- transaction is resuming: x stages piece 2 of 'three' (piece 1 is two days old) +-- and stays open while a fresh session z runs its first fragment call, which runs +-- the cleanup. After x commits, piece 3 completes the value. +DELETE FROM t; DELETE FROM t_cloudsync; DELETE FROM cloudsync_payload_fragments; +SELECT n AS _n FROM dblink('y', 'SELECT apply_part(''three'', 1)') AS r(n INT) \gset +UPDATE cloudsync_payload_fragments SET created_at = created_at - 172800; +SELECT dblink_exec('x', 'BEGIN') AS _b \gset +SELECT n AS _n FROM dblink('x', 'SELECT apply_part(''three'', 2)') AS r(n INT) \gset +SELECT dblink_connect('z', format('dbname=%s user=%s application_name=cloudsync_60_z', current_database(), current_user)) AS _cz \gset +SELECT n AS _n FROM dblink('z', 'SELECT apply_part(''two'', 1)') AS r(n INT) \gset +SELECT dblink_exec('x', 'COMMIT') AS _c \gset +SELECT n AS _n FROM dblink('y', 'SELECT apply_part(''three'', 3)') AS r(n INT) \gset +SELECT (SELECT length(value) FROM t WHERE id = 'three') IS NOT DISTINCT FROM 600000 AS cleanup_ok \gset +\if :cleanup_ok +\echo [PASS] (:testid) cleanup keeps the pieces of a value being reconstructed +\else +\echo [FAIL] (:testid) cleanup removed the old piece of a value being resumed +SELECT (:fail::int + 1) AS fail \gset +\endif + +SELECT dblink_disconnect('x') AS _dx \gset +SELECT dblink_disconnect('y') AS _dy \gset +SELECT dblink_disconnect('z') AS _dz \gset + +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_60_src; +DROP DATABASE IF EXISTS cloudsync_test_60_dst; diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index 4b9c2058..38a9273f 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -67,6 +67,7 @@ \ir 57_audit_regressions.sql \ir 58_v3_denied_checkpoint.sql \ir 59_rls_denial_retry.sql +\ir 60_fragment_concurrency.sql -- 'Test summary' \echo '\nTest summary:' From 8ebc9512f3e5583c8c9a33ee7175d574a5630d88 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 21:34:56 -0600 Subject: [PATCH 23/28] fix(receive): fail a fragmented stream whose final chunk has no watermark Without a watermark the network fallback checkpointed through LAST_APPLIED, the mode direct SQL calls use, which no longer moves the cursor for a v3 fragment. A stream ending in a fragment then applied the value but never advanced check_dbversion, and every call replayed the same window while reporting complete. The current server always sends a watermark on chunked responses, so this needs an older or non-conforming server. The fallback now has its own mode, CLOUDSYNC_CHECKPOINT_STREAM_LEGACY: a monolithic final chunk keeps the last-applied checkpoint plus the stream's completeness check, while a v3 payload fails before staging. A fragment's final chunk may apply nothing new (pieces of a value the stream already applied are skipped), which leaves no position to checkpoint and would bring the silent replay back. Direct SQL calls are unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- src/cloudsync.c | 67 +++++++++++++++++++++++++++---------------- src/cloudsync.h | 17 +++++++---- src/network/network.c | 4 +-- test/network_unit.c | 48 +++++++++++++++++++++++++++++-- 4 files changed, 102 insertions(+), 34 deletions(-) diff --git a/src/cloudsync.c b/src/cloudsync.c index a966bfe7..8597ef4d 100644 --- a/src/cloudsync.c +++ b/src/cloudsync.c @@ -4217,6 +4217,30 @@ static int cloudsync_payload_apply_fragment_row (cloudsync_context *data, clouds // #ifndef CLOUDSYNC_OMIT_RLS_VALIDATION +// Called when a receive stream's final chunk has applied. A fragmented value the stream +// delivered must be complete by now: its pieces are removed when it is applied, here or +// by another connection. Only values this stream staged are checked. Staging left by +// other streams or direct calls is ignored on purpose: a value_id identifies a value, +// not the stream that delivered it, and a replay can legitimately omit an old value +// (replaced, lost a conflict, filtered out, untracked), so requiring the whole staging +// table to be empty could stop the cursor for good. The age cleanup bounds such +// leftovers. A value another connection completes while this stream is still +// delivering it leaves this stream's later pieces staged again: the check fails once, +// and the replay from the first page applies the value again as a no-op and passes. +static int cloudsync_receive_stream_check_complete (cloudsync_context *data) { + for (int i = 0; i < data->stream_values_count; i++) { + if (data->stream_values[i].applied) continue; + dbvm_t *vm = NULL; + int rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_EXISTS, &vm, 0); + if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 1, data->stream_values[i].value_id, -1); + if (rc == DBRES_OK) rc = databasevm_step(vm); + if (vm) databasevm_finalize(vm); + if (rc == DBRES_ROW) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: the receive stream ended with an incomplete fragmented value", DBRES_ERROR); + if (rc != DBRES_DONE) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to check staged fragments", rc); + } + return DBRES_OK; +} + // Advance the durable receive cursor (check_dbversion/check_seq) after a payload // (or a fully-applied chunk stream) has been applied. See the checkpoint-mode // documentation on cloudsync_payload_apply in cloudsync.h. The advance is @@ -4226,33 +4250,20 @@ static int cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t int64_t target_seq; if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_NONE) return DBRES_OK; - if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_LAST_APPLIED) { + bool stream_end = (checkpoint_db_version >= 0 || checkpoint_db_version == CLOUDSYNC_CHECKPOINT_STREAM_LEGACY); + if (stream_end) { + int rc = cloudsync_receive_stream_check_complete(data); + if (rc != DBRES_OK) return rc; + } + if (checkpoint_db_version < 0) { // Nothing applied -> nothing to checkpoint. - if (data->apply_last_db_version < 0) return DBRES_OK; + if (data->apply_last_db_version < 0) { + if (stream_end) cloudsync_receive_stream_reset(data); + return DBRES_OK; + } target_db_version = data->apply_last_db_version; target_seq = data->apply_last_seq; } else { - // The final chunk of a receive stream. A fragmented value the stream delivered - // must be complete by now: its pieces are removed when it is applied, here or - // by another connection. Only values this stream staged are checked. Staging - // left by other streams or direct calls is ignored on purpose: a value_id - // identifies a value, not the stream that delivered it, and a replay can - // legitimately omit an old value (replaced, lost a conflict, filtered out, - // untracked), so requiring the whole staging table to be empty could stop the - // cursor for good. The age cleanup bounds such leftovers. A value another - // connection completes while this stream is still delivering it leaves this - // stream's later pieces staged again: the check fails once, and the replay - // from the first page applies the value again as a no-op and passes. - for (int i = 0; i < data->stream_values_count; i++) { - if (data->stream_values[i].applied) continue; - dbvm_t *vm = NULL; - int rc = databasevm_prepare(data, SQL_PAYLOAD_FRAGMENTS_EXISTS, &vm, 0); - if (rc == DBRES_OK) rc = databasevm_bind_text(vm, 1, data->stream_values[i].value_id, -1); - if (rc == DBRES_OK) rc = databasevm_step(vm); - if (vm) databasevm_finalize(vm); - if (rc == DBRES_ROW) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: the receive stream ended with an incomplete fragmented value", DBRES_ERROR); - if (rc != DBRES_DONE) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to check staged fragments", rc); - } target_db_version = checkpoint_db_version; target_seq = checkpoint_seq; } @@ -4273,7 +4284,7 @@ static int cloudsync_payload_apply_checkpoint (cloudsync_context *data, int64_t } if (rc != DBRES_OK) return cloudsync_set_error(data, "Error on cloudsync_payload_apply: unable to write the receive checkpoint", rc); } - if (checkpoint_db_version >= 0) cloudsync_receive_stream_reset(data); + if (stream_end) cloudsync_receive_stream_reset(data); return DBRES_OK; } @@ -4455,6 +4466,14 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b if (clone) cloudsync_memory_free(clone); return cloudsync_set_error(data, "Error on cloudsync_payload_apply: invalid v3 column count", DBRES_MISUSE); } + // Without a watermark the stream end falls back to the last applied position, + // but a fragment's final chunk may apply nothing new (its pieces belong to a + // value this stream already applied), which leaves no position to checkpoint: + // fail instead of leaving the cursor in place and replaying the same window. + if (checkpoint_db_version == CLOUDSYNC_CHECKPOINT_STREAM_LEGACY) { + if (clone) cloudsync_memory_free(clone); + return cloudsync_set_error(data, "Error on cloudsync_payload_apply: the final chunk of a fragmented stream has no watermark", DBRES_MISUSE); + } for (uint32_t i = 0; i < header.nrows; ++i) { size_t seek = 0; cloudsync_payload_fragment_row row = {0}; diff --git a/src/cloudsync.h b/src/cloudsync.h index 0d9ed6ab..fdd5a75d 100644 --- a/src/cloudsync.h +++ b/src/cloudsync.h @@ -146,13 +146,20 @@ const char *cloudsync_table_schema (cloudsync_context *data, const char *table_n // CLOUDSYNC_CHECKPOINT_LAST_APPLIED advance to this artifact's last applied // (db_version, seq). Legacy/monolithic // behavior: safe only for a complete payload -// that ends on a db_version boundary. A v3 -// fragment never moves the cursor this way. -// NONE and an explicit watermark mark the call as part of a receive stream: the +// that ends on a db_version boundary. Used by +// direct SQL calls; a v3 fragment never moves +// the cursor this way. +// CLOUDSYNC_CHECKPOINT_STREAM_LEGACY the final chunk of a stream from a server +// that sends no watermark: LAST_APPLIED plus the +// stream's completeness check. A v3 fragment +// fails: its final chunk may apply nothing new, +// leaving no position to checkpoint. +// Every mode but LAST_APPLIED marks the call as part of a receive stream: the // fragmented values it stages are tracked until applied. Reset that tracking with // cloudsync_receive_stream_reset whenever a stream starts from its first page. -#define CLOUDSYNC_CHECKPOINT_NONE (-1) -#define CLOUDSYNC_CHECKPOINT_LAST_APPLIED (-2) +#define CLOUDSYNC_CHECKPOINT_NONE (-1) +#define CLOUDSYNC_CHECKPOINT_LAST_APPLIED (-2) +#define CLOUDSYNC_CHECKPOINT_STREAM_LEGACY (-3) void cloudsync_receive_stream_reset (cloudsync_context *data); int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int blen, int *nrows, int64_t checkpoint_db_version, int64_t checkpoint_seq); int cloudsync_payload_encode_step (cloudsync_payload_context *payload, cloudsync_context *data, int argc, dbvalue_t **argv); diff --git a/src/network/network.c b/src/network/network.c index 88c3b135..7dcf4995 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -1580,13 +1580,13 @@ static int network_apply_check_chunk(sqlite3_context *context, const char *chunk // A non-final chunk must never advance the receive cursor (see cloudsync.h): // landing mid-db_version would let the next /check skip the unapplied // remainder. Only the final chunk advances -- to the explicit watermark, or - // the legacy last-applied fallback when it is absent. + // the legacy last-applied fallback when it is absent (a fragment then fails). int64_t watermark = json_extract_int(chunk_json, chunk_json_len, "watermark", -1); int64_t checkpoint_db_version; if (!final_chunk) { checkpoint_db_version = CLOUDSYNC_CHECKPOINT_NONE; } else { - checkpoint_db_version = (watermark < 0) ? CLOUDSYNC_CHECKPOINT_LAST_APPLIED : watermark; + checkpoint_db_version = (watermark < 0) ? CLOUDSYNC_CHECKPOINT_STREAM_LEGACY : watermark; } int64_t checkpoint_seq = 0; diff --git a/test/network_unit.c b/test/network_unit.c index ba3b019e..63df1cf6 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -267,7 +267,7 @@ extern char *network_test_base64_encode(const unsigned char *, size_t); #define SPOOL_MAX 16 static const char *spool[SPOOL_MAX]; static int spool_pages; -static int64_t spool_watermark; +static int64_t spool_watermark; // < 0: chunks carry no watermark (an older server) static int64_t req_cursor[64], req_since[64]; static int nreq; @@ -289,8 +289,10 @@ static NETWORK_RESULT spool_responder(const char *endpoint, const char *request) size_t len = (size_t)snprintf(json, cap, "{\"data\":{\"chunks\":["); int64_t k = cursor; for (; k < spool_pages && k < cursor + max; k++) { - len += (size_t)snprintf(json + len, cap - len, "%s{\"cursor\":%lld,\"payload\":\"%s\",\"watermark\":%lld}", - k > cursor ? "," : "", (long long)k, spool[k], (long long)spool_watermark); + char watermark[48] = ""; + if (spool_watermark >= 0) snprintf(watermark, sizeof(watermark), ",\"watermark\":%lld", (long long)spool_watermark); + len += (size_t)snprintf(json + len, cap - len, "%s{\"cursor\":%lld,\"payload\":\"%s\"%s}", + k > cursor ? "," : "", (long long)k, spool[k], watermark); } bool final = k >= spool_pages; snprintf(json + len, cap - len, "],\"final\":%s,\"nextCursor\":%lld}}", final ? "true" : "false", (long long)(final ? -1 : k)); @@ -531,6 +533,45 @@ static bool test_stream_fragments(void) { return ok; } +// A server that sends no watermark: a monolithic final chunk still advances to its last +// applied change, while a stream ending in a fragment fails instead of leaving the +// checkpoint in place and replaying the same window forever. +static bool test_stream_no_watermark(void) { + bool ok = true; + sqlite3 *src = stream_db(false), *dst = stream_db(true); + char *pages[SPOOL_MAX] = {0}; + ok = ok && src && dst; + ok = ok && db_exec(src, "INSERT INTO t VALUES('k1',x'01'); INSERT INTO t VALUES('k2',x'02');") == SQLITE_OK; + int n = ok ? stream_payloads(src, "SELECT cloudsync_payload_encode(tbl,pk,col_name,col_value,col_version,db_version,site_id,cl,seq) FROM cloudsync_changes GROUP BY pk ORDER BY pk", pages, SPOOL_MAX) : 0; + ok = ok && expect(n == 2, "two pages", NULL); + for (int i = 0; i < n; i++) spool[i] = pages[i]; + spool_pages = n; + spool_watermark = -1; + network_test_set_responder(spool_responder); + char *json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "\"complete\":true") && db_checkpoint(dst) == db_int(src, "SELECT max(db_version) FROM cloudsync_changes"), "monolithic stream without watermark", json); + + stream_close(dst); + dst = stream_db(true); + int nf = 0; + ok = ok && db_exec(src, "SELECT cloudsync_set('payload_max_chunk_size','262144'); INSERT INTO t VALUES('big', randomblob(700000));") == SQLITE_OK; + char query[256]; + snprintf(query, sizeof(query), "SELECT payload FROM cloudsync_payload_chunks(%lld) WHERE substr(payload,5,1)=x'03' ORDER BY chunk_index", (long long)db_int(src, "SELECT max(db_version) FROM cloudsync_changes") - 1); + nf = ok ? stream_payloads(src, query, pages + n, SPOOL_MAX - n) : 0; + ok = ok && expect(nf >= 2, "fragments", NULL); + for (int i = 0; i < nf; i++) spool[i] = pages[n + i]; + spool_pages = nf; + json = ok ? stream_receive(dst, 0) : ""; + ok = ok && expect(strstr(json, "has no watermark") && strstr(json, "\"complete\":false") && db_checkpoint(dst) == 0, "fragmented stream without watermark fails", json); + ok = ok && expect(db_int(dst, "SELECT count(*) FROM t WHERE id='big'") == 0, "the value is not applied", NULL); + + network_test_set_responder(NULL); + for (int i = 0; i < n + nf; i++) cloudsync_memory_free(pages[i]); + stream_close(src); + stream_close(dst); + return ok; +} + int main(void) { #if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) check("HTTP deadlines: API elapsed cap and artifact stall cap:", test_stalled_http_timeout()); @@ -546,6 +587,7 @@ int main(void) { check("network_compute_status:", test_compute_status()); check("receive stream: capped paging, failure replay, checkpoint errors:", test_stream_paging()); check("receive stream: fragmented values:", test_stream_fragments()); + check("receive stream: server without watermark:", test_stream_no_watermark()); if (failures) { printf("\n%d test(s) FAILED\n", failures); return 1; } printf("\nAll network unit tests passed\n"); return 0; From 9ded93314bf11449df86bf05335947bf5eb713b5 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 23:02:01 -0600 Subject: [PATCH 24/28] test(postgres): let expected errors pass when ON_ERROR_STOP is on CI runs the suite with psql -v ON_ERROR_STOP=on, so an SQL error that a test expects must be wrapped in \set ON_ERROR_STOP off/on, as tests 58 and 59 do. The denied applies rewritten in 27 and 29 raised unwrapped and stopped the whole run at 27_rls_batch_merge.sql:284 on PostgreSQL 15, 17 and 18. Test 39's lock attempt, which is expected to fail on Supabase and then skip the lock-contention case, is wrapped the same way: once an earlier test turns ON_ERROR_STOP back on, that failure would stop a Supabase run too. Co-Authored-By: Claude Opus 5 (1M context) --- test/postgresql/27_rls_batch_merge.sql | 3 +++ test/postgresql/29_rls_multicol.sql | 9 +++++++++ test/postgresql/39_concurrent_write_apply.sql | 2 ++ 3 files changed, 14 insertions(+) diff --git a/test/postgresql/27_rls_batch_merge.sql b/test/postgresql/27_rls_batch_merge.sql index aa692118..7943f135 100644 --- a/test/postgresql/27_rls_batch_merge.sql +++ b/test/postgresql/27_rls_batch_merge.sql @@ -281,8 +281,11 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_5 FROM cloudsync_changes \gset SELECT coalesce((SELECT value::BIGINT FROM cloudsync_settings WHERE key='check_dbversion'), 0) AS ckpt_before_denied \gset SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; +-- the denial is expected to raise +\set ON_ERROR_STOP off SELECT cloudsync_payload_apply(decode(:'payload_hex_5', 'hex')) AS apply_5 \gset \set apply_5_state :SQLSTATE +\set ON_ERROR_STOP on -- Reconnect for clean state after expected RLS denial \connect cloudsync_test_27_b diff --git a/test/postgresql/29_rls_multicol.sql b/test/postgresql/29_rls_multicol.sql index d21e3dfe..2064e0b7 100644 --- a/test/postgresql/29_rls_multicol.sql +++ b/test/postgresql/29_rls_multicol.sql @@ -228,8 +228,11 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_4 FROM cloudsync_changes \gset \ir helper_psql_conn_setup.sql SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; +-- the denial is expected to raise +\set ON_ERROR_STOP off SELECT cloudsync_payload_apply(decode(:'payload_hex_4', 'hex')) AS apply_4 \gset \set apply_4_state :SQLSTATE +\set ON_ERROR_STOP on -- Reconnect for clean state after expected RLS denial \connect cloudsync_test_29_b @@ -314,8 +317,11 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_6 FROM cloudsync_changes \gset \ir helper_psql_conn_setup.sql SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; +-- the denial is expected to raise +\set ON_ERROR_STOP off SELECT cloudsync_payload_apply(decode(:'payload_hex_6', 'hex')) AS apply_6 \gset \set apply_6_state :SQLSTATE +\set ON_ERROR_STOP on -- Reconnect for clean state after expected RLS denial \connect cloudsync_test_29_b @@ -361,8 +367,11 @@ SELECT COALESCE(max(db_version), 0) AS max_dbv_7 FROM cloudsync_changes \gset \ir helper_psql_conn_setup.sql SET app.current_user_id = :'USER1'; SET ROLE test_rls_user; +-- the denial is expected to raise +\set ON_ERROR_STOP off SELECT cloudsync_payload_apply(decode(:'payload_hex_7', 'hex')) AS apply_7 \gset \set apply_7_state :SQLSTATE +\set ON_ERROR_STOP on -- Reconnect for clean verification as superuser \connect cloudsync_test_29_b diff --git a/test/postgresql/39_concurrent_write_apply.sql b/test/postgresql/39_concurrent_write_apply.sql index 8b59cbde..ef6c5630 100644 --- a/test/postgresql/39_concurrent_write_apply.sql +++ b/test/postgresql/39_concurrent_write_apply.sql @@ -72,7 +72,9 @@ SELECT dblink_exec('locker', 'BEGIN') AS _begin \gset -- Try to acquire EXCLUSIVE lock — if this fails (e.g. permission denied on -- Supabase), _lock won't be set and we skip the lock-contention test \unset _lock +\set ON_ERROR_STOP off SELECT dblink_exec('locker', 'LOCK TABLE concurrent_tbl IN EXCLUSIVE MODE') AS _lock \gset +\set ON_ERROR_STOP on \if :{?_lock} -- ===== Lock acquired — run lock-contention test ===== From 4e2d36ec373b43fdf579ef1d2182af889fb1055c Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 23:18:05 -0600 Subject: [PATCH 25/28] fix(network): resolve names on a thread so DNS lookups honor the deadlines libcurl was built with --disable-threaded-resolver. Its synchronous resolver can only time out a lookup with SIGALRM, which CURLOPT_NOSIGNAL disables, so a hung DNS server blocked a request past every connect and request deadline. Build with --enable-threaded-resolver instead (c-ares stays disabled). The --disable-pthreads flag is dropped: curl 8.12 does not recognize it, and pthreads are detected and used by the threaded resolver on POSIX builds. A network unit test asserts that the linked libcurl resolves names asynchronously (CURL_VERSION_ASYNCHDNS); it fails against the previous build. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 2 +- Makefile | 3 +-- test/network_unit.c | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3442fb74..f3817cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. +- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request, name resolution included: libcurl is now built with its threaded resolver, since the synchronous one cannot time out a hung DNS lookup. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **A compressed payload can no longer make the extension allocate an arbitrary amount of memory.** The decompressed size is read from the payload header and allocated before decompressing, so a few forged bytes could previously claim gigabytes. A declared size larger than LZ4 can actually produce from the compressed bytes (255:1) is now rejected up front, as is one beyond LZ4's own `INT_MAX` limit, where the previous code handed LZ4 a negative capacity. There is no fixed size limit: any payload the library produces, however large, still applies. ### Changed diff --git a/Makefile b/Makefile index c2e352d6..e94242db 100644 --- a/Makefile +++ b/Makefile @@ -410,9 +410,8 @@ endif --disable-ntlm-wb \ --disable-progress-meter \ --disable-proxy \ - --disable-pthreads \ --disable-socketpair \ - --disable-threaded-resolver \ + --enable-threaded-resolver \ --disable-tls-srp \ --disable-verbose \ --disable-versioned-symbols \ diff --git a/test/network_unit.c b/test/network_unit.c index 63df1cf6..97233b76 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -572,9 +572,23 @@ static bool test_stream_no_watermark(void) { return ok; } +#ifndef CLOUDSYNC_OMIT_CURL +#include +// With the synchronous resolver and CURLOPT_NOSIGNAL, curl cannot time out a name +// lookup, so a hung DNS server would outlast every deadline. The build must link a +// libcurl that resolves names asynchronously (the threaded resolver). +static bool test_curl_async_dns(void) { + const curl_version_info_data *info = curl_version_info(CURLVERSION_NOW); + return info && (info->features & CURL_VERSION_ASYNCHDNS); +} +#endif + int main(void) { #if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) check("HTTP deadlines: API elapsed cap and artifact stall cap:", test_stalled_http_timeout()); +#endif +#ifndef CLOUDSYNC_OMIT_CURL + check("libcurl resolves names asynchronously (DNS honors deadlines):", test_curl_async_dns()); #endif check("JSON keys only match root object members:", test_json_scope()); check("Gateway data envelope is unwrapped before scoped lookups:", test_json_envelope()); From 443eae7891429e6ccaca4af0550adcd76f1eec12 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 23:21:33 -0600 Subject: [PATCH 26/28] feat(network): cancel a transfer in flight with sqlite3_interrupt() Nothing could stop a network call before its deadline. Each libcurl handle now gets an XFERINFO callback that aborts the transfer as soon as the connection that started it is interrupted (sqlite3_is_interrupted, SQLite 3.41+; on an older host library the call still ends on its deadline). The connection is recorded on network_data when it is created; a network_data without one, as in the curl timeout test helper, is never interrupted. A cancelled call fails with SQLITE_INTERRUPT instead of SQLITE_ERROR, from the two places that report network errors, so a caller can tell a deliberate stop from a failure worth retrying. The logout, init and cleanup paths already pass the database's own result code through. Adds a test that a transfer to a stalled server on an interrupted connection aborts at once with CURLE_ABORTED_BY_CALLBACK. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 2 ++ CHANGELOG.md | 2 +- src/network/network.c | 54 +++++++++++++++++++++++++++++++---- src/network/network_private.h | 2 +- test/network_unit.c | 10 ++++++- 5 files changed, 61 insertions(+), 9 deletions(-) diff --git a/API.md b/API.md index 9a66edcd..f478285e 100644 --- a/API.md +++ b/API.md @@ -677,6 +677,8 @@ Fix the cause and deliver the payload again: the changes already applied merge a ## Network Functions +Every network request has a deadline, so a stalled server cannot hold the connection indefinitely. A network call in progress can also be cancelled with `sqlite3_interrupt()` on its connection: it stops within about a second and fails with `SQLITE_INTERRUPT` (SQLite 3.41 or later), so a deliberate stop can be told apart from a failure worth retrying. + ### `cloudsync_network_init(managedDatabaseId)` **Description:** Initializes the `sqlite-sync` network component. This function configures the endpoints for the CloudSync service and initializes the cURL library. diff --git a/CHANGELOG.md b/CHANGELOG.md index f3817cb8..0121aec6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request, name resolution included: libcurl is now built with its threaded resolver, since the synchronous one cannot time out a hung DNS lookup. API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. +- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request, name resolution included: libcurl is now built with its threaded resolver, since the synchronous one cannot time out a hung DNS lookup. A network call in flight can also be cancelled with `sqlite3_interrupt()`, and then fails with `SQLITE_INTERRUPT` rather than a generic error (SQLite 3.41 or later). API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **A compressed payload can no longer make the extension allocate an arbitrary amount of memory.** The decompressed size is read from the payload header and allocated before decompressing, so a few forged bytes could previously claim gigabytes. A declared size larger than LZ4 can actually produce from the compressed bytes (255:1) is now rejected up front, as is one beyond LZ4's own `INT_MAX` limit, where the previous code handed LZ4 a negative capacity. There is no fixed size limit: any payload the library produces, however large, still applies. ### Changed diff --git a/src/network/network.c b/src/network/network.c index 7dcf4995..9877fcff 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -98,6 +98,7 @@ struct network_data { // A failed chunk also restarts from page 0 on the next call. int64_t check_cursor; // next page index to request (0 = fresh drain) int64_t check_cursor_since; // the check_dbversion check_cursor belongs to + sqlite3 *db; // interrupting it cancels a transfer in flight (NULL: never) #ifndef CLOUDSYNC_OMIT_CURL CURL *api_curl; CURL *artifact_curl; @@ -310,6 +311,18 @@ void network_data_free (network_data *data) { // MARK: - Utils - +// sqlite3_is_interrupted exists from SQLite 3.41: on an older host library an +// interrupt does not cancel a transfer, which then ends on its deadline. +static bool network_db_interrupted (sqlite3 *db) { + return db && sqlite3_libversion_number() >= 3041000 && sqlite3_is_interrupted(db); +} + +// A network call cancelled with sqlite3_interrupt() reports SQLITE_INTERRUPT, so the +// caller can tell a deliberate stop from a failure worth retrying. +static int network_error_code (sqlite3_context *context) { + return network_db_interrupted(sqlite3_context_db_handle(context)) ? SQLITE_INTERRUPT : SQLITE_ERROR; +} + static bool network_endpoint_is_api(network_data *data, const char *endpoint) { if (!data || !endpoint) return false; return (data->check_endpoint && strcmp(endpoint, data->check_endpoint) == 0) || @@ -372,13 +385,22 @@ static bool network_curl_pool_enabled(network_data *data) { return data->curl_pool_enabled > 0; } +// Called by libcurl while a transfer runs, idle included: a non-zero return aborts it, +// so sqlite3_interrupt() on the connection cancels a network call in flight. +static int network_curl_progress (void *xdata, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) { + return network_db_interrupted(((network_data *)xdata)->db) ? 1 : 0; +} + // API calls carry small JSON, so a cap on elapsed time is the right shape for them. // Artifact transfers are bulk and are bounded on progress instead: a large payload // inside a 300s cap would demand a sustained transfer rate, killing a healthy transfer // on a slow link. Low-speed also detects a genuine stall sooner than the absolute cap does. -static void network_curl_apply_deadlines(CURL *handle, bool is_api) { +static void network_curl_apply_deadlines(CURL *handle, network_data *data, bool is_api) { curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(handle, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(handle, CURLOPT_XFERINFOFUNCTION, network_curl_progress); + curl_easy_setopt(handle, CURLOPT_XFERINFODATA, data); if (is_api) { curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); return; @@ -394,7 +416,7 @@ static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, if (!network_curl_pool_enabled(data)) { CURL *handle = curl_easy_init(); if (!handle) return NULL; - network_curl_apply_deadlines(handle, is_api); + network_curl_apply_deadlines(handle, data, is_api); return handle; } @@ -405,7 +427,7 @@ static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, curl_easy_reset(*slot); } if (!*slot) return NULL; - network_curl_apply_deadlines(*slot, is_api); + network_curl_apply_deadlines(*slot, data, is_api); curl_easy_setopt(*slot, CURLOPT_MAXCONNECTS, CLOUDSYNC_CURL_MAXCONNECTS); curl_easy_setopt(*slot, CURLOPT_MAXAGE_CONN, CLOUDSYNC_CURL_MAXAGE_CONN_SECONDS); @@ -442,6 +464,23 @@ bool network_test_curl_timeout(const char *url, bool use_pool, bool as_api) { if (data.artifact_curl) curl_easy_cleanup(data.artifact_curl); return ok; } + +// A transfer against a server that never answers, on a connection already interrupted: +// the progress callback must abort it at once instead of waiting for a deadline. +bool network_test_curl_interrupt(const char *url, sqlite3 *db) { + network_data data = {0}; + data.curl_pool_enabled = -1; + data.db = db; + CURL *handle = network_curl_for_endpoint(&data, url, NULL); + if (!handle) return false; + curl_easy_setopt(handle, CURLOPT_URL, url); + curl_easy_setopt(handle, CURLOPT_PROXY, ""); + CURLcode rc = curl_easy_perform(handle); + double seconds = 0; + curl_easy_getinfo(handle, CURLINFO_TOTAL_TIME, &seconds); + curl_easy_cleanup(handle); + return rc == CURLE_ABORTED_BY_CALLBACK && seconds < CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME; +} #endif static bool network_buffer_check (network_buffer *data, size_t needed) { @@ -784,7 +823,7 @@ int network_set_sqlite_result (sqlite3_context *context, NETWORK_RESULT *result) case CLOUDSYNC_NETWORK_ERROR: sqlite3_result_error(context, (result->buffer) ? result->buffer : "Memory error.", -1); - sqlite3_result_error_code(context, SQLITE_ERROR); + sqlite3_result_error_code(context, network_error_code(context)); rc = -1; break; @@ -1248,7 +1287,7 @@ static bool network_compute_endpoints_with_address (sqlite3_context *context, ne void network_result_to_sqlite_error (sqlite3_context *context, NETWORK_RESULT res, const char *default_error_message) { sqlite3_result_error(context, ((res.code == CLOUDSYNC_NETWORK_ERROR) && (res.buffer)) ? res.buffer : default_error_message, -1); - sqlite3_result_error_code(context, SQLITE_ERROR); + sqlite3_result_error_code(context, network_error_code(context)); } // MARK: - Init / Cleanup - @@ -1259,7 +1298,10 @@ network_data *cloudsync_network_data (sqlite3_context *context) { if (netdata) return netdata; netdata = (network_data *)cloudsync_memory_zeroalloc(sizeof(network_data)); - if (netdata) cloudsync_set_auxdata(data, netdata); + if (netdata) { + netdata->db = sqlite3_context_db_handle(context); + cloudsync_set_auxdata(data, netdata); + } return netdata; } diff --git a/src/network/network_private.h b/src/network/network_private.h index bd4fc92e..6e2363fb 100644 --- a/src/network/network_private.h +++ b/src/network/network_private.h @@ -21,7 +21,7 @@ // Artifact transfers are bulk, so they are bounded by lack of progress rather than // by elapsed time: a large payload on a slow link would otherwise be killed // mid-flight. The absolute value is only a backstop against a transfer that -// trickles just fast enough to stay alive, since nothing can cancel one in flight. +// trickles just fast enough to stay alive; sqlite3_interrupt() also cancels one. #ifndef CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS #define CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS 3600L #endif diff --git a/test/network_unit.c b/test/network_unit.c index 97233b76..3007ae64 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -230,7 +230,9 @@ static bool test_unicode(void) { #include #include #include +#include "sqlite3.h" extern bool network_test_curl_timeout(const char *, bool, bool); +extern bool network_test_curl_interrupt(const char *, sqlite3 *); static bool test_stalled_http_timeout(void) { int fd = socket(AF_INET, SOCK_STREAM, 0); if (fd < 0) return false; @@ -247,6 +249,12 @@ static bool test_stalled_http_timeout(void) { // Both shapes must abort against a server that accepts and then sends nothing. if (ok) ok = network_test_curl_timeout(url, false, true) && network_test_curl_timeout(url, true, true); if (ok) ok = network_test_curl_timeout(url, false, false) && network_test_curl_timeout(url, true, false); + // sqlite3_interrupt() cancels a transfer in flight: with no statement running the + // flag stays set, so the transfer sees it from its first progress callback. + sqlite3 *db = NULL; + if (ok) ok = sqlite3_open(":memory:", &db) == SQLITE_OK; + if (ok) { sqlite3_interrupt(db); ok = network_test_curl_interrupt(url, db); } + if (db) sqlite3_close(db); close(fd); return ok; } @@ -585,7 +593,7 @@ static bool test_curl_async_dns(void) { int main(void) { #if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) - check("HTTP deadlines: API elapsed cap and artifact stall cap:", test_stalled_http_timeout()); + check("HTTP deadlines and interrupt: API cap, artifact stall, cancel:", test_stalled_http_timeout()); #endif #ifndef CLOUDSYNC_OMIT_CURL check("libcurl resolves names asynchronously (DNS honors deadlines):", test_curl_async_dns()); From 1820a152247c672f3e4842f2c179889a97454895 Mon Sep 17 00:00:00 2001 From: Andrea Donetti Date: Fri, 18 Sep 2026 23:25:41 -0600 Subject: [PATCH 27/28] feat(network): tune the deadlines at runtime with cloudsync_set The deadlines were compile-time macros only, so a too-tight value on a slow link needed a rebuild. They can now be set per database through the existing settings mechanism, which works on every host (mobile included, where process environment variables are impractical): network_connect_timeout, network_request_timeout, network_artifact_timeout (seconds), network_artifact_low_speed_limit (bytes/s), network_artifact_low_speed_time (seconds) They are read from cloudsync_settings for every request, so a change applies to the next one. A missing or non-positive value keeps the compiled default; a value may raise or lower it, clamped to 24 h (1 GiB/s for the speed limit), and the connect timeout never exceeds the request's total. No SQL function or signature changes: these are cloudsync_set keys, like payload_max_chunk_size. Tests against a stalled local server: a raised request deadline takes effect, a non-positive value keeps the default, and sqlite3_interrupt() from another thread cancels a call with SQLITE_INTERRUPT well before its deadline. Co-Authored-By: Claude Opus 5 (1M context) --- API.md | 17 ++++++-- CHANGELOG.md | 2 +- src/network/network.c | 30 ++++++++++---- src/network/network_private.h | 10 +++++ test/network_unit.c | 74 +++++++++++++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 12 deletions(-) diff --git a/API.md b/API.md index f478285e..48d789c2 100644 --- a/API.md +++ b/API.md @@ -53,18 +53,25 @@ This document provides a reference for the SQL functions provided by the `sqlite **Description:** Stores a global CloudSync setting in the current database. Settings persist across database reopens and are loaded automatically by the extension. -The following payload setting is supported: +The following settings are supported: | Key | Description | Default | Minimum | Maximum | |---|---|---:|---:|---:| | `payload_max_chunk_size` | Maximum transport payload size generated by [`cloudsync_payload_chunks()`](#cloudsync_payload_chunkssince_db_version-filter_site_id-until_db_version-exclude_filter_site_id). Values outside the range are clamped. | `5242880` (5 MB) | `262144` (256 KB) | `33554432` (32 MB) | +| `network_connect_timeout` | Seconds allowed to connect, name resolution included. Never longer than the request's own deadline. | `30` | `1` | `86400` | +| `network_request_timeout` | Seconds allowed for a whole API request (check, upload URL, apply, status). | `300` | `1` | `86400` | +| `network_artifact_timeout` | Absolute limit, in seconds, for a payload upload or download. | `3600` | `1` | `86400` | +| `network_artifact_low_speed_limit` | A payload transfer slower than this many bytes per second for `network_artifact_low_speed_time` is aborted. | `1024` | `1` | `1073741824` | +| `network_artifact_low_speed_time` | Seconds a payload transfer may stay below `network_artifact_low_speed_limit`. | `60` | `1` | `86400` | + +The `network_*` settings apply from the next network request. A missing, zero or negative value uses the default, and larger values are clamped to the maximum. Settings are stored in the database, so a value tuned for a slow network stays in place until it is changed or removed with `cloudsync_set(key, NULL)`. `payload_max_chunk_size` affects only chunk generation. [`cloudsync_payload_apply()`](#cloudsync_payload_applypayload) continues to accept legacy payloads, monolithic payloads, and v3 chunk-fragment payloads even when they are larger than the local setting. This preserves compatibility between peers using different settings. **Parameters:** - `key` (TEXT): The setting key. -- `value` (TEXT): The setting value. For `payload_max_chunk_size`, pass the value in bytes. +- `value` (TEXT): The setting value, in the unit given in the table above. `NULL` removes the setting. **Returns:** SQLite returns no value. PostgreSQL returns `true` on success. @@ -76,6 +83,10 @@ SELECT cloudsync_set('payload_max_chunk_size', '1048576'); -- Restore the default 5 MB transport chunks SELECT cloudsync_set('payload_max_chunk_size', '5242880'); + +-- Allow API requests up to 2 minutes instead of 5, then restore the default +SELECT cloudsync_set('network_request_timeout', '120'); +SELECT cloudsync_set('network_request_timeout', NULL); ``` --- @@ -677,7 +688,7 @@ Fix the cause and deliver the payload again: the changes already applied merge a ## Network Functions -Every network request has a deadline, so a stalled server cannot hold the connection indefinitely. A network call in progress can also be cancelled with `sqlite3_interrupt()` on its connection: it stops within about a second and fails with `SQLITE_INTERRUPT` (SQLite 3.41 or later), so a deliberate stop can be told apart from a failure worth retrying. +Every network request has a deadline, so a stalled server cannot hold the connection indefinitely. The deadlines can be tuned with the `network_*` settings of [`cloudsync_set()`](#cloudsync_setkey-value). A network call in progress can also be cancelled with `sqlite3_interrupt()` on its connection: it stops within about a second and fails with `SQLITE_INTERRUPT` (SQLite 3.41 or later), so a deliberate stop can be told apart from a failure worth retrying. ### `cloudsync_network_init(managedDatabaseId)` diff --git a/CHANGELOG.md b/CHANGELOG.md index 0121aec6..199d9043 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added -- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request, name resolution included: libcurl is now built with its threaded resolver, since the synchronous one cannot time out a hung DNS lookup. A network call in flight can also be cancelled with `sqlite3_interrupt()`, and then fails with `SQLITE_INTERRUPT` rather than a generic error (SQLite 3.41 or later). API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Override at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. +- **Network requests now have deadlines**, where previously a stalled server left a sync call waiting indefinitely. 30 seconds to connect, on every request, name resolution included: libcurl is now built with its threaded resolver, since the synchronous one cannot time out a hung DNS lookup. A network call in flight can also be cancelled with `sqlite3_interrupt()`, and then fails with `SQLITE_INTERRUPT` rather than a generic error (SQLite 3.41 or later). API calls are then capped at 300 seconds of elapsed time. Artifact transfers are bounded on progress instead — they abort after 60 seconds below 1 KB/s — because a large payload on a slow link would otherwise be killed mid-flight by an elapsed-time cap, and because a genuine stall is detected sooner this way. A 1-hour absolute backstop still bounds an artifact transfer that trickles just fast enough to stay alive. Tune them at runtime with `cloudsync_set` (`network_connect_timeout`, `network_request_timeout`, `network_artifact_timeout`, `network_artifact_low_speed_limit`, `network_artifact_low_speed_time`), raising or lowering the defaults, or change the defaults at build time with `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS`, `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT`, `CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME` and `CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS`. - **A compressed payload can no longer make the extension allocate an arbitrary amount of memory.** The decompressed size is read from the payload header and allocated before decompressing, so a few forged bytes could previously claim gigabytes. A declared size larger than LZ4 can actually produce from the compressed bytes (255:1) is now rejected up front, as is one beyond LZ4's own `INT_MAX` limit, where the previous code handed LZ4 a negative capacity. There is no fixed size limit: any payload the library produces, however large, still applies. ### Changed diff --git a/src/network/network.c b/src/network/network.c index 9877fcff..cb920f92 100644 --- a/src/network/network.c +++ b/src/network/network.c @@ -99,6 +99,7 @@ struct network_data { int64_t check_cursor; // next page index to request (0 = fresh drain) int64_t check_cursor_since; // the check_dbversion check_cursor belongs to sqlite3 *db; // interrupting it cancels a transfer in flight (NULL: never) + cloudsync_context *cloudsync; // runtime deadline settings (NULL: compiled defaults) #ifndef CLOUDSYNC_OMIT_CURL CURL *api_curl; CURL *artifact_curl; @@ -391,23 +392,35 @@ static int network_curl_progress (void *xdata, curl_off_t dltotal, curl_off_t dl return network_db_interrupted(((network_data *)xdata)->db) ? 1 : 0; } +// A deadline from cloudsync_settings, read for every request so cloudsync_set applies to +// the next one. A missing or non-positive value keeps the compiled default. +static long network_deadline_setting (network_data *data, const char *key, long fallback, long ceiling) { + int64_t value = (data && data->cloudsync) ? dbutils_settings_get_int64_value(data->cloudsync, key) : 0; + if (value <= 0) return fallback; + return (value > ceiling) ? ceiling : (long)value; +} + // API calls carry small JSON, so a cap on elapsed time is the right shape for them. // Artifact transfers are bulk and are bounded on progress instead: a large payload // inside a 300s cap would demand a sustained transfer rate, killing a healthy transfer // on a slow link. Low-speed also detects a genuine stall sooner than the absolute cap does. static void network_curl_apply_deadlines(CURL *handle, network_data *data, bool is_api) { - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS); + long total = is_api + ? network_deadline_setting(data, CLOUDSYNC_KEY_NETWORK_REQUEST_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS, CLOUDSYNC_NETWORK_MAX_SECONDS) + : network_deadline_setting(data, CLOUDSYNC_KEY_NETWORK_ARTIFACT_TIMEOUT, CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS, CLOUDSYNC_NETWORK_MAX_SECONDS); + // connecting is part of the request, so it cannot take longer than the whole of it + long connect = network_deadline_setting(data, CLOUDSYNC_KEY_NETWORK_CONNECT_TIMEOUT, CLOUDSYNC_CONNECT_TIMEOUT_SECONDS, CLOUDSYNC_NETWORK_MAX_SECONDS); + if (connect > total) connect = total; + + curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, connect); + curl_easy_setopt(handle, CURLOPT_TIMEOUT, total); curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(handle, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(handle, CURLOPT_XFERINFOFUNCTION, network_curl_progress); curl_easy_setopt(handle, CURLOPT_XFERINFODATA, data); - if (is_api) { - curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_REQUEST_TIMEOUT_SECONDS); - return; - } - curl_easy_setopt(handle, CURLOPT_TIMEOUT, CLOUDSYNC_ARTIFACT_TIMEOUT_SECONDS); - curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT); - curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME); + if (is_api) return; + curl_easy_setopt(handle, CURLOPT_LOW_SPEED_LIMIT, network_deadline_setting(data, CLOUDSYNC_KEY_NETWORK_ARTIFACT_LOW_SPEED_LIMIT, CLOUDSYNC_ARTIFACT_LOW_SPEED_LIMIT, CLOUDSYNC_NETWORK_MAX_LOW_SPEED_LIMIT)); + curl_easy_setopt(handle, CURLOPT_LOW_SPEED_TIME, network_deadline_setting(data, CLOUDSYNC_KEY_NETWORK_ARTIFACT_LOW_SPEED_TIME, CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME, CLOUDSYNC_NETWORK_MAX_SECONDS)); } static CURL *network_curl_for_endpoint(network_data *data, const char *endpoint, bool *pooled) { @@ -1300,6 +1313,7 @@ network_data *cloudsync_network_data (sqlite3_context *context) { netdata = (network_data *)cloudsync_memory_zeroalloc(sizeof(network_data)); if (netdata) { netdata->db = sqlite3_context_db_handle(context); + netdata->cloudsync = data; cloudsync_set_auxdata(data, netdata); } return netdata; diff --git a/src/network/network_private.h b/src/network/network_private.h index 6e2363fb..25221e56 100644 --- a/src/network/network_private.h +++ b/src/network/network_private.h @@ -31,6 +31,16 @@ #ifndef CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME #define CLOUDSYNC_ARTIFACT_LOW_SPEED_TIME 60L #endif +// cloudsync_set keys overriding the deadlines above at runtime (seconds, and bytes per +// second for the low-speed limit). A missing or non-positive value keeps the default; +// larger values are clamped to the ceilings below. +#define CLOUDSYNC_KEY_NETWORK_CONNECT_TIMEOUT "network_connect_timeout" +#define CLOUDSYNC_KEY_NETWORK_REQUEST_TIMEOUT "network_request_timeout" +#define CLOUDSYNC_KEY_NETWORK_ARTIFACT_TIMEOUT "network_artifact_timeout" +#define CLOUDSYNC_KEY_NETWORK_ARTIFACT_LOW_SPEED_LIMIT "network_artifact_low_speed_limit" +#define CLOUDSYNC_KEY_NETWORK_ARTIFACT_LOW_SPEED_TIME "network_artifact_low_speed_time" +#define CLOUDSYNC_NETWORK_MAX_SECONDS 86400L +#define CLOUDSYNC_NETWORK_MAX_LOW_SPEED_LIMIT 1073741824L #define CLOUDSYNC_ENDPOINT_PREFIX "v2/cloudsync/databases" #define CLOUDSYNC_ENDPOINT_UPLOAD "upload" #define CLOUDSYNC_ENDPOINT_CHECK "check" diff --git a/test/network_unit.c b/test/network_unit.c index 3007ae64..71d1001d 100644 --- a/test/network_unit.c +++ b/test/network_unit.c @@ -580,6 +580,77 @@ static bool test_stream_no_watermark(void) { return ok; } +#if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) +#include +#include +static double monotonic_seconds(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9; +} +static void *interrupt_after_300ms(void *db) { + struct timespec delay = {0, 300000000}; + nanosleep(&delay, NULL); + sqlite3_interrupt((sqlite3 *)db); + return NULL; +} +// Runs cloudsync_network_receive_changes() and returns its result code and duration. +static int timed_receive(sqlite3 *db, double *seconds) { + sqlite3_stmt *vm = NULL; + double start = monotonic_seconds(); + int rc = sqlite3_prepare_v2(db, "SELECT cloudsync_network_receive_changes()", -1, &vm, NULL); + if (rc == SQLITE_OK) rc = sqlite3_step(vm); + if (rc != SQLITE_ROW) rc = sqlite3_errcode(db); + sqlite3_finalize(vm); + *seconds = monotonic_seconds() - start; + return rc; +} +// The deadlines can be tuned at runtime with cloudsync_set, in both directions, and a +// call in flight is cancelled by sqlite3_interrupt() with SQLITE_INTERRUPT. The server is +// a listening socket that never answers; this build compiles a 1 s request deadline. +static bool test_runtime_deadlines(void) { + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return false; + struct sockaddr_in address = {0}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + bool ok = bind(fd, (struct sockaddr *)&address, sizeof(address)) == 0 && listen(fd, 8) == 0; + socklen_t len = sizeof(address); + ok = ok && getsockname(fd, (struct sockaddr *)&address, &len) == 0; + char init[160]; + snprintf(init, sizeof(init), "SELECT cloudsync_network_init_custom('http://127.0.0.1:%u', 'test-managed-database-id');", ntohs(address.sin_port)); + sqlite3 *db = ok ? stream_db(false) : NULL; + ok = ok && db && db_exec(db, init) == SQLITE_OK; + double seconds = 0; + char detail[96]; + + // raised above the compiled default: the call waits for the longer deadline + ok = ok && db_exec(db, "SELECT cloudsync_set('network_request_timeout', '3');") == SQLITE_OK; + int rc = ok ? timed_receive(db, &seconds) : SQLITE_OK; + snprintf(detail, sizeof(detail), "rc=%d after %.2fs", rc, seconds); + ok = ok && expect(rc != SQLITE_ROW && rc != SQLITE_INTERRUPT && seconds > 2.5 && seconds < 6, "a raised request deadline takes effect", detail); + + // a non-positive value keeps the compiled default + ok = ok && db_exec(db, "SELECT cloudsync_set('network_request_timeout', '0');") == SQLITE_OK; + rc = ok ? timed_receive(db, &seconds) : SQLITE_OK; + snprintf(detail, sizeof(detail), "rc=%d after %.2fs", rc, seconds); + ok = ok && expect(rc != SQLITE_ROW && seconds < 2.5, "a non-positive setting keeps the default", detail); + + // cancelled from another thread well before a 30 s deadline + ok = ok && db_exec(db, "SELECT cloudsync_set('network_request_timeout', '30');") == SQLITE_OK; + pthread_t thread; + bool started = ok && pthread_create(&thread, NULL, interrupt_after_300ms, db) == 0; + rc = started ? timed_receive(db, &seconds) : SQLITE_OK; + if (started) pthread_join(thread, NULL); + snprintf(detail, sizeof(detail), "rc=%d after %.2fs", rc, seconds); + ok = ok && expect(started && (rc & 0xFF) == SQLITE_INTERRUPT && seconds < 5, "sqlite3_interrupt cancels the call with SQLITE_INTERRUPT", detail); + + stream_close(db); + close(fd); + return ok; +} +#endif + #ifndef CLOUDSYNC_OMIT_CURL #include // With the synchronous resolver and CURLOPT_NOSIGNAL, curl cannot time out a name @@ -610,6 +681,9 @@ int main(void) { check("receive stream: capped paging, failure replay, checkpoint errors:", test_stream_paging()); check("receive stream: fragmented values:", test_stream_fragments()); check("receive stream: server without watermark:", test_stream_no_watermark()); +#if !defined(_WIN32) && !defined(CLOUDSYNC_OMIT_CURL) + check("runtime deadlines (cloudsync_set) and interrupt of a call:", test_runtime_deadlines()); +#endif if (failures) { printf("\n%d test(s) FAILED\n", failures); return 1; } printf("\nAll network unit tests passed\n"); return 0; From 9d0abb306e44d9a92dbe2a12c735b93d1bf0ee41 Mon Sep 17 00:00:00 2001 From: Marco Bambini Date: Sat, 19 Sep 2026 10:57:19 +0200 Subject: [PATCH 28/28] fix(postgres): bound stale fragment cleanup lock usage --- CHANGELOG.md | 2 + docs/internal/audit-regressions.md | 111 ++++++++++-------- docs/internal/cloud-e2e.md | 94 +++++++++++++++ src/postgresql/sql_postgresql.c | 12 +- .../61_fragment_cleanup_backlog.sql | 84 +++++++++++++ test/postgresql/full_test.sql | 1 + 6 files changed, 255 insertions(+), 49 deletions(-) create mode 100644 docs/internal/cloud-e2e.md create mode 100644 test/postgresql/61_fragment_cleanup_backlog.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 199d9043..0197c1de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed +- **PostgreSQL stale-fragment cleanup makes bounded progress through large backlogs.** Each pass considers at most 64 groups before acquiring advisory locks, so a large backlog no longer exhausts the shared lock table and rolls back all cleanup. Values being applied concurrently remain protected. + - **A received change whose write fails is no longer dropped silently.** Previously the error was discarded and the change was lost without a trace. The apply now fails with that error (see Changed), and the network functions report it as `receive.error`, together with the rows applied before it. - **A received update to a row that is not there is no longer recorded as applied.** Several columns of an existing row are written with one UPDATE; when it changed nothing (the row was deleted while sync was disabled, or a policy's USING clause hides it), the changes were recorded as applied without being stored. The row is now written with the upsert instead, which inserts it or reports the policy. - **A receive stream no longer checkpoints past a fragmented value it delivered incompletely.** When the stream's final chunk arrives while a value it delivered is still missing pieces, the call now fails with an error and the checkpoint stays in place. Pieces staged by other streams or direct calls are ignored, so they never block a receive. After any failed receive, the next call downloads the stream again from its first page, which re-checks every value against the whole stream; the checkpoint still moves only when the stream completes. diff --git a/docs/internal/audit-regressions.md b/docs/internal/audit-regressions.md index f5e8b71d..1135398e 100644 --- a/docs/internal/audit-regressions.md +++ b/docs/internal/audit-regressions.md @@ -19,20 +19,30 @@ The previously reported `MAX_PARAMS` issue is outside this change. `cloudsync_payload_encode` / `cloudsync_payload_save` remain loadable; a forged header can no longer trigger a large allocation. A genuine repetitive 8 MiB value compresses to 254.3:1, inside the bound. -- Curl requests now have a 30-second connection deadline and a 300-second total - deadline, including reused handles. Build overrides are - `CLOUDSYNC_CONNECT_TIMEOUT_SECONDS` and `CLOUDSYNC_REQUEST_TIMEOUT_SECONDS`. -- Payload writes that fail on their data (constraint, raising trigger, type - error) are skipped, logged as a warning and counted (`receive.failed`), and the - receive cursor advances: they fail identically on every retry. Transient - failures (busy/locked, deadlock, serialization failure, cancel, out of memory or - disk, I/O) and, on PostgreSQL, failures not contained by a savepoint fail the - apply and leave the cursor in place. PostgreSQL's RLS WITH CHECK rejection stays - a separate outcome: every row is applied in its own savepoint so a denial raised - inside the cloudsync_changes trigger (block columns, GOS tables) is contained too; - denied rows are retried after the rest of the payload while retries make progress, - then skipped with one summary WARNING. `receive.denied` was removed: denials never - occur on the SQLite client where the network functions run. +- Curl API requests have a 30-second connection deadline and a 300-second total + deadline, including reused handles and DNS resolution (threaded resolver). Artifact + transfers instead have a 1-hour backstop and abort after 60 seconds below 1 KiB/s. + `cloudsync_set` can tune the deadlines at runtime; `sqlite3_interrupt` cancels a + transfer on SQLite 3.41 or newer. See the network settings in API.md. +- An apply stops at the first failed write, including constraint/trigger/type errors, + RLS denials and transient failures. Later changes are not processed and the receive + checkpoint stays put. PostgreSQL preserves the original SQLSTATE and rolls back the + failing SQL statement. SQLite can retain earlier successful groups, which re-merge + on redelivery after the cause is fixed. There is no internal RLS retry loop and no + `receive.failed` or `receive.denied` counter. Network results report `receive.error` + and the applied prefix in `receive.rows`. +- Fragment calls stage, reconstruct, apply and remove their pieces under a savepoint. + Receive streams check their incomplete values before advancing a final watermark; + direct v3 applies do not move the receive checkpoint. Failed streams restart at page + zero. A fragmented final chunk without a watermark fails explicitly. +- PostgreSQL serializes concurrent fragments of one value under READ COMMITTED; + SERIALIZABLE may require a retry, and REPEATABLE READ is refused. Cleanup preserves + groups another transaction is applying, and only removes groups whose newest piece + is older than 24 hours. Each cleanup materializes at most 64 candidate groups before + acquiring advisory locks, preventing a large backlog from exhausting the lock table + in one maintenance pass. Subsequent calls drain further batches; maintenance remains + throttled to once a minute per connection. Long caller transactions can still retain + locks from multiple applies: commit upload jobs individually. - Block materialization writes a row's pending columns first, and block and GOS column writes update an existing row in place, so they pass INSERT policies and NOT NULL constraints on other columns. An update that changes no row falls back to the @@ -41,12 +51,10 @@ The previously reported `MAX_PARAMS` issue is outside this change. - PostgreSQL savepoints restore the caller's resource owner and memory context, so a payload applied from a table scan no longer trips a foreign buffer pin. - Block-column migration skips tracked rows whose base row cannot be read. -- Each PK group of a payload is applied under one savepoint (`cloudsync_merge_group`) - that also covers the metadata its rows write before the flush (sentinel, zeroed - clocks, block values, winner clocks); a failed flush rolls it all back, so a retried or - re-delivered row applies cleanly. The flush uses that savepoint instead of its own. - Skipped entries are counted from `merge_pending_batch.rows` (payload rows that joined - the batch, including an explicit sentinel, excluding one implied by a column row). +- Pending ordinary columns and their metadata share a PK-group savepoint, so a + rejected flush rolls back its sentinel, reset clocks and winner clocks. Before a + block is materialized, pending ordinary columns are flushed. This is not an atomic + whole-payload guarantee; on SQLite that earlier flush can survive a later block error. - Only the outermost savepoint opened by cloudsync swaps the active snapshot when it ends; nested ones advance the command counter, keeping the caller's snapshot stack balanced when an enclosing subtransaction rolls back. @@ -69,16 +77,17 @@ The previously reported `MAX_PARAMS` issue is outside this change. | 64-bit clocks | Incoming column/database versions, causal length and sequence above UINT32_MAX | | Double encoding | Golden bytes, decoding a deployed fixture, negative-value roundtrip; the unit and regression suites on a real big-endian host (`make unittest-s390x`, s390x under QEMU) | | Virtual-table planner | Unusable/unsupported constraints before an accepted constraint; no accepted constraints | -| RLS denials | Block-column and GOS denials skipped; every column of a permitted block/GOS row written; a column hidden from UPDATE not recorded as applied; order-dependent denials applied on retry in both the batched and trigger paths; permanently denied rows skipped with checkpoint advanced | +| RLS denials | Block/GOS denials fail with SQLSTATE 42501 and unchanged checkpoint; permitted rows written in full; hidden UPDATE rows not recorded as applied; redelivery after policy/dependency repair; permanent denials keep failing | | Block materialization errors | Write failure via cloudsync_text_materialize keeps cause, code/SQLSTATE and names stage, column, table; single-shot allocation failure at every allocation never yields a blank error | | Error origin | SQLSTATE 40001/23505 through the block and metadata triggers; 40001 and a non-policy 42501 through payload apply, not skipped; SQLite trigger failure keeps SQLITE_CONSTRAINT and names column and table | -| Group atomicity | Resurrected row rejected (data and transient failure): no metadata left, re-delivery creates it, 3 entries counted; existing row keeps its clocks; RLS retry of a resurrected row; apply inside an aborted caller subtransaction | -| Payload failures | First, middle and final PK errors skipped with a warning and checkpoint advanced; locked database fails and keeps the checkpoint (SQLite rollback journal, WAL, PostgreSQL lock_timeout); expanded-size bound (forged 4GB/268MB headers, exact 255:1 boundary, genuine 254:1 payload) | +| Group atomicity | Resurrected row rejected (data and transient failure): no metadata left, re-delivery creates it; existing row keeps its clocks; RLS retry of a resurrected row; apply inside an aborted caller subtransaction | +| Payload failures | First, middle and final PK errors stop the apply with the checkpoint unchanged; locked database fails and keeps the checkpoint (SQLite rollback journal, WAL, PostgreSQL lock_timeout); expanded-size bound (forged 4GB/268MB headers, exact 255:1 boundary, genuine 254:1 payload) | +| Fragment cleanup | 30,000 stale groups; bounded lock count and forward progress across calls; locked and fresh groups preserved; real fragmented value still completes | | Metadata refill | Trigger rejects insertion of a missing column clock | | Block LWW | Insert/update rollback on block write failure; migration past orphaned metadata; allocation failure at each split/list/diff allocation | | PostgreSQL ownership | Block failure while another SPI cursor is active; no invalid tuple-table cleanup | | JSON | Root-only member lookup, string values resembling keys, nested keys, Unicode and invalid surrogates | -| Curl | Stalled loopback HTTP server, unpooled handle and pooled handle before/after reset | +| Curl | Stalled loopback HTTP server, unpooled and reset pooled handles, runtime deadline changes, interruption, paging and fragment replay | | Node | ia32 rejection on all OS families and preservation of x64/arm64-musl selection | | Fractional indexing | 4096-byte common prefix plus the existing module suite | | Test harness | Temporary directory cleanup, memory accounting and sanitizer-safe RowID generation | @@ -90,34 +99,42 @@ checks from `packages/node` with `npm test -- --run`, `npm run typecheck`, and `npm run build`. PostgreSQL's `test/postgresql/full_test.sql` includes the focused audit cases in -`57_audit_regressions.sql`. Run it only against a disposable PostgreSQL instance: -the existing suite creates and drops its test databases. +`57_audit_regressions.sql` through `61_fragment_cleanup_backlog.sql`. Run it only +against a disposable PostgreSQL instance: the suite creates and drops its databases. The fractional-indexing changes and its new test are inside a Git submodule; they must be recorded there before updating the parent repository's submodule pointer when preparing a commit. -## Validation performed (2026-09-11) +## Validation -- macOS arm64: all 150 existing unit checks and the new audit regressions passed. -- The same SQLite suites passed AddressSanitizer and UndefinedBehaviorSanitizer - with `UBSAN_OPTIONS=halt_on_error=1`, including zero outstanding SQLite memory. -- The forced big-endian conversion regression build passed. -- All 7 network tests and all 340 fractional-indexing module tests passed. -- PostgreSQL 17, rebuilt in an isolated Linux container: 479 reported checks - passed across 57 test groups, with zero failures and no SPI cleanup warnings. -- Node: 14 tests, TypeScript checking, and CJS/ESM/declaration builds passed. -- PostgreSQL migration compatibility check and Git whitespace checks passed. +Independent review of commit `1820a15` on 2026-09-19: -No live cloud-service integration was run. Windows, Android and WebAssembly -runtime suites were not executed. The x86_64 runtime attempt was unavailable on -this host (`Bad CPU type in executable`, Rosetta not available). The endian -test exercises conversion logic; it is not a substitute for real big-endian -hardware testing. +- PostgreSQL 15.19, 17.11 and 18.6: 516 checks per version, zero failures. +- SQLite macOS arm64: 150 unit checks and the focused regressions passed, including + the extension core built with AddressSanitizer/UndefinedBehaviorSanitizer. +- Three SQLite replicas: 30,000 randomized writes, 10,800 payload deliveries and 600 + convergence checks, with no divergence or outstanding SQLite allocations. +- 100 randomized fragment trials with duplicates, failure/rollback and replay passed. +- All 14 Linux network test groups and 340 fractional-indexing tests passed. +- Node: 14 tests, TypeScript checking and CJS/ESM/declaration builds passed. +- Migration compatibility (`1.1.3 -> 1.1.4`, unchanged SQL surface) and whitespace passed. -Update: `make endian-unittest` has since been removed. Once `pk.c` stopped using -host-order conversions, forcing `__BYTE_ORDER__` produced a byte-identical object, so -it could no longer fail; the host-order helpers it exercised are gone from -`cloudsync_endian.h`. Big-endian coverage is now `make unittest-s390x`: the SQLite unit -and regression suites run on s390x under QEMU, and with the pre-1.1.4 double encoding -restored they fail there on the golden bytes while passing on little-endian hosts. +The review found the unbounded cleanup-lock regression now covered by test 61. The +pre-fix query rolls back all maintenance with `out of shared memory` on a 30,000-group +backlog. With the bounded query, all 521 checks pass on PostgreSQL 15.19, 17.11 +and 18.6. The new test passes with the fix and fails on the previous query, confirming +that it detects the regression. It checks bounded progress, retained locks, concurrent +protection, the next cleanup batch and completion of the genuine fragmented value. + +The review did not contact a live cloud service or run Windows, Android, WebAssembly, +iOS or s390x. `make unittest-s390x` is the big-endian runtime target; the old +`endian-unittest` target was removed because forcing endian macros produced identical +objects and could not detect the bug. See [cloud-e2e.md](cloud-e2e.md) for the existing +real-cloud integration path and how to verify that cases actually ran. + +Known pre-existing limits identified during review: a failed deferred-constraint +commit on SQLite can leave a transaction open; PostgreSQL's fixed savepoint-owner +storage does not cover deeply nested callers (the scan/apply reproduction fails at +126 user savepoints); allocation-fault injection still exposes cleanup gaps. These +are not covered by a claim that all error paths or all platforms have been validated. diff --git a/docs/internal/cloud-e2e.md b/docs/internal/cloud-e2e.md new file mode 100644 index 00000000..cfbd202e --- /dev/null +++ b/docs/internal/cloud-e2e.md @@ -0,0 +1,94 @@ +# Testing against the real cloud + +The repository already runs real-cloud tests in `test/integration.c`, through +`.github/workflows/main.yml`. These are separate from `network-unittest`, whose +fixtures and loopback server do not contact the cloud. + +## Existing test tenants: the shortest path + +The repository has all seven `INTEGRATION_TEST_*` secrets referenced below. GitHub +exposes their names, not their values; they do not need to be copied to a laptop. +Use the existing branch workflow after pushing the client change: + +```sh +gh workflow run main.yml --repo sqliteai/sqlite-sync --ref pg-fixes11092026 +gh run list --repo sqliteai/sqlite-sync --branch pg-fixes11092026 --limit 5 +gh run watch RUN_ID --repo sqliteai/sqlite-sync --exit-status +``` + +A push already starts that workflow, so do not dispatch a second run for the same +commit unnecessarily. The workflow cancels older runs of the same branch. Avoid +running another branch or a local process against the shared chunked tenant at the +same time: the negative-cache test requires an idle, exclusive tenant, and the +workflow's concurrency group is per branch, not per tenant. + +Inspect the **linux-x86_64 build + test** job. Only that matrix leg receives +`INTEGRATION_TEST_CHUNKED_DATABASE_ID`. A green job alone is insufficient: optional +cases can report `SKIPPED`. Require `OK` for: + +- Init+Sync and Token Auth; +- Chunked Paths, Rowset, Single-Sync Drain, Capped Receive, Batched Receive, + Failure and Negative Cache; +- Offline Error and Failure Path. + +Evidence checked on 2026-09-19: [run 35424353890, Linux job 105847622981](https://github.com/sqliteai/sqlite-sync/actions/runs/35424353890/job/105847622981) +reported `OK` for all of those cases on client commit `1820a15`. That full workflow +failed in its Android x86_64 job; the Linux cloud cases did run and passed. This is +existing CI evidence, not a claim that the cleanup fix has already been deployed +and tested on the cloud server. + +## What the tenant fixtures must contain + +| Variable / secret | Required fixture | +| --- | --- | +| `INTEGRATION_TEST_CLOUDSYNC_ADDRESS` | Sync API base address accepted by `cloudsync_network_init_custom` | +| `INTEGRATION_TEST_APIKEY` | Credential authorized for the dedicated test tenants and gateway token minting | +| `INTEGRATION_TEST_WEBLITE_ADDRESS` | Gateway base address; the test calls `POST /v2/tokens` | +| `INTEGRATION_TEST_DATABASE_ID` | Normal sync fixture matching `db_init()` in `test/integration.c`: users, activities and workouts; initial activities present and workouts empty | +| `INTEGRATION_TEST_CHUNKED_DATABASE_ID` | Exclusive chunked fixture matching `test_chunked_schema_init()` and `test_chunked_failure_schema_init()`; server `payload_max_chunk_size=262144` | +| `INTEGRATION_TEST_OFFLINE_DATABASE_ID` | Paused database returning the expected `database_paused` / HTTP 503 error | +| `INTEGRATION_TEST_FAILURE_DATABASE_ID` | Reachable tenant whose backing node is deliberately not initialized for cloudsync, so asynchronous apply/check jobs report `cloudsync is not initialized` | + +Use the established fixtures if possible. A newly created empty database is not a +substitute for all four tenants: the normal case asserts existing seed data, the +chunked tests need a server page limit, and the two negative fixtures intentionally +have different failure modes. The tests write/delete remote rows and mint an expiring +synthetic-user token; use test tenants, not an application database. Failed runs can +leave test rows behind. Retain fixture seeds when resetting a tenant. + +For a local run, export the variables from a private environment/secret manager and +run `make e2e` from a disposable checkout after building the desired revision. Do not +commit a `.env` file, put credentials into shell history, or enable network tracing +on a shared log. The integration binary uses `./dist/cloudsync` and recreates +`health-track.sqlite` in its working directory. `make e2e` will load a local `.env` if +one exists. Review optional-test statuses; missing variables or a missing curl CLI +can skip coverage without failing the process. + +## Testing the server-side cleanup fix + +Building the SQLite client in CI does **not** deploy the PostgreSQL extension to +remote nodes. To cover this PR's cleanup regression against a real service: + +1. Provision a separate staging node/tenant, install the fixed PostgreSQL extension, + and verify `cloudsync_version()` plus the deployed build/commit identifier (the + version alone cannot distinguish two 1.1.4 builds). Recycle backend sessions that + had loaded the old shared library. +2. Run the real-cloud integration suite against correctly provisioned staging fixtures. + Check row contents and final receive completion, not only HTTP success. +3. On the staging node only, use the setup and assertions from + `test/postgresql/61_fragment_cleanup_backlog.sql`: 30,000 synthetic stale groups, + one group held by another transaction, and a genuine two-piece payload. Do not run + the whole SQL file on a shared node: its harness creates/drops a test database and + assumes local administrative access. Adapt its setup to the isolated tenant. +4. Trigger a real fragment upload through the gateway, then verify bounded cleanup + progress, preserved in-flight pieces and successful reconstruction. To observe a + subsequent maintenance pass on a persistent worker, allow the 60-second throttle + to expire (or use a new backend). Confirm progress again after releasing the lock. +5. Correlate client output with gateway/worker logs: no `out of shared memory`, no lost + fragment and no receive checkpoint beyond incomplete data. Record client, server, + gateway versions, tenant configuration and whether every optional case ran. + +Without staging-node deployment/admin access we can exercise the real network and +existing server behavior, but cannot honestly certify this PostgreSQL fix on the +remote server. Local PostgreSQL regression/concurrency tests remain the deterministic +check for the fix until that deployment is available. diff --git a/src/postgresql/sql_postgresql.c b/src/postgresql/sql_postgresql.c index 13a76702..dac1c0da 100644 --- a/src/postgresql/sql_postgresql.c +++ b/src/postgresql/sql_postgresql.c @@ -140,9 +140,17 @@ const char * const SQL_PAYLOAD_FRAGMENTS_DELETE = "DELETE FROM cloudsync_payload_fragments WHERE value_id=$1;"; const char * const SQL_PAYLOAD_FRAGMENTS_CLEANUP_STALE = + // Materialize the bounded candidate set BEFORE trying advisory locks. These locks + // live until transaction end; locking every stale value can exhaust PostgreSQL's + // shared lock table and roll back the entire cleanup, leaving a large backlog + // permanently stuck. Later cleanup calls drain another batch. Do not loop over + // batches here: releasing a savepoint does not release transaction-level locks. + "WITH stale AS MATERIALIZED (" + "SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " + "HAVING MAX(created_at) < $1 AND COUNT(*) < MAX(part_count) " + "ORDER BY value_id LIMIT 64) " "DELETE FROM cloudsync_payload_fragments WHERE value_id IN (" - "SELECT value_id FROM (SELECT value_id FROM cloudsync_payload_fragments GROUP BY value_id " - "HAVING MAX(created_at) < $1 AND COUNT(*) < MAX(part_count)) stale " + "SELECT value_id FROM stale " // skip a value another transaction is applying (see database_fragment_lock) "WHERE pg_try_advisory_xact_lock(1129530962, hashtext(value_id)));"; diff --git a/test/postgresql/61_fragment_cleanup_backlog.sql b/test/postgresql/61_fragment_cleanup_backlog.sql new file mode 100644 index 00000000..c2e22ed6 --- /dev/null +++ b/test/postgresql/61_fragment_cleanup_backlog.sql @@ -0,0 +1,84 @@ +-- A large stale-fragment backlog must make bounded progress without exhausting +-- PostgreSQL's shared advisory-lock table. A value being resumed stays protected. +\set testid '61-fragment-cleanup-backlog' +\ir helper_test_init.sql +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE IF EXISTS cloudsync_test_61; +CREATE DATABASE cloudsync_test_61; +\connect cloudsync_test_61 +\ir helper_psql_conn_setup.sql +CREATE EXTENSION cloudsync; +CREATE EXTENSION dblink; +CREATE TABLE t(id TEXT PRIMARY KEY NOT NULL, value TEXT); +SELECT cloudsync_init('t') AS _init \gset +SELECT cloudsync_set('payload_max_chunk_size', '1') AS _size \gset +INSERT INTO t VALUES ('big', repeat('A', 300000)); +CREATE TABLE transport AS SELECT chunk_index, payload FROM cloudsync_payload_chunks(); + +INSERT INTO cloudsync_payload_fragments + (value_id, part_index, part_count, total_size, checksum, created_at, + tbl, pk, col_name, col_version, db_version, site_id, cl, seq, fragment) +SELECT md5(i::text), 0, 2, 2, '0000000000000000', + extract(epoch FROM now())::bigint - 172800, + 'stale', decode('00','hex'), 'value', 1, 1, decode('00','hex'), 1, 0, decode('00','hex') +FROM generate_series(1, 30000) i; +SELECT value_id AS locked_value FROM cloudsync_payload_fragments ORDER BY value_id LIMIT 1 \gset +SELECT dblink_connect('busy', format('dbname=%s user=%s', current_database(), current_user)) AS _c \gset +SELECT dblink_exec('busy', 'BEGIN') AS _b \gset +SELECT n AS _n FROM dblink('busy', format( + 'SELECT 1 FROM pg_advisory_xact_lock(1129530962, hashtext(%L))', :'locked_value')) AS r(n INT) \gset + +BEGIN; +SELECT cloudsync_payload_apply(payload) AS _applied FROM transport WHERE chunk_index = 0 \gset +SELECT count(*) = 29937 AS progress_ok FROM cloudsync_payload_fragments WHERE tbl = 'stale' \gset +\if :progress_ok +\echo [PASS] (:testid) 30000 stale groups: cleanup removes 63 unlocked candidates without exhausting locks +\else +\echo [FAIL] (:testid) cleanup did not make bounded progress through the backlog +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT count(*) <= 65 AS locks_ok FROM pg_locks +WHERE pid = pg_backend_pid() AND locktype = 'advisory' AND classid = 1129530962 \gset +\if :locks_ok +\echo [PASS] (:testid) cleanup retains at most 64 locks plus the incoming value lock +\else +\echo [FAIL] (:testid) cleanup retained too many advisory locks +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT EXISTS (SELECT FROM cloudsync_payload_fragments WHERE value_id = :'locked_value') + AND (SELECT count(*) FROM cloudsync_payload_fragments WHERE tbl = 't') = 1 AS preserved_ok \gset +\if :preserved_ok +\echo [PASS] (:testid) locked stale value and incoming incomplete value are preserved +\else +\echo [FAIL] (:testid) cleanup removed a protected or fresh value +SELECT (:fail::int + 1) AS fail \gset +\endif +COMMIT; +SELECT dblink_exec('busy', 'COMMIT') AS _c \gset +SELECT dblink_disconnect('busy') AS _d \gset + +-- A new connection is eligible for maintenance immediately; an existing connection +-- is throttled to once a minute. The next batch must advance through the backlog. +\connect cloudsync_test_61 +\ir helper_psql_conn_setup.sql +SELECT cloudsync_payload_apply(payload) AS _applied FROM transport WHERE chunk_index = 0 \gset +SELECT (SELECT count(*) FROM cloudsync_payload_fragments WHERE tbl = 'stale') = 29873 + AND NOT EXISTS (SELECT FROM cloudsync_payload_fragments WHERE value_id = :'locked_value') AS next_ok \gset +\if :next_ok +\echo [PASS] (:testid) the next cleanup removes another 64 groups, including the released value +\else +\echo [FAIL] (:testid) subsequent cleanup made no progress or left the released candidate +SELECT (:fail::int + 1) AS fail \gset +\endif +SELECT cloudsync_payload_apply(payload) AS _applied FROM transport WHERE chunk_index = 1 \gset +SELECT NOT EXISTS (SELECT FROM cloudsync_payload_fragments WHERE tbl = 't') AS complete_ok \gset +\if :complete_ok +\echo [PASS] (:testid) the real fragmented value still completes with a large stale backlog +\else +\echo [FAIL] (:testid) the real fragmented value did not complete +SELECT (:fail::int + 1) AS fail \gset +\endif +\connect postgres +\ir helper_psql_conn_setup.sql +DROP DATABASE cloudsync_test_61; diff --git a/test/postgresql/full_test.sql b/test/postgresql/full_test.sql index 38a9273f..81e6cc92 100644 --- a/test/postgresql/full_test.sql +++ b/test/postgresql/full_test.sql @@ -68,6 +68,7 @@ \ir 58_v3_denied_checkpoint.sql \ir 59_rls_denial_retry.sql \ir 60_fragment_concurrency.sql +\ir 61_fragment_cleanup_backlog.sql -- 'Test summary' \echo '\nTest summary:'