From f3ab500b689d25704740e925e469a3d6cf0a9a3d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 1 Sep 2026 20:02:55 +0000 Subject: [PATCH 1/2] fix: fan UPDATE rows out to covering projections Co-authored-by: Cursor --- src/columnar_tableam.c | 10 +++++++ src/columnar_write_state.c | 4 ++- test/projection_update.sh | 58 ++++++++++++++++++++++++++++++++++++++ test/run_all_versions.sh | 1 + 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100755 test/projection_update.sh diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 6046f162..da305279 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -1562,6 +1562,16 @@ pgcolumnar_tuple_update(COLUMNAR_TUPLE_UPDATE_ARGS) rowNumber = PgColumnarWriteRow(writeState, rel, slot->tts_values, slot->tts_isnull); + /* + * The new version is a new row number. Projections store that number as + * their join key and skip deleted base numbers via the delete vector, so + * DELETE needs no fan-out. UPDATE does: without it the projection still + * holds the old number, the delete vector hides that number, and a covering + * projection scan (or read_projection) answers as if the row is gone. + */ + PgColumnarProjectionFanoutRow(rel, writeState, rowNumber, slot->tts_values, + slot->tts_isnull); + /* * The new row version makes its block not all-visible, exactly as a plain * insert does (gap 28). This half of update-as-delete-plus-insert was the diff --git a/src/columnar_write_state.c b/src/columnar_write_state.c index afd60ce2..097c6b7e 100644 --- a/src/columnar_write_state.c +++ b/src/columnar_write_state.c @@ -2876,7 +2876,9 @@ pgcolumnar_flush_row_group(PgColumnarWriteState *writeState) * reusing the base stripe encoder (PgColumnarWriteRow + pgcolumnar_flush_row_group). * The base row number is stored as a leading int8 column so the projection can * be joined back to the base; deletes/visibility come from the base delete_vector, so - * only INSERT fans out (see design/gaps/26-IMPL-projections-phase2-plan.md). + * DELETE does not rewrite the projection. UPDATE is delete-old plus insert-new + * and must fan the new number out or a covering projection scan will miss it + * (see design/gaps/26-IMPL-projections-phase2-plan.md). * ------------------------------------------------------------------------- */ /* one buffered projection row: [rownumber, projcol1..projcolK] */ diff --git a/test/projection_update.sh b/test/projection_update.sh new file mode 100755 index 00000000..7959d35a --- /dev/null +++ b/test/projection_update.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# pgColumnar: UPDATE must fan the new row number out to covering projections. +# +# A projection stores the base row number and filters with the base delete +# vector, so DELETE needs no rewrite of the projection. UPDATE is delete-old +# plus insert-new. The insert half used to skip fan-out, so the projection kept +# only the deleted number. read_projection then returned no rows, and a planner +# covering-projection scan (SELECT of projected columns with a sort-key qual) +# answered as if the updated rows had been deleted. +# +# Usage: test/projection_update.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +psql_run "CREATE TABLE pu (a int, b text, c int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.add_projection('pu', 'pc', ARRAY['a','c'], ARRAY['c']);" +psql_run "INSERT INTO pu SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" +psql_run "CREATE TABLE pu_h (a int, b text, c int) USING heap;" +psql_run "INSERT INTO pu_h SELECT g, 'r'||g, (g*7)%1000 FROM generate_series(1,20000) g;" + +check "premise: planner uses the covering projection" \ + "$(q "EXPLAIN (COSTS OFF) SELECT a, c FROM pu WHERE c BETWEEN 100 AND 200;" | grep -c 'Columnar Projection: pc')" "1" +check "premise: projection matches the heap before any update" \ + "$(pgc_set_hash "SELECT a, c FROM pu WHERE c BETWEEN 100 AND 200")" \ + "$(pgc_set_hash "SELECT a, c FROM pu_h WHERE c BETWEEN 100 AND 200")" + +psql_run "UPDATE pu SET a = a + 1 WHERE a % 10 = 0;" +psql_run "UPDATE pu_h SET a = a + 1 WHERE a % 10 = 0;" + +check "covering projection scan matches heap after updating a projected column" \ + "$(pgc_set_hash "SELECT a, c FROM pu WHERE c BETWEEN 100 AND 200")" \ + "$(pgc_set_hash "SELECT a, c FROM pu_h WHERE c BETWEEN 100 AND 200")" +check "read_projection matches the live base after that update" \ + "$(pgc_set_hash "SELECT pgcolumnar.read_projection('pu','pc')")" \ + "$(pgc_set_hash "SELECT a::text || '|' || c::text FROM pu")" +check "full-table projection scan still matches heap" \ + "$(pgc_set_hash "SELECT a, c FROM pu")" \ + "$(pgc_set_hash "SELECT a, c FROM pu_h")" + +# A column the projection does not store still allocates a new base row number. +psql_run "UPDATE pu SET b = 'y' WHERE a % 7 = 0;" +psql_run "UPDATE pu_h SET b = 'y' WHERE a % 7 = 0;" + +check "covering projection scan matches heap after updating a non-projected column" \ + "$(pgc_set_hash "SELECT a, c FROM pu WHERE c BETWEEN 100 AND 200")" \ + "$(pgc_set_hash "SELECT a, c FROM pu_h WHERE c BETWEEN 100 AND 200")" +check "read_projection still matches the live base" \ + "$(pgc_set_hash "SELECT pgcolumnar.read_projection('pu','pc')")" \ + "$(pgc_set_hash "SELECT a::text || '|' || c::text FROM pu")" +check "reconstruct via the projection still rebuilds every live row" \ + "$(pgc_set_hash "SELECT pgcolumnar.reconstruct_via_projection('pu','pc')")" \ + "$(pgc_set_hash "SELECT a::text || '|' || b || '|' || c::text FROM pu")" + +pgc_summary diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 28d85fa0..e4780004 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -238,6 +238,7 @@ SUITES=( planner_choice_quality preimage_rewrite projection_privilege + projection_update projections pushdown_report qual_order_selectivity From 3a9249f0d16a393e1113fb151b0b79c700002d77 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 1 Sep 2026 20:28:05 -0600 Subject: [PATCH 2/2] docs: an update writes projections too, and a delete still does not Four sentences across three documents said that inserts write projections, and stopped there. That was true when only insert fanned out. This PR makes UPDATE fan out as well, so all four now understate what a projection costs and, worse, a reader sizing a projection would budget for the wrong write volume. docs/features.md:99 "Every insert fans out to each projection." docs/administration.md:250 "New inserts write to the base table and its projections." docs/administration.md:264 "A projection adds write cost and storage, because inserts write it too." docs/how-to.md:151 "Later inserts write to the base table and to every projection" Each now says updates write too. Each also says why a delete does not, because "insert and update but not delete" reads as an oversight unless the reason is there: a projection takes visibility from the base delete vector, so the vector hides a deleted row number from the projection without any rewrite. The CHANGELOG entry states the failure a reader would have hit rather than the call that fixes it: the base table was always correct, and the loss was confined to reads the planner served from a projection, so the same query returned different answers depending on whether the projection was chosen. Verified rather than assumed: test/plain_language_check.py passes on all fifteen user-facing documents, and CHANGELOG.md carries no em or en dash, which is the other thing docs_style.sh gates. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WDbfRym2V1sYFmMZ5gnsQL --- CHANGELOG.md | 22 ++++++++++++++++++++++ docs/administration.md | 6 ++++-- docs/features.md | 6 ++++-- docs/how-to.md | 4 ++-- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 333c91c8..96b7af89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,28 @@ true until the next version shipped. ### Fixed +- `UPDATE` now fans the new row version out to every covering projection, so a + projection scan stops answering as if the updated rows were gone. + + **A projection stored the base row number, and `UPDATE` never gave it the new + one.** A projection joins back to the base table on the row number and takes + visibility from the base delete vector, so `DELETE` needs no rewrite: the + vector hides the old number from the projection too. `UPDATE` is delete-old + plus insert-new, and only the delete half reached the projection. The + projection kept the old number, the delete vector hid it, and the new number + was nowhere. `read_projection` returned no rows, and a covering projection + scan (projected columns with a sort-key qual) answered as if the updated rows + had been removed. + + The base table was always correct. The loss was confined to reads the planner + served from a projection, which makes it the worse shape: the same query + returns different answers depending on whether the projection is chosen. + + `tuple_update` now calls the same fan-out that insert calls. + + `docs/features.md`, `docs/administration.md` and `docs/how-to.md` said only + that inserts write projections. All three now say updates do too, and say why + a delete does not. - Arrow import reads the temporal unit and carrier width the file declares, rather than assuming the ones our own exporter writes (#864, #865). diff --git a/docs/administration.md b/docs/administration.md index 9c7b0f4c..0d195ac8 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -248,7 +248,8 @@ SELECT pgcolumnar.add_projection( ``` When you add the projection, pgColumnar fills it with the rows that exist. New inserts write to -the base table and its projections. Projection scans are on by default +the base table and its projections. Updates write there too, because an update +creates a new row version. Projection scans are on by default (`pgcolumnar.enable_projection_scan`). Drop a projection with `pgcolumnar.drop_projection`. @@ -261,7 +262,8 @@ it built. A second run builds nothing. A physical backup (`pg_basebackup`) preserves the projections themselves, which `test/replication.sh` verifies against a standby. -A projection adds write cost and storage, because inserts write it too. Add one +A projection adds write cost and storage, because inserts and updates both write +it. Add one for a query pattern that a covering, sorted column subset serves, and measure the result. Confirm the plan uses it with `EXPLAIN`, which names the chosen projection. diff --git a/docs/features.md b/docs/features.md index 4f6cfffb..f7500f65 100644 --- a/docs/features.md +++ b/docs/features.md @@ -97,8 +97,10 @@ coverage. `pgcolumnar.add_projection(table, name, columns, sort_key)` declares an extra physical copy of a subset of the columns. That copy has its own sort order. It shares the row identity of the table. Every insert fans - out to each projection. A projection stored sorted has tight per-chunk minimum - and maximum ranges. + out to each projection. So does every update, because an update writes a new + row version with a new row number. A delete needs no fan-out, because the base + delete vector hides the old row number from projection scans. A projection + stored sorted has tight per-chunk minimum and maximum ranges. - The planner scans a projection instead of the base table when it covers the query's columns and its leading sort column is restricted. `EXPLAIN` shows `Columnar Projection: `. Deletes and MVCC visibility come from the base, diff --git a/docs/how-to.md b/docs/how-to.md index df738240..6fb39640 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -148,8 +148,8 @@ SELECT pgcolumnar.add_projection( EXPLAIN SELECT sum(amount) FROM events WHERE customer_id = 42; ``` -Adding a projection fills it from the existing rows. Later inserts write to the -base table and to every projection, so each projection adds write cost. Drop one +Adding a projection fills it from the existing rows. Later inserts and updates write to +the base table and to every projection, so each projection adds write cost. Drop one with `pgcolumnar.drop_projection('events', 'events_by_customer')`. **Tuning.** Add a projection only for a hot access pattern the base sort order