From 777ba63c240f118f86c2b62570e117503935201e Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 3 Sep 2026 15:15:21 +0800 Subject: [PATCH 1/9] datalake_fdw: read and write Parquet through Arrow The format layer had an interface and no implementation. This is the Parquet one, and the conversions either side of it: PostgreSQL tuples into an Arrow batch, and an Arrow batch back into Datums. Arrow rather than libparquet alone, because libparquet is written in terms of Arrow's types, so linking one links the other. It is a build dependency now, found with pkg-config -- and whatever -std= its .pc file asks for is filtered out, because those flags land after CXXFLAGS and the Arrow project's own packages say -std=c++11. A fragment is a range of row groups rather than a whole file, so several segments can read one large file. Reading is single-threaded: a worker thread that fails has no way to report it through PostgreSQL. A reader and a writer hold a descriptor and memory from Arrow's allocator, which transaction abort does not reclaim, so both register with the resource owner -- the shape PAX uses in comm/pax_resource.cc. Types: bool, the four integers, both floats, text, varchar, char, bytea, date, timestamp, timestamptz. Anything else is refused by name. The 30-year epoch shift is range checked in both directions: PostgreSQL's range runs past what Arrow holds as microseconds from 1970, and a round trip through two unchecked halves would agree with itself. datalake_fdw_test is a second extension in the same library with the two functions this can be run with from SQL. Its regression case round-trips every supported type and checks that reading row groups separately gives back what reading the whole file does. Arrow 9.0.0 (EPEL 9) builds and passes; 17.0.0 (Rocky 10, gcc 14) and 17.0.0 and 21.0.0 (Rocky 8, gcc 8) compile without warnings. What is written here reads back correctly in pyarrow 21, which is not the implementation that wrote it. --- .github/workflows/build-cloudberry.yml | 28 ++ contrib/datalake_fdw/Makefile | 82 +++- .../datalake_fdw/datalake_fdw_test--1.0.sql | 51 ++ .../datalake_fdw/datalake_fdw_test.control | 24 + contrib/datalake_fdw/exports.txt | 7 + .../src/am_iceberg/pg_iceberg_extensible.c | 2 + .../src/am_iceberg/pg_iceberg_guc.c | 24 + .../src/am_iceberg/pg_iceberg_guc.h | 1 + contrib/datalake_fdw/src/common/dl_err.c | 12 +- contrib/datalake_fdw/src/common/dl_err.h | 15 +- contrib/datalake_fdw/src/common/dl_resource.c | 139 ++++++ contrib/datalake_fdw/src/common/dl_resource.h | 78 +++ contrib/datalake_fdw/src/common/dl_wrappers.h | 23 +- .../src/common/file_system_wrapper.cpp | 12 +- .../src/common/s3_file_system.cpp | 12 +- .../datalake_fdw/src/format/arrow_builder.cpp | 341 +++++++++++++ .../datalake_fdw/src/format/arrow_builder.h | 95 ++++ .../datalake_fdw/src/format/arrow_decode.c | 355 ++++++++++++++ .../datalake_fdw/src/format/arrow_decode.h | 75 +++ .../datalake_fdw/src/format/arrow_support.cpp | 179 +++++++ .../datalake_fdw/src/format/arrow_support.h | 75 +++ contrib/datalake_fdw/src/format/format.h | 77 ++- .../datalake_fdw/src/format/format_registry.c | 24 +- .../src/format/parquet/parquet_format.cpp | 55 +++ .../src/format/parquet/parquet_format.h | 45 ++ .../src/format/parquet/parquet_internal.h | 47 ++ .../src/format/parquet/parquet_read.cpp | 280 +++++++++++ .../src/format/parquet/parquet_write.cpp | 451 ++++++++++++++++++ .../datalake_fdw/src/test/datalake_fdw_test.c | 411 ++++++++++++++++ .../datalake_fdw/test/automation/README.md | 13 +- .../scripts/test/run_smoke_tests.sh | 14 +- .../expected/parquet_roundtrip.out | 250 ++++++++++ .../format_parquet/sql/parquet_roundtrip.sql | 174 +++++++ 33 files changed, 3429 insertions(+), 42 deletions(-) create mode 100644 contrib/datalake_fdw/datalake_fdw_test--1.0.sql create mode 100644 contrib/datalake_fdw/datalake_fdw_test.control create mode 100644 contrib/datalake_fdw/src/common/dl_resource.c create mode 100644 contrib/datalake_fdw/src/common/dl_resource.h create mode 100644 contrib/datalake_fdw/src/format/arrow_builder.cpp create mode 100644 contrib/datalake_fdw/src/format/arrow_builder.h create mode 100644 contrib/datalake_fdw/src/format/arrow_decode.c create mode 100644 contrib/datalake_fdw/src/format/arrow_decode.h create mode 100644 contrib/datalake_fdw/src/format/arrow_support.cpp create mode 100644 contrib/datalake_fdw/src/format/arrow_support.h create mode 100644 contrib/datalake_fdw/src/format/parquet/parquet_format.cpp create mode 100644 contrib/datalake_fdw/src/format/parquet/parquet_format.h create mode 100644 contrib/datalake_fdw/src/format/parquet/parquet_internal.h create mode 100644 contrib/datalake_fdw/src/format/parquet/parquet_read.cpp create mode 100644 contrib/datalake_fdw/src/format/parquet/parquet_write.cpp create mode 100644 contrib/datalake_fdw/src/test/datalake_fdw_test.c create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index b790d3411ec..ef84e33dc7f 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -1463,6 +1463,34 @@ jobs: exit 0 fi + # datalake_fdw needs the Arrow and Parquet C++ libraries, which the + # build image does not carry -- nothing in the RPM uses them, so they + # would be weight every other job paid for. + if [[ "${PGXS_EXTENSION}" == "contrib/datalake_fdw" ]]; then + . /etc/os-release + if [[ "${VERSION_ID%%.*}" == "8" ]]; then + # EPEL 8 has them, but its libarrow-devel needs a utf8proc-devel + # that modular filtering keeps out of PowerTools, so it cannot be + # installed. The Arrow project's own repository can. Pinned to + # the version EPEL 10 carries, both because that is one version + # fewer to have working and because the newest wants C++20, which + # Rocky 8's gcc 8 does not have. + # EPEL as well, and not only for Arrow itself: arrow-devel needs + # re2-devel and parquet-devel needs thrift-devel, and on EL8 both + # of those live in EPEL. + dnf install -y \ + https://apache.jfrog.io/artifactory/arrow/almalinux/8/apache-arrow-release-latest.rpm + dnf install -y --enablerepo=epel --enablerepo=powertools \ + arrow-devel-17.0.0-1.el8 parquet-devel-17.0.0-1.el8 + else + # From EPEL, which the image has enrolled but left disabled, + # exactly as it does for its own EPEL packages; CRB carries what + # they depend on. + dnf install -y --enablerepo=epel --enablerepo=crb \ + libarrow-devel parquet-libs-devel + fi + fi + # The RPM installs as root; the build runs as gpadmin, as everywhere # else in this job. -H follows the command-line symlink. chown -RH gpadmin:gpadmin "${BUILD_DESTINATION}/" diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile index bd1d0179513..0653764581c 100644 --- a/contrib/datalake_fdw/Makefile +++ b/contrib/datalake_fdw/Makefile @@ -18,8 +18,11 @@ # contrib/datalake_fdw/Makefile MODULE_big = datalake_fdw -EXTENSION = datalake_fdw -DATA = datalake_fdw--1.0.sql + +# A second extension, so that installing datalake_fdw does not put the test +# entry points in a production database. One library, so they reach internals. +EXTENSION = datalake_fdw datalake_fdw_test +DATA = datalake_fdw--1.0.sql datalake_fdw_test--1.0.sql OBJS = \ src/am_iceberg/pg_iceberg_am_handler.o \ @@ -36,12 +39,46 @@ OBJS = \ src/meta/meta_engine_init.o \ src/meta/engine_stub/stub_engine.o \ src/format/format_registry.o \ + src/format/arrow_support.o \ + src/format/arrow_builder.o \ + src/format/arrow_decode.o \ + src/format/parquet/parquet_format.o \ + src/format/parquet/parquet_read.o \ + src/format/parquet/parquet_write.o \ src/common/dl_err.o \ + src/common/dl_resource.o \ src/common/dl_option_util.o \ src/common/parser_option.o \ src/common/file_system_wrapper.o \ src/common/s3_file_system.o \ - src/common/backend_registry.o + src/common/backend_registry.o \ + src/test/datalake_fdw_test.o + +# libparquet is written in terms of Arrow's types, so linking one links both. +PKG_CONFIG ?= pkg-config +ARROW_MODULES = arrow parquet +HAVE_ARROW := $(shell $(PKG_CONFIG) --exists $(ARROW_MODULES) 2>/dev/null && echo yes) +# Without the filter, whichever C++ standard Arrow's .pc file names wins: these +# land in CPPFLAGS, which pgxs.mk puts after CXXFLAGS on the command line. The +# Arrow project's own packages say -std=c++11, and their headers then fail to +# compile against themselves. +ARROW_CPPFLAGS := $(filter-out -std=%,\ + $(shell $(PKG_CONFIG) --cflags $(ARROW_MODULES) 2>/dev/null)) +ARROW_LIBS := $(shell $(PKG_CONFIG) --libs $(ARROW_MODULES) 2>/dev/null) + +# This has to refuse at parse time. A recipe hung off `all` would run after +# pgxs.mk's own all-lib, so the compiler would fail on a missing arrow/api.h +# first and this would never be reached. Not for the clean targets, because +# contrib/Makefile recurses here for those even when the module is not +# configured in. +ifneq ($(HAVE_ARROW),yes) +ifeq ($(filter clean distclean maintainer-clean,$(MAKECMDGOALS)),) +$(error datalake_fdw needs the Apache Arrow and Parquet C++ libraries, and \ +pkg-config found neither "arrow" nor "parquet". They are libarrow-devel and \ +parquet-libs-devel on Rocky and RHEL, libarrow-dev and libparquet-dev on \ +Debian and Ubuntu) +endif +endif # Use the documented PGXS knobs: pgxs.mk appends these AFTER the flags configure # chose, so optimization/warning settings survive. A pre-include @@ -49,7 +86,7 @@ OBJS = \ # Makefile.global's own "CFLAGS = @CFLAGS@" assignment. PG_CFLAGS = -fvisibility=hidden PG_CXXFLAGS = -fvisibility=hidden -fvisibility-inlines-hidden -std=c++17 -PG_CPPFLAGS = -I$(srcdir)/src +PG_CPPFLAGS = -I$(srcdir)/src $(ARROW_CPPFLAGS) # The regression cases live with the rest of the test material rather than in a # second place of their own; pg_regress is pointed at them. REGRESS_OPTS is @@ -65,6 +102,10 @@ REGRESS = iceberg_am_ddl iceberg_am_reject iceberg_am_acl REGRESS_OPTS = --temp-config=$(srcdir)/datalake_fdw.conf \ --inputdir=$(srcdir)/test/automation/sqlrepo/smoke/iceberg_am +# A second category, and pg_regress takes one --inputdir, so it is a second run. +FORMAT_PARQUET_REGRESS = parquet_roundtrip +FORMAT_PARQUET_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/format_parquet + EXTRA_CLEAN = exports_darwin.list exports.map # Keep the aggregate target as make's default goal. @@ -87,10 +128,10 @@ endif # Shared libraries are linked with $(CC) (see src/Makefile.shlib COMPILER), so a # module containing C++ translation units must pull in the C++ runtime itself. -SHLIB_LINK += -lstdc++ +SHLIB_LINK += -lstdc++ $(ARROW_LIBS) -# Arrow and other C++ dependencies land in this module later; the export list is -# the single place that decides what stays visible, so the mechanism ships now. +# The export list is the single place that decides what stays visible -- which +# now also means none of Arrow's symbols become symbols this module offers. ifeq ($(PORTNAME), darwin) EXPORT_LIST = exports_darwin.list SHLIB_LINK += -Wl,-exported_symbols_list,exports_darwin.list @@ -107,3 +148,30 @@ endif all: $(EXPORT_LIST) $(shlib): $(EXPORT_LIST) + +# Hung off check and installcheck so that both get both categories. REGRESS_OPTS +# has to come along: pgxs.mk is where --dbname=$(CONTRIB_TESTDB) is added to it, +# and without that pg_regress falls back to "regression" -- which it DROPs and +# recreates, taking the core suite's database with it. The second --inputdir +# wins over the one in REGRESS_OPTS. submake and REGRESS_PREP are the same +# prerequisites pgxs.mk gives its own targets, so that a parallel make cannot +# start pg_regress before it has been built. +installcheck: installcheck-format-parquet + +installcheck-format-parquet: submake $(REGRESS_PREP) + $(pg_regress_installcheck) $(REGRESS_OPTS) \ + --inputdir=$(FORMAT_PARQUET_INPUTDIR) $(FORMAT_PARQUET_REGRESS) + +.PHONY: installcheck-format-parquet + +# "make check" is in-tree only -- under PGXS pgxs.mk refuses the target -- and +# it is the only run that supplies the temp-config that preloads this module. +ifndef USE_PGXS +check: check-format-parquet + +check-format-parquet: submake $(REGRESS_PREP) + $(pg_regress_check) $(REGRESS_OPTS) \ + --inputdir=$(FORMAT_PARQUET_INPUTDIR) $(FORMAT_PARQUET_REGRESS) + +.PHONY: check-format-parquet +endif diff --git a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql new file mode 100644 index 00000000000..bb76a5023ba --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * contrib/datalake_fdw/datalake_fdw_test--1.0.sql + */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION datalake_fdw_test" to load this file. \quit + +/* + * Both functions name a path on the server's file system and run as the + * operating system user the server does, so they are as privileged as + * pg_read_server_files and are granted the same way: to nobody, until someone + * decides otherwise. + * + * The reader is pinned to the coordinator. Without that the planner may put a + * function scan on the segments, where each of them would read the whole file + * and the rows would come back as many times as there are segments. The writer + * cannot say the same -- EXECUTE ON is only accepted for a set-returning + * function -- but it does not need to: it is called in a target list with no + * FROM clause, which is evaluated on the coordinator, and the query it runs is + * dispatched from there like any other. + */ +CREATE FUNCTION datalake_parquet_write(path text, + query text, + row_group_size int DEFAULT 0) +RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; + +REVOKE EXECUTE ON FUNCTION datalake_parquet_write(text, text, int) FROM PUBLIC; + +CREATE FUNCTION datalake_parquet_read(path text, + first_row_group int DEFAULT 0, + n_row_groups int DEFAULT 0) +RETURNS SETOF record AS 'MODULE_PATHNAME' LANGUAGE C STRICT EXECUTE ON COORDINATOR; + +REVOKE EXECUTE ON FUNCTION datalake_parquet_read(text, int, int) FROM PUBLIC; diff --git a/contrib/datalake_fdw/datalake_fdw_test.control b/contrib/datalake_fdw/datalake_fdw_test.control new file mode 100644 index 00000000000..8df236864f0 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw_test.control @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/datalake_fdw_test.control + +comment = 'entry points into datalake_fdw internals, for testing' +default_version = '1.0' +module_pathname = '$libdir/datalake_fdw' +relocatable = false +requires = 'datalake_fdw' diff --git a/contrib/datalake_fdw/exports.txt b/contrib/datalake_fdw/exports.txt index 0db251366df..e89c850b512 100644 --- a/contrib/datalake_fdw/exports.txt +++ b/contrib/datalake_fdw/exports.txt @@ -30,3 +30,10 @@ pg_finfo_iceberg_catalog_fdw_validator iceberg_catalog_fdw_validator pg_finfo_iceberg_volume_fdw_validator iceberg_volume_fdw_validator + +# datalake_fdw_test: not part of what this module offers, but a SQL-callable +# function has to be found by name in the library like any other. +pg_finfo_datalake_parquet_write +datalake_parquet_write +pg_finfo_datalake_parquet_read +datalake_parquet_read diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c index f675a1ef24d..26e1f7a68c5 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c @@ -33,6 +33,7 @@ #include "access/table.h" #include "access/tableam.h" #include "am_iceberg/pg_iceberg_ddl.h" +#include "common/dl_resource.h" #include "am_iceberg/pg_iceberg_guc.h" #include "am_iceberg/pg_iceberg_options.h" #include "am_iceberg/pg_iceberg_reject.h" @@ -951,6 +952,7 @@ _PG_init(void) errmsg("datalake_fdw must be loaded via shared_preload_libraries"), errhint("Add \"datalake_fdw\" to shared_preload_libraries and restart the server."))); + dl_resource_init(); pg_iceberg_define_gucs(); pg_iceberg_register_reloptions(); DatalakeRegisterMetaEngines(); diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c index 71dc5c53ef1..5e9ec353f64 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c @@ -33,6 +33,7 @@ char *iceberg_default_catalog; char *iceberg_default_volume; +int iceberg_batch_rows; void pg_iceberg_define_gucs(void) @@ -64,4 +65,27 @@ pg_iceberg_define_gucs(void) NULL, NULL, NULL); + + /* + * How many rows travel between the executor and a data file at a time. + * Every per-batch cost is paid once per this many rows, and the batch and + * its Arrow copy are held while it is built, so the right value trades + * memory for that -- which depends on how wide the table is, and is why + * this is a setting rather than a constant. + * + * The ceiling is Parquet's default row group length: a batch bigger than + * the unit a file is written in buys nothing. + */ + DefineCustomIntVariable("iceberg.batch_rows", + "Rows per batch exchanged with a lake table's data files.", + NULL, + &iceberg_batch_rows, + 16384, + 1, + 1024 * 1024, + PGC_USERSET, + 0, + NULL, + NULL, + NULL); } diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h index 8cd8d0a4400..68d30bb1908 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h @@ -31,6 +31,7 @@ extern char *iceberg_default_catalog; extern char *iceberg_default_volume; +extern int iceberg_batch_rows; extern void pg_iceberg_define_gucs(void); diff --git a/contrib/datalake_fdw/src/common/dl_err.c b/contrib/datalake_fdw/src/common/dl_err.c index 4780d9d44a0..44f32cf307b 100644 --- a/contrib/datalake_fdw/src/common/dl_err.c +++ b/contrib/datalake_fdw/src/common/dl_err.c @@ -160,10 +160,20 @@ dl_err_message(DlErrCode code) void dl_error_report(int elevel, DlErrCode code, const char *prefix) { - const DlErrorDetail *detail = dl_error_get(); + /* + * A copy, because reporting consumes the record: what is left otherwise is + * a description of a failure that has already been reported, waiting for + * the next failure with the same code to adopt it -- and the check below + * cannot tell those two apart. The reset has to happen before the ereport, + * which at ERROR does not come back. + */ + DlErrorDetail detail_copy = *dl_error_get(); + const DlErrorDetail *detail = &detail_copy; StringInfoData detail_buf; bool has_detail; + dl_error_reset(); + /* * Detail recorded against a different code belongs to some other failure -- * an implementation that reported this one without recording anything, for diff --git a/contrib/datalake_fdw/src/common/dl_err.h b/contrib/datalake_fdw/src/common/dl_err.h index 67949644c95..7d0e652ed4d 100644 --- a/contrib/datalake_fdw/src/common/dl_err.h +++ b/contrib/datalake_fdw/src/common/dl_err.h @@ -109,10 +109,23 @@ extern const char *dl_err_message(DlErrCode code); * when the session asked for log-level detail -- a stack is for whoever is * debugging the implementation, not for whoever ran the statement. * - * Detail recorded against a different code is ignored rather than misattributed. + * Detail recorded against a different code is ignored rather than misattributed, + * and reporting consumes what it used. Matching on the code alone cannot tell + * this failure's detail from an earlier failure's with the same code, so the + * record is not left behind for the next one to inherit. */ extern void dl_error_report(int elevel, DlErrCode code, const char *prefix); +/* + * An argument that was not what it had to be. A caller reporting the code + * alone would say "invalid parameter" about a call the user never wrote, so + * this names the entry point instead -- there is no user error to describe, + * only which one of ours was called wrongly. + */ +#define DL_ARG_ERROR(operation) \ + (dl_error_set(DL_ERR_INVALID_OPTION, (operation), NULL, \ + "a required argument was missing"), DL_ERR_INVALID_OPTION) + #ifdef __cplusplus } #endif diff --git a/contrib/datalake_fdw/src/common/dl_resource.c b/contrib/datalake_fdw/src/common/dl_resource.c new file mode 100644 index 00000000000..691180c2407 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_resource.c @@ -0,0 +1,139 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_resource.c + * Cleanups that happen even when nothing calls them. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_resource.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include + +#include "storage/ipc.h" +#include "utils/resowner.h" + +#include "common/dl_resource.h" + +typedef struct DlResourceEntry +{ + struct DlResourceEntry *next; + ResourceOwner owner; + DlResourceRelease release; + void *arg; +} DlResourceEntry; + +/* + * There are a handful of these at a time -- one per open data file -- so a list + * walked linearly is the whole structure needed. + */ +static DlResourceEntry *dl_resources; + +/* + * malloc rather than palloc: this outlives the memory context that was current + * when it was remembered, by construction, and is walked while the transaction + * is being torn down. + */ +static void +dl_resource_release_callback(ResourceReleasePhase phase, bool isCommit, + bool isTopLevel, void *arg) +{ + DlResourceEntry **link; + + /* + * After locks, so that anything the release path might touch is still + * usable. Nothing to do while the process is exiting: the descriptors go + * with it, and running C++ destructors on the way out is a way to turn an + * exit into a crash. + */ + if (phase != RESOURCE_RELEASE_AFTER_LOCKS || proc_exit_inprogress) + return; + + link = &dl_resources; + while (*link != NULL) + { + DlResourceEntry *entry = *link; + + if (entry->owner != CurrentResourceOwner) + { + link = &entry->next; + continue; + } + + /* + * Reaching here on a commit means the owner released nothing: the + * statement finished and left a file open. The resource is still + * cleaned up, but quietly doing so would hide the bug that let it + * happen, which is the same call PostgreSQL's own resource owners make. + */ + if (isCommit) + elog(WARNING, "datalake_fdw leaked a resource: %p", entry->arg); + + *link = entry->next; + entry->release(entry->arg); + free(entry); + } +} + +void +dl_resource_init(void) +{ + RegisterResourceReleaseCallback(dl_resource_release_callback, NULL); +} + +bool +dl_resource_remember(DlResourceRelease release, void *arg) +{ + DlResourceEntry *entry = malloc(sizeof(DlResourceEntry)); + + if (entry == NULL) + return false; + + entry->owner = CurrentResourceOwner; + entry->release = release; + entry->arg = arg; + entry->next = dl_resources; + dl_resources = entry; + + return true; +} + +void +dl_resource_forget(DlResourceRelease release, void *arg) +{ + DlResourceEntry **link = &dl_resources; + + while (*link != NULL) + { + DlResourceEntry *entry = *link; + + if (entry->release == release && entry->arg == arg) + { + *link = entry->next; + free(entry); + return; + } + + link = &entry->next; + } +} diff --git a/contrib/datalake_fdw/src/common/dl_resource.h b/contrib/datalake_fdw/src/common/dl_resource.h new file mode 100644 index 00000000000..00ecc51c7d8 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_resource.h @@ -0,0 +1,78 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_resource.h + * Cleanups that happen even when nothing calls them. + * + * A reader or a writer holds an operating system file descriptor and memory + * from an allocator that is not PostgreSQL's. Neither is reclaimed by + * transaction abort, so anything that owns one has to be released by name -- + * and a caller that raises an error before it reaches its own cleanup would + * never get to. Registering here makes the resource owner do it instead, and + * makes a caller that simply forgot a warning rather than a descriptor that is + * gone until the backend exits. + * + * This is the shape PAX uses (contrib/pax_storage/src/cpp/comm/pax_resource.cc): + * a callback registered once, and a list of what to release keyed by the owner + * that was current when it was remembered. PostgreSQL 16 also has a typed + * resource-kind API, which this server does not carry. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_resource.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_RESOURCE_H +#define DL_RESOURCE_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * Runs during transaction abort, so the same rules as any other cleanup path: + * noexcept, and it must not raise. + */ +typedef void (*DlResourceRelease) (void *arg); + +/* Installs the callback. Called once, from _PG_init. */ +extern void dl_resource_init(void); + +/* + * Remembers that `release(arg)` has to happen before the current resource owner + * goes away. Returns false only when it could not record that, which the + * caller has to treat as a failure to acquire the resource at all -- releasing + * it itself and reporting -- because nothing else is going to. + */ +extern bool dl_resource_remember(DlResourceRelease release, void *arg); + +/* + * Drops that record, for the ordinary path where the caller releases the + * resource itself. Silent when there is nothing to drop: a cleanup that runs + * twice reaches this the second time. + */ +extern void dl_resource_forget(DlResourceRelease release, void *arg); + +#ifdef __cplusplus +} +#endif + +#endif /* DL_RESOURCE_H */ diff --git a/contrib/datalake_fdw/src/common/dl_wrappers.h b/contrib/datalake_fdw/src/common/dl_wrappers.h index 737d97fae7d..5aadc9b4bce 100644 --- a/contrib/datalake_fdw/src/common/dl_wrappers.h +++ b/contrib/datalake_fdw/src/common/dl_wrappers.h @@ -136,15 +136,36 @@ dl_can_log_cleanup_warning(void) errmsg("datalake_fdw: %s", dl_error_msg_))); \ } while (0) +/* + * Class 2 guards record what happened as well as that it happened. + * + * The detail is one per-backend record, and dl_error_report() decides whether + * it belongs to the failure being reported by comparing codes -- which cannot + * tell this DL_ERR_INTERNAL from an earlier one. A guard that set the code and + * recorded nothing would therefore report the previous statement's message as + * the cause of this one. So it always records, and what a C++ exception has to + * say is the only description of it there is going to be. PAX takes the same + * line in CBDB_END_TRY(), where an unnamed failure falls back to the function + * it happened in rather than to whatever was there before. + * + * `operation` names the call, the way the metadata engine's dispatch does. + */ #define DL_ABI_GUARD_BEGIN \ try \ { -#define DL_ABI_GUARD_END(errvar) \ +#define DL_ABI_GUARD_END(errvar, operation) \ + } \ + catch (const std::exception &e) \ + { \ + (errvar) = DL_ERR_INTERNAL; \ + dl_error_set(DL_ERR_INTERNAL, (operation), NULL, e.what()); \ } \ catch (...) \ { \ (errvar) = DL_ERR_INTERNAL; \ + dl_error_set(DL_ERR_INTERNAL, (operation), NULL, \ + "unknown C++ exception"); \ } #define DL_CLEANUP_GUARD_BEGIN \ diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp index 2b576fe4344..c0c8b2d3e51 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -71,7 +71,7 @@ datalake_fs_open(const DatalakeLocation *location, } } } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "fs_open"); return rc; } @@ -116,7 +116,7 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, rc = fs->ops->fs_list(fs, prefix, names_out, nnames_out); } } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "fs_list"); return rc; } @@ -144,7 +144,7 @@ datalake_file_open(DatalakeFileSystem fs, const char *path, (*file_out)->ops = fs->ops; } } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "file_open"); return rc; } @@ -165,7 +165,7 @@ datalake_file_read(DatalakeFile file, void *buffer, int64_t length, rc = file->ops->file_read(file, buffer, length, nread); } } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "file_read"); return rc; } @@ -182,7 +182,7 @@ datalake_file_write(DatalakeFile file, const void *buffer, int64_t length) else rc = file->ops->file_write(file, buffer, length); } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "file_write"); return rc; } @@ -209,7 +209,7 @@ datalake_file_close(DatalakeFile *file) rc = doomed->ops->file_close(doomed); } } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "file_close"); return rc; } diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp index d255200528b..ebb7fadf2ad 100644 --- a/contrib/datalake_fdw/src/common/s3_file_system.cpp +++ b/contrib/datalake_fdw/src/common/s3_file_system.cpp @@ -122,7 +122,7 @@ s3_fs_open(const DatalakeLocation *location, const DlKeyValue *credentials, *fs_out = handle.release(); } } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "s3_fs_open"); return rc; } @@ -153,7 +153,7 @@ s3_fs_list(DatalakeFileSystem fs, const char *prefix, char ***names_out, else rc = handle->impl->List(prefix, names_out, nnames_out); } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "s3_fs_list"); return rc; } @@ -173,7 +173,7 @@ s3_file_open(DatalakeFileSystem fs, const char *path, DatalakeFileMode mode, else rc = handle->impl->OpenFile(path, mode, file_out); } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "s3_file_open"); return rc; } @@ -194,7 +194,7 @@ s3_file_read(DatalakeFile file, void *buffer, int64_t length, int64_t *nread) rc = DL_ERR_NOT_SUPPORTED; } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "s3_file_read"); return rc; } @@ -212,7 +212,7 @@ s3_file_write(DatalakeFile file, const void *buffer, int64_t length) rc = DL_ERR_NOT_SUPPORTED; } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "s3_file_write"); return rc; } @@ -228,7 +228,7 @@ s3_file_close(DatalakeFile file) rc = DL_ERR_NOT_SUPPORTED; } - DL_ABI_GUARD_END(rc); + DL_ABI_GUARD_END(rc, "s3_file_close"); return rc; } diff --git a/contrib/datalake_fdw/src/format/arrow_builder.cpp b/contrib/datalake_fdw/src/format/arrow_builder.cpp new file mode 100644 index 00000000000..3dd68f4006a --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_builder.cpp @@ -0,0 +1,341 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_builder.cpp + * Accumulation of PostgreSQL tuples into an Arrow batch. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_builder.cpp + * + *------------------------------------------------------------------------- + */ + +#include +#include + +#include +#include + +#include "format/arrow_support.h" + +#include "common/dl_resource.h" +#include "common/dl_wrappers.h" +#include "format/arrow_builder.h" + +extern "C" +{ +#include "catalog/pg_type.h" +#include "utils/date.h" +#include "utils/timestamp.h" +#include "varatt.h" +} + +/* + * PostgreSQL counts from 2000-01-01 and Arrow from 1970-01-01. Everything + * below that touches a date or a timestamp shifts by this, and getting the sign + * wrong is a 30-year error that no round trip through our own code would + * notice -- both halves would agree. It is written once, here. + */ +#define DL_EPOCH_DELTA_DAYS ((int32) (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE)) +#define DL_EPOCH_DELTA_USECS (((int64) DL_EPOCH_DELTA_DAYS) * USECS_PER_DAY) + +struct DlArrowBuilderData +{ + std::shared_ptr schema; + std::vector types; + std::vector> builders; + int64_t nrows; +}; + +/* + * One value into the builder for its column. The Datum has already been + * detoasted by the caller, so nothing here allocates and nothing here can + * raise. + */ +static arrow::Status +dl_append_datum(arrow::ArrayBuilder *builder, Oid atttypid, Datum value) +{ + switch (atttypid) + { + case BOOLOID: + return static_cast(builder) + ->Append(DatumGetBool(value)); + case INT2OID: + return static_cast(builder) + ->Append(DatumGetInt16(value)); + case INT4OID: + return static_cast(builder) + ->Append(DatumGetInt32(value)); + case INT8OID: + return static_cast(builder) + ->Append(DatumGetInt64(value)); + case FLOAT4OID: + return static_cast(builder) + ->Append(DatumGetFloat4(value)); + case FLOAT8OID: + return static_cast(builder) + ->Append(DatumGetFloat8(value)); + + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + { + struct varlena *v = (struct varlena *) DatumGetPointer(value); + + return static_cast(builder) + ->Append(VARDATA_ANY(v), VARSIZE_ANY_EXHDR(v)); + } + + case BYTEAOID: + { + struct varlena *v = (struct varlena *) DatumGetPointer(value); + + return static_cast(builder) + ->Append(VARDATA_ANY(v), VARSIZE_ANY_EXHDR(v)); + } + + case DATEOID: + { + DateADT date = DatumGetDateADT(value); + + /* + * Parquet has no way to say "infinity", and writing the + * sentinel would hand the next reader a date 5.8 million years + * out as if it were a real one. + */ + if (DATE_NOT_FINITE(date)) + return arrow::Status::NotImplemented( + "an infinite date cannot be written to a data file"); + + return static_cast(builder) + ->Append(date + DL_EPOCH_DELTA_DAYS); + } + + case TIMESTAMPOID: + case TIMESTAMPTZOID: + { + Timestamp ts = DatumGetTimestamp(value); + + if (TIMESTAMP_NOT_FINITE(ts)) + return arrow::Status::NotImplemented( + "an infinite timestamp cannot be written to a data file"); + + /* + * PostgreSQL's range runs about 34 years past the last instant + * Arrow can hold in microseconds from 1970, so the shift below + * is not always representable. Without this the addition wraps + * -- quietly, because the build sets -fwrapv -- and a year + * 294250 timestamp is written as one 292000 years before the + * epoch, with the write reporting success. The read side has + * the mirror of this guard, and neither can stand in for the + * other: a round trip through both would agree. + */ + if (ts > PG_INT64_MAX - DL_EPOCH_DELTA_USECS) + return arrow::Status::Invalid( + "timestamp is too far in the future to be written to a " + "data file"); + + return static_cast(builder) + ->Append(ts + DL_EPOCH_DELTA_USECS); + } + + default: + + /* + * Unreachable until a new type is added to one of the two + * switches and not the other, which is why it names the OID. A + * number and not a name: resolving one means a catalog lookup, and + * nothing on this side of the ABI may allocate or raise. + */ + return arrow::Status::NotImplemented( + "no Arrow type is mapped for PostgreSQL type OID ", atttypid); + } +} + +/* What the resource owner calls if nothing else did. */ +extern "C" void +dl_arrow_builder_release(void *arg) +{ + DL_CLEANUP_GUARD_BEGIN + { + delete static_cast(arg); + } + DL_CLEANUP_GUARD_END; +} + +extern "C" DlErrCode +dl_arrow_builder_open(void *tupdesc_arg, DlArrowBuilder *out) +{ + DlErrCode result = DL_OK; + + if (out == NULL) + return DL_ARG_ERROR("open_builder"); + *out = NULL; + + if (tupdesc_arg == NULL) + return DL_ARG_ERROR("open_builder"); + + DL_ABI_GUARD_BEGIN + { + TupleDesc tupdesc = (TupleDesc) tupdesc_arg; + std::unique_ptr builder(new DlArrowBuilderData()); + + builder->schema = DlArrowSchemaFromTupleDesc(tupdesc); + if (builder->schema == nullptr) + return DL_ERR_NOT_SUPPORTED; /* detail already recorded */ + + builder->nrows = 0; + builder->types.reserve(tupdesc->natts); + builder->builders.reserve(tupdesc->natts); + + for (int i = 0; i < tupdesc->natts; i++) + { + std::unique_ptr column; + arrow::Status status = arrow::MakeBuilder(arrow::default_memory_pool(), + builder->schema->field(i)->type(), + &column); + + if (!status.ok()) + return DlArrowStatus(status, "create an Arrow array builder"); + + builder->types.push_back(TupleDescAttr(tupdesc, i)->atttypid); + builder->builders.push_back(std::move(column)); + } + + /* + * The buffers behind the builder come from Arrow's allocator, which + * transaction abort knows nothing about. Last thing that may fail. + */ + if (!dl_resource_remember(dl_arrow_builder_release, builder.get())) + { + dl_error_set(DL_ERR_INTERNAL, "open_builder", NULL, + "could not record the batch builder for cleanup"); + return DL_ERR_INTERNAL; + } + + *out = builder.release(); + } + DL_ABI_GUARD_END(result, "open_builder"); + + return result; +} + +extern "C" DlErrCode +dl_arrow_builder_append(DlArrowBuilder builder, const Datum *values, + const bool *nulls, int nvalues) +{ + DlErrCode result = DL_OK; + + if (builder == NULL || values == NULL || nulls == NULL) + return DL_ARG_ERROR("append_row"); + + if (nvalues != (int) builder->builders.size()) + { + dl_error_set(DL_ERR_INTERNAL, "append an Arrow row", NULL, + "the row has a different number of columns than the batch"); + return DL_ERR_INTERNAL; + } + + DL_ABI_GUARD_BEGIN + { + for (int i = 0; i < nvalues; i++) + { + arrow::Status status = nulls[i] + ? builder->builders[i]->AppendNull() + : dl_append_datum(builder->builders[i].get(), builder->types[i], + values[i]); + + if (!status.ok()) + return DlArrowStatus(status, "append a value to an Arrow array"); + } + + builder->nrows++; + } + DL_ABI_GUARD_END(result, "append_row"); + + return result; +} + +extern "C" int64_t +dl_arrow_builder_nrows(DlArrowBuilder builder) +{ + return builder == NULL ? 0 : builder->nrows; +} + +extern "C" DlErrCode +dl_arrow_builder_flush(DlArrowBuilder builder, struct ArrowArray *out) +{ + DlErrCode result = DL_OK; + + if (builder == NULL || out == NULL) + return DL_ARG_ERROR("build_batch"); + + DL_ABI_GUARD_BEGIN + { + std::vector> columns; + + columns.reserve(builder->builders.size()); + + for (auto &column : builder->builders) + { + std::shared_ptr array; + + /* Finish() also resets the builder, so the next batch starts here. */ + arrow::Status status = column->Finish(&array); + + if (!status.ok()) + return DlArrowStatus(status, "finish an Arrow array"); + + columns.push_back(std::move(array)); + } + + std::shared_ptr batch = + arrow::RecordBatch::Make(builder->schema, builder->nrows, columns); + + /* + * The schema travels with the writer, which was opened from the same + * descriptor, so exporting it with every batch would be a copy nobody + * reads. + */ + arrow::Status status = arrow::ExportRecordBatch(*batch, out, nullptr); + + if (!status.ok()) + return DlArrowStatus(status, "export an Arrow batch"); + + builder->nrows = 0; + } + DL_ABI_GUARD_END(result, "build_batch"); + + return result; +} + +extern "C" void +dl_arrow_builder_close(DlArrowBuilder *builder) +{ + if (builder == NULL || *builder == NULL) + return; + + /* Cleared first; see the note in parquet_reader_close(). */ + DlArrowBuilderData *impl = *builder; + + *builder = NULL; + dl_resource_forget(dl_arrow_builder_release, impl); + + dl_arrow_builder_release(impl); +} diff --git a/contrib/datalake_fdw/src/format/arrow_builder.h b/contrib/datalake_fdw/src/format/arrow_builder.h new file mode 100644 index 00000000000..d829199beb2 --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_builder.h @@ -0,0 +1,95 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_builder.h + * Accumulation of PostgreSQL tuples into an Arrow batch. + * + * This is the write half of the boundary the format layer is built on: rows + * arrive one at a time from an executor, and a data file wants them a column at + * a time. Nothing above this knows Arrow, and nothing here knows which format + * the batch ends up in. + * + * postgres.h must be included before this header; the Datum in the append + * signature is the whole reason a tuple can be handed over without copying it + * first. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_builder.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_ARROW_BUILDER_H +#define DL_ARROW_BUILDER_H + +#include "common/dl_err.h" +#include "format/format.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +typedef struct DlArrowBuilderData *DlArrowBuilder; + +/* + * How many rows to accumulate before flushing is the caller's decision, not the + * builder's -- the caller is the one that also has to decide when to roll to a + * new file. iceberg.batch_rows is the setting they take it from. + */ + +/* + * `tupdesc` is a TupleDesc. It is taken as void * so that this header stays + * usable from the C++ side without dragging PostgreSQL's headers into it in a + * particular order; the type is checked by the only two callers there are. + * + * The descriptor has to outlive the builder, which is no constraint in + * practice: the tuples being appended come from it. + */ +extern DlErrCode dl_arrow_builder_open(void *tupdesc, DlArrowBuilder *out); + +/* + * Appends one row. Varlena values must already be detoasted -- this runs on + * the C++ side, where a PostgreSQL error would unwind through frames that + * cannot handle one, so it does not call anything that allocates. + */ +extern DlErrCode dl_arrow_builder_append(DlArrowBuilder builder, + const Datum *values, + const bool *nulls, + int nvalues); + +/* Rows accumulated since the last flush. */ +extern int64_t dl_arrow_builder_nrows(DlArrowBuilder builder); + +/* + * Hands over what has accumulated and starts a new batch. The caller owns the + * exported array and releases it -- or gives it to a writer, which consumes it. + * Flushing nothing is not an error and produces an empty batch. + */ +extern DlErrCode dl_arrow_builder_flush(DlArrowBuilder builder, + struct ArrowArray *out); + +/* Cleanup entry point: releases the builder and clears the caller's handle. */ +extern void dl_arrow_builder_close(DlArrowBuilder *builder); + +#ifdef __cplusplus +} +#endif + +#endif /* DL_ARROW_BUILDER_H */ diff --git a/contrib/datalake_fdw/src/format/arrow_decode.c b/contrib/datalake_fdw/src/format/arrow_decode.c new file mode 100644 index 00000000000..6574eeb2a9b --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_decode.c @@ -0,0 +1,355 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_decode.c + * PostgreSQL values out of an Arrow batch. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_decode.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include + +#include "catalog/pg_type.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/fmgrprotos.h" +#include "utils/timestamp.h" +#include "varatt.h" + +#include "format/arrow_decode.h" + +/* + * The same shift as in arrow_builder.cpp, in the other direction: PostgreSQL + * counts from 2000-01-01 and Arrow from 1970-01-01. + */ +#define DL_EPOCH_DELTA_DAYS ((int32) (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE)) +#define DL_EPOCH_DELTA_USECS (((int64) DL_EPOCH_DELTA_DAYS) * USECS_PER_DAY) + +/* + * Arrow spells its types as a short string. Only the ones a column of ours can + * be stored as are listed; anything else is a file we did not write, or one + * written by a version that knows more types than this one. + */ +#define DL_ARROW_FORMAT_BOOL "b" +#define DL_ARROW_FORMAT_INT16 "s" +#define DL_ARROW_FORMAT_INT32 "i" +#define DL_ARROW_FORMAT_INT64 "l" +#define DL_ARROW_FORMAT_FLOAT32 "f" +#define DL_ARROW_FORMAT_FLOAT64 "g" +#define DL_ARROW_FORMAT_UTF8 "u" +#define DL_ARROW_FORMAT_BINARY "z" +#define DL_ARROW_FORMAT_DATE32 "tdD" + +/* A timestamp is "tsu:" followed by the time zone, which may be empty. */ +#define DL_ARROW_FORMAT_TIMESTAMP_US "tsu:" + +static DlErrCode +dl_arrow_decode_refuse(const char *arrow_format, Oid atttypid) +{ + char message[256]; + + snprintf(message, sizeof(message), + "a column stored as Arrow type \"%s\" cannot be read as %s", + arrow_format == NULL ? "" : arrow_format, + format_type_be(atttypid)); + + dl_error_set(DL_ERR_NOT_SUPPORTED, "decode an Arrow column", NULL, message); + return DL_ERR_NOT_SUPPORTED; +} + +DlErrCode +dl_arrow_decode_check(const struct ArrowSchema *field, Oid atttypid) +{ + const char *format; + const char *expected; + + if (field == NULL || field->format == NULL) + { + dl_error_set(DL_ERR_INTERNAL, "decode an Arrow column", NULL, + "the batch has a column with no type"); + return DL_ERR_INTERNAL; + } + + format = field->format; + + switch (atttypid) + { + case BOOLOID: + expected = DL_ARROW_FORMAT_BOOL; + break; + case INT2OID: + expected = DL_ARROW_FORMAT_INT16; + break; + case INT4OID: + expected = DL_ARROW_FORMAT_INT32; + break; + case INT8OID: + expected = DL_ARROW_FORMAT_INT64; + break; + case FLOAT4OID: + expected = DL_ARROW_FORMAT_FLOAT32; + break; + case FLOAT8OID: + expected = DL_ARROW_FORMAT_FLOAT64; + break; + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + expected = DL_ARROW_FORMAT_UTF8; + break; + case BYTEAOID: + expected = DL_ARROW_FORMAT_BINARY; + break; + case DATEOID: + expected = DL_ARROW_FORMAT_DATE32; + break; + + case TIMESTAMPOID: + case TIMESTAMPTZOID: + { + const char *zone; + size_t prefix_len = strlen(DL_ARROW_FORMAT_TIMESTAMP_US); + + if (strncmp(format, DL_ARROW_FORMAT_TIMESTAMP_US, prefix_len) != 0) + return dl_arrow_decode_refuse(format, atttypid); + + /* + * Arrow stores a zoned timestamp as the instant in UTC and + * keeps the zone only to display it, so which zone the file + * names does not change the value -- but whether it names one + * at all is the difference between the two PostgreSQL types, + * and reading one as the other would shift every value by the + * session's offset from UTC. + */ + zone = format + prefix_len; + if ((zone[0] != '\0') != (atttypid == TIMESTAMPTZOID)) + return dl_arrow_decode_refuse(format, atttypid); + + return DL_OK; + } + + default: + return dl_arrow_decode_refuse(format, atttypid); + } + + if (strcmp(format, expected) != 0) + return dl_arrow_decode_refuse(format, atttypid); + + return DL_OK; +} + +/* + * Arrow keeps the validity bitmap in the first buffer, and a column with no + * nulls may leave it out entirely. Bit set means present. + */ +static bool +dl_arrow_value_is_null(const struct ArrowArray *column, int64_t row) +{ + const uint8 *validity; + int64 index; + + if (column->n_buffers < 1) + return false; + + validity = (const uint8 *) column->buffers[0]; + if (validity == NULL) + return false; + + index = column->offset + row; + return (validity[index >> 3] & (1 << (index & 7))) == 0; +} + +/* The values buffer of a fixed-width column, already advanced past the offset. */ +#define DL_ARROW_VALUES(column, type) \ + (((const type *) (column)->buffers[1]) + (column)->offset) + +static DlErrCode +dl_arrow_out_of_range(const char *what) +{ + char message[128]; + + snprintf(message, sizeof(message), + "the file holds a %s outside the range PostgreSQL can represent", + what); + + dl_error_set(DL_ERR_INVALID_OPTION, "decode an Arrow column", NULL, message); + return DL_ERR_INVALID_OPTION; +} + +/* + * A variable-length value: an offsets buffer of int32 and one run of bytes. + * Both text and bytea are laid out this way, and differ only in the header the + * copy gets. + */ +static void +dl_arrow_varlen(const struct ArrowArray *column, int64_t row, + const char **data, int32 *length) +{ + const int32 *offsets = DL_ARROW_VALUES(column, int32); + const char *bytes = (const char *) column->buffers[2]; + + *data = bytes + offsets[row]; + *length = offsets[row + 1] - offsets[row]; +} + +DlErrCode +dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, + Oid atttypid, int32 atttypmod, + Datum *value, bool *isnull) +{ + *value = (Datum) 0; + *isnull = true; + + if (row < 0 || row >= column->length) + { + dl_error_set(DL_ERR_INTERNAL, "decode an Arrow column", NULL, + "a row was asked for past the end of the batch"); + return DL_ERR_INTERNAL; + } + + if (dl_arrow_value_is_null(column, row)) + return DL_OK; + + *isnull = false; + + switch (atttypid) + { + case BOOLOID: + { + /* Booleans are a bitmap of their own, not a byte per value. */ + const uint8 *bits = (const uint8 *) column->buffers[1]; + int64 index = column->offset + row; + + *value = BoolGetDatum((bits[index >> 3] & (1 << (index & 7))) != 0); + return DL_OK; + } + + case INT2OID: + *value = Int16GetDatum(DL_ARROW_VALUES(column, int16)[row]); + return DL_OK; + case INT4OID: + *value = Int32GetDatum(DL_ARROW_VALUES(column, int32)[row]); + return DL_OK; + case INT8OID: + *value = Int64GetDatum(DL_ARROW_VALUES(column, int64)[row]); + return DL_OK; + case FLOAT4OID: + *value = Float4GetDatum(DL_ARROW_VALUES(column, float)[row]); + return DL_OK; + case FLOAT8OID: + *value = Float8GetDatum(DL_ARROW_VALUES(column, double)[row]); + return DL_OK; + + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + { + const char *data; + int32 length; + + dl_arrow_varlen(column, row, &data, &length); + *value = PointerGetDatum(cstring_to_text_with_len(data, length)); + + /* + * The file records the bytes and nothing about the length the + * column was declared with, so the value is put through the + * same coercion an inserted one would be: char(n) comes back + * padded to n, and a value too long for a varchar(n) is an + * error rather than something the executor has to meet later. + */ + if (atttypmod >= 0 && atttypid == BPCHAROID) + *value = DirectFunctionCall3(bpchar, *value, + Int32GetDatum(atttypmod), + BoolGetDatum(false)); + else if (atttypmod >= 0 && atttypid == VARCHAROID) + *value = DirectFunctionCall3(varchar, *value, + Int32GetDatum(atttypmod), + BoolGetDatum(false)); + + return DL_OK; + } + + case BYTEAOID: + { + const char *data; + int32 length; + bytea *result; + + dl_arrow_varlen(column, row, &data, &length); + result = (bytea *) palloc(VARHDRSZ + length); + SET_VARSIZE(result, VARHDRSZ + length); + memcpy(VARDATA(result), data, length); + *value = PointerGetDatum(result); + return DL_OK; + } + + case DATEOID: + { + int32 days = DL_ARROW_VALUES(column, int32)[row]; + DateADT date; + + /* + * Shifting the epoch is a subtraction that can leave the range + * of the type it lands in, so the guard has to come first: by + * the time an overflowed value could be checked it is already + * a different, plausible-looking date. + */ + if (days < DATETIME_MIN_JULIAN - UNIX_EPOCH_JDATE) + return dl_arrow_out_of_range("date"); + + date = days - DL_EPOCH_DELTA_DAYS; + if (!IS_VALID_DATE(date)) + return dl_arrow_out_of_range("date"); + + *value = DateADTGetDatum(date); + return DL_OK; + } + + case TIMESTAMPOID: + case TIMESTAMPTZOID: + { + int64 micros = DL_ARROW_VALUES(column, int64)[row]; + Timestamp ts; + + if (micros < MIN_TIMESTAMP + DL_EPOCH_DELTA_USECS) + return dl_arrow_out_of_range("timestamp"); + + ts = micros - DL_EPOCH_DELTA_USECS; + if (!IS_VALID_TIMESTAMP(ts)) + return dl_arrow_out_of_range("timestamp"); + + *value = TimestampGetDatum(ts); + return DL_OK; + } + + default: + + /* + * Unreachable: dl_arrow_decode_check() refused every type this + * switch does not list. + */ + *isnull = true; + return dl_arrow_decode_refuse(NULL, atttypid); + } +} diff --git a/contrib/datalake_fdw/src/format/arrow_decode.h b/contrib/datalake_fdw/src/format/arrow_decode.h new file mode 100644 index 00000000000..184a1edff2a --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_decode.h @@ -0,0 +1,75 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_decode.h + * PostgreSQL values out of an Arrow batch. + * + * The read half of the boundary the format layer is built on, and the mirror of + * arrow_builder.h. This side is C: turning a column into Datums means + * allocating text and bytea, an allocation can fail, and a failure in + * PostgreSQL unwinds with longjmp -- which is safe here and would not be if it + * had to pass through C++ frames on the way out. + * + * It reads the buffers of the Arrow C data interface directly rather than + * handing them back to Arrow, which keeps the read path free of C++ and makes + * it a real check on what our own writer exports. + * + * postgres.h must be included before this header. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_decode.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_ARROW_DECODE_H +#define DL_ARROW_DECODE_H + +#include "common/dl_err.h" +#include "format/format.h" + +/* + * Whether a column of this Arrow type can be read as this PostgreSQL type. + * Called once per column per batch: the answer depends only on the schema, and + * checking it per value would be the same answer several million times. + * + * `field` is one child of the batch's schema. + */ +extern DlErrCode dl_arrow_decode_check(const struct ArrowSchema *field, + Oid atttypid); + +/* + * One value. Only valid for a column dl_arrow_decode_check() accepted, which + * is what lets this trust the buffer layout instead of re-deriving it. + * + * Values that point at memory -- text, bytea -- are copied into the current + * memory context, because the batch is released long before the tuples built + * from it are done with. + * + * `atttypmod` is the modifier the column was declared with, or -1. A file this + * module did not write has no idea what it was, so a char(n) in it need not be + * padded to n and a varchar(n) need not be within n; without applying it, a + * value that breaks the type's own rules would reach the executor. + */ +extern DlErrCode dl_arrow_decode_value(const struct ArrowArray *column, + int64_t row, Oid atttypid, + int32 atttypmod, + Datum *value, bool *isnull); + +#endif /* DL_ARROW_DECODE_H */ diff --git a/contrib/datalake_fdw/src/format/arrow_support.cpp b/contrib/datalake_fdw/src/format/arrow_support.cpp new file mode 100644 index 00000000000..8addade807f --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_support.cpp @@ -0,0 +1,179 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_support.cpp + * The type mapping and the error translation shared by the Arrow-facing + * parts of this module. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_support.cpp + * + *------------------------------------------------------------------------- + */ + +/* + * Arrow's headers come first throughout this module: PostgreSQL's c.h defines + * Abs, Min and Max as macros, and a template header has no way to defend + * itself against them. + */ +#include +#include + +#include + +#include "format/arrow_support.h" + +extern "C" +{ +#include "catalog/pg_type.h" +} + +DlErrCode +DlArrowStatus(const arrow::Status &status, const char *operation) +{ + DlErrCode code; + + if (status.ok()) + return DL_OK; + + switch (status.code()) + { + case arrow::StatusCode::IOError: + code = DL_ERR_IO; + break; + case arrow::StatusCode::NotImplemented: + code = DL_ERR_NOT_SUPPORTED; + break; + case arrow::StatusCode::Invalid: + case arrow::StatusCode::TypeError: + case arrow::StatusCode::KeyError: + code = DL_ERR_INVALID_OPTION; + break; + default: + code = DL_ERR_INTERNAL; + break; + } + + dl_error_set(code, operation, arrow::Status::CodeAsString(status.code()).c_str(), + status.message().c_str()); + return code; +} + +std::shared_ptr +DlArrowTypeForPgType(Oid atttypid) +{ + switch (atttypid) + { + case BOOLOID: + return arrow::boolean(); + case INT2OID: + return arrow::int16(); + case INT4OID: + return arrow::int32(); + case INT8OID: + return arrow::int64(); + case FLOAT4OID: + return arrow::float32(); + case FLOAT8OID: + return arrow::float64(); + + /* + * All three of PostgreSQL's string types are one Arrow type: the + * length limit is a constraint PostgreSQL enforces before a value + * reaches us, and Parquet has nowhere to record it. A char(n) + * value arrives already padded, so what is written is what + * PostgreSQL stores. + */ + case TEXTOID: + case VARCHAROID: + case BPCHAROID: + return arrow::utf8(); + + case BYTEAOID: + return arrow::binary(); + case DATEOID: + return arrow::date32(); + + /* + * PostgreSQL keeps both timestamp types in microseconds, so + * microseconds is the unit that loses nothing. timestamptz is a + * point in time held in UTC, which is exactly what an Arrow + * timestamp with a "UTC" zone means; timestamp without time zone + * has no zone, and Arrow says that by leaving it empty. + */ + case TIMESTAMPOID: + return arrow::timestamp(arrow::TimeUnit::MICRO); + case TIMESTAMPTZOID: + return arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + + default: + return nullptr; + } +} + +std::shared_ptr +DlArrowSchemaFromTupleDesc(TupleDesc tupdesc) +{ + std::vector> fields; + + fields.reserve(tupdesc->natts); + + for (int i = 0; i < tupdesc->natts; i++) + { + Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + + /* + * A dropped column has no type to write and no name worth recording. + * Leaving a placeholder in the file would keep column positions + * aligned, but nothing reads such a file yet, so refusing is the + * answer that cannot be silently wrong. + */ + if (attr->attisdropped) + { + dl_error_set(DL_ERR_NOT_SUPPORTED, "arrow schema", nullptr, + "a dropped column cannot be written to a data file"); + return nullptr; + } + + std::shared_ptr type = DlArrowTypeForPgType(attr->atttypid); + + if (type == nullptr) + { + std::string message = std::string("column \"") + + NameStr(attr->attname) + "\" has a type that lake tables " + "cannot store yet"; + + dl_error_set(DL_ERR_NOT_SUPPORTED, "arrow schema", nullptr, + message.c_str()); + return nullptr; + } + + /* + * Every field is nullable, including one PostgreSQL marked NOT NULL. + * Recording it as required would buy nothing -- PostgreSQL has already + * rejected the nulls before a tuple reaches this layer -- and would + * turn any later relaxation of the constraint into a write failure + * against files already on disk. + */ + fields.push_back(arrow::field(NameStr(attr->attname), type, + /* nullable */ true)); + } + + return arrow::schema(fields); +} diff --git a/contrib/datalake_fdw/src/format/arrow_support.h b/contrib/datalake_fdw/src/format/arrow_support.h new file mode 100644 index 00000000000..d79b5934ddc --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_support.h @@ -0,0 +1,75 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_support.h + * What every Arrow-facing translation unit in this module needs: how a + * PostgreSQL column type is stored, and how an Arrow failure is reported. + * + * The type mapping is in one place because the writer, the builder that feeds + * it and the reader that decodes what comes back all have to agree on it, and a + * disagreement between them would show up as wrong values rather than as an + * error. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_support.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_ARROW_SUPPORT_H +#define DL_ARROW_SUPPORT_H + +#include + +#include +#include +#include + +#include "common/dl_err.h" +#include "common/dl_pg_api.h" + +extern "C" +{ +#include "access/tupdesc.h" +} + +/* + * Turns an Arrow status into this module's error code, recording what Arrow + * said -- its own class and message are the only thing that makes a failure in + * a third-party library diagnosable, and the code alone throws them away. + * `operation` names what was being attempted. A successful status records + * nothing and returns DL_OK, so call sites can wrap every Arrow call. + */ +extern DlErrCode DlArrowStatus(const arrow::Status &status, const char *operation); + +/* + * The Arrow type a column of this PostgreSQL type is stored as, or a null + * pointer when the type has no mapping yet. Callers report the refusal + * themselves, because only they know which column it was about. + */ +extern std::shared_ptr DlArrowTypeForPgType(Oid atttypid); + +/* + * The whole descriptor. Returns a null pointer and records which column was + * the problem in the error detail: a type with no mapping, or a dropped column, + * which has no type to write and which nothing reads a file for yet. + */ +extern std::shared_ptr DlArrowSchemaFromTupleDesc(TupleDesc tupdesc); + +#endif /* DL_ARROW_SUPPORT_H */ diff --git a/contrib/datalake_fdw/src/format/format.h b/contrib/datalake_fdw/src/format/format.h index 003101ae550..db793eb9689 100644 --- a/contrib/datalake_fdw/src/format/format.h +++ b/contrib/datalake_fdw/src/format/format.h @@ -64,10 +64,44 @@ struct ArrowArray { }; #endif /* ARROW_C_DATA_INTERFACE */ -typedef struct Fragment Fragment; /* opaque in skeleton */ -typedef struct ProjectionSet ProjectionSet; +/* + * One unit of read work. A fragment is a range of row groups rather than a + * whole file, because that is the granularity a scan can be divided at: several + * segments can then read one large file at once, which file-at-a-time + * assignment cannot express. + */ +typedef struct Fragment +{ + const char *path; + int first_row_group; /* 0-based */ + int n_row_groups; /* 0 == to the end of the file */ +} Fragment; + +/* + * The columns to materialise, as 0-based indexes into the file schema. A NULL + * set, or one with no columns, means every column: "read nothing" is not a + * projection anyone asks for, so it is not worth a second way to say "all". + */ +typedef struct ProjectionSet +{ + const int *columns; + int ncolumns; +} ProjectionSet; + +/* + * A writer holds a whole row group before it can write one, so this is a bound + * on memory as much as on the file's shape. It is Parquet's own default + * maximum, which is what makes it a sane ceiling for any format. + */ +#define DL_MAX_ROW_GROUP_ROWS (1024 * 1024) + +typedef struct WriterOptions +{ + const char *compression; /* format-defined name; NULL for the default */ + int64_t row_group_size; /* rows per row group; 0 for the default */ +} WriterOptions; + typedef struct RowGroupFilterSet RowGroupFilterSet; -typedef struct WriterOptions WriterOptions; typedef struct FileMeta FileMeta; typedef struct DeleteFileSet DeleteFileSet; @@ -75,28 +109,49 @@ typedef struct DeleteFileSet DeleteFileSet; * No global slots or trampolines, ever. */ typedef struct FormatReader FormatReader; typedef struct FormatReaderOps { - /* Each batch yields ArrowArray+ArrowSchema; last column is a hidden int64 file-row - * ordinal (for MoR positional deletes). */ + /* Each batch yields ArrowArray+ArrowSchema. A hidden trailing int64 column + * carrying the file-row ordinal is what merge-on-read positional deletes + * will match against; it arrives with them, so a batch is the projected + * columns and nothing else for now. */ DlErrCode (*next_batch)(FormatReader *, struct ArrowArray *out, struct ArrowSchema *schema, bool *eof); - void (*close)(FormatReader *); /* void cleanup ABI: noexcept, idempotent, never ereport */ + /* void cleanup ABI: noexcept, never ereport. Takes the caller's handle so + * that it can clear it -- these run on the resource-owner path during + * abort, where the same cleanup can be reached twice, and a second call + * has to find nothing left rather than a freed reader. */ + void (*close)(FormatReader **); } FormatReaderOps; struct FormatReader { const FormatReaderOps *ops; void *impl; }; typedef struct FormatWriter FormatWriter; typedef struct FormatWriterOps { - DlErrCode (*write_batch)(FormatWriter *, struct ArrowArray *batch); /* success == consumed */ + /* + * The batch is consumed whether or not the write succeeds: an + * implementation hands it to a library that takes ownership at the call, + * and there is no point at which it could hand it back. The caller is + * left with a released ArrowArray either way. + */ + DlErrCode (*write_batch)(FormatWriter *, struct ArrowArray *batch); /* Rolling support: actual bytes encoded into the sink so far. Valid to query after a * successful write_batch; on failure returns an error code and *out is invalid. * The write.c orchestration layer rolls files (finish -> new open_writer) when this - * reaches the soft target; overshoot of at most one batch is allowed. */ + * reaches the soft target. A format writes in units it cannot split -- a Parquet + * row group is one -- and what has not been written is not counted, so the target + * is overshot by at most one such unit. */ DlErrCode (*bytes_written)(FormatWriter *, int64_t *out); - DlErrCode (*finish)(FormatWriter *, FileMeta **meta); /* reportable close-time errors - * surface ONLY here */ - void (*abort)(FormatWriter *); /* void cleanup ABI: noexcept, idempotent, never ereport */ + /* Reportable close-time errors surface ONLY here. The writer is consumed + * and the caller's handle cleared whether or not it succeeds: a file whose + * footer could not be written is not one anything can retry against. */ + DlErrCode (*finish)(FormatWriter **, FileMeta **meta); + /* void cleanup ABI, as for close() above: discards the file being written + * and clears the caller's handle. */ + void (*abort)(FormatWriter **); } FormatWriterOps; struct FormatWriter { const FormatWriterOps *ops; void *impl; }; +/* Bumped when an existing field changes meaning; appending does not need it. */ +#define DL_FORMAT_ABI_VERSION 1 + typedef struct FormatRoutine { uint32_t abi_version, struct_size; /* same prefix-compat semantics as meta engine */ const char *name; /* "parquet" */ diff --git a/contrib/datalake_fdw/src/format/format_registry.c b/contrib/datalake_fdw/src/format/format_registry.c index 83370a51f64..44bf9ec0732 100644 --- a/contrib/datalake_fdw/src/format/format_registry.c +++ b/contrib/datalake_fdw/src/format/format_registry.c @@ -26,15 +26,35 @@ *------------------------------------------------------------------------- */ +#include #include +#include +#include "common/dl_err.h" #include "format/format.h" +#include "format/parquet/parquet_format.h" -/* No formats in the skeleton; parquet lands in PR-3/4. Callers must treat - * NULL as not-supported. */ +/* + * Parquet is the only format so far. A name that reaches here came from a + * table option, so an unknown one is an ordinary mistake and the caller has to + * be able to say which name it was -- returning a bare NULL would leave every + * caller to write that message again, and get it wrong differently. The name + * goes into the error detail, so a caller that reports DL_ERR_NOT_SUPPORTED + * gets it without knowing this function exists. + */ const FormatRoutine * GetFormatRoutine(const char *format) { + char message[128]; + + if (format != NULL && strcmp(format, "parquet") == 0) + return GetParquetFormatRoutine(); + + snprintf(message, sizeof(message), + "\"%s\" is not a data file format this build can read or write", + format == NULL ? "" : format); + dl_error_set(DL_ERR_NOT_SUPPORTED, "get_format", NULL, message); + return NULL; } diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_format.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_format.cpp new file mode 100644 index 00000000000..ac094389ee9 --- /dev/null +++ b/contrib/datalake_fdw/src/format/parquet/parquet_format.cpp @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parquet_format.cpp + * Parquet as a format this module can read and write. + * + * Parquet is reached through Arrow rather than through libparquet on its own, + * because libparquet is written in terms of Arrow's types: linking it already + * links Arrow, and going around Arrow would mean re-deriving the definition and + * repetition levels, the four ways a decimal can be stored, and the timestamp + * unit rules that arrow::parquet already gets right. + * + * Nothing here reads or writes anything but a local file yet. The storage + * facade in common/file_system_wrapper.h is where object storage arrives, as an + * arrow::io::RandomAccessFile over it; parquet_read.cpp and parquet_write.cpp + * are the only files that have to change when it does. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/parquet/parquet_format.cpp + * + *------------------------------------------------------------------------- + */ + +#include "format/parquet/parquet_format.h" +#include "format/parquet/parquet_internal.h" + +static const FormatRoutine parquet_format_routine = { + DL_FORMAT_ABI_VERSION, + sizeof(FormatRoutine), + "parquet", + parquet_open_reader, + parquet_open_writer +}; + +extern "C" const FormatRoutine * +GetParquetFormatRoutine(void) +{ + return &parquet_format_routine; +} diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_format.h b/contrib/datalake_fdw/src/format/parquet/parquet_format.h new file mode 100644 index 00000000000..20f3ff788fa --- /dev/null +++ b/contrib/datalake_fdw/src/format/parquet/parquet_format.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parquet_format.h + * The Parquet reader and writer. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/parquet/parquet_format.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_PARQUET_FORMAT_H +#define DL_PARQUET_FORMAT_H + +#include "format/format.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +extern const FormatRoutine *GetParquetFormatRoutine(void); + +#ifdef __cplusplus +} +#endif + +#endif /* DL_PARQUET_FORMAT_H */ diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_internal.h b/contrib/datalake_fdw/src/format/parquet/parquet_internal.h new file mode 100644 index 00000000000..f7acbdd2f94 --- /dev/null +++ b/contrib/datalake_fdw/src/format/parquet/parquet_internal.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parquet_internal.h + * What the halves of the Parquet format say to each other. + * + * Reading and writing a Parquet file have nothing in common but the name of + * the format, so they are separate translation units; this is the only thing + * they share, and parquet_format.cpp is the only other file that needs it. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/parquet/parquet_internal.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_PARQUET_INTERNAL_H +#define DL_PARQUET_INTERNAL_H + +#include "format/format.h" + +extern DlErrCode parquet_open_reader(const Fragment *fragment, + const ProjectionSet *projection, + const RowGroupFilterSet *filters, + FormatReader **out); + +extern DlErrCode parquet_open_writer(const char *path, void *tupdesc, + const WriterOptions *options, + FormatWriter **out); + +#endif /* DL_PARQUET_INTERNAL_H */ diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp new file mode 100644 index 00000000000..9c245672fbe --- /dev/null +++ b/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp @@ -0,0 +1,280 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parquet_read.cpp + * Reading a Parquet file. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/parquet/parquet_read.cpp + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "format/arrow_support.h" + +#include "am_iceberg/pg_iceberg_guc.h" +#include "common/dl_resource.h" +#include "common/dl_wrappers.h" +#include "format/parquet/parquet_internal.h" + +struct ParquetReader +{ + FormatReader base; + std::shared_ptr file; + std::unique_ptr reader; + std::shared_ptr batches; +}; + +static DlErrCode +parquet_reader_next_batch(FormatReader *reader, struct ArrowArray *out, + struct ArrowSchema *schema, bool *eof) +{ + DlErrCode result = DL_OK; + + if (reader == NULL || out == NULL || eof == NULL) + return DL_ARG_ERROR("next_batch"); + + *eof = false; + + DL_ABI_GUARD_BEGIN + { + ParquetReader *impl = static_cast(reader->impl); + std::shared_ptr batch; + arrow::Status status = impl->batches->ReadNext(&batch); + + if (!status.ok()) + return DlArrowStatus(status, "read a Parquet batch"); + + if (batch == nullptr) + { + *eof = true; + return DL_OK; + } + + return DlArrowStatus(arrow::ExportRecordBatch(*batch, out, schema), + "export a Parquet batch"); + } + DL_ABI_GUARD_END(result, "next_batch"); + + return result; +} + +/* + * What the resource owner calls if nothing else did. C linkage because a C + * function pointer is what it is handed to. + */ +extern "C" void +parquet_reader_release(void *arg) +{ + DL_CLEANUP_GUARD_BEGIN + { + delete static_cast(arg); + } + DL_CLEANUP_GUARD_END; +} + +static void +parquet_reader_close(FormatReader **reader) +{ + if (reader == NULL || *reader == NULL) + return; + + /* + * Clear the caller's handle first. DL_CLEANUP_GUARD_END can elog(WARNING), + * and an escalation there would longjmp past the assignment -- leaving the + * caller holding a reader that has already been released, which is the + * thing taking the handle by address exists to prevent. + */ + ParquetReader *impl = static_cast((*reader)->impl); + + *reader = NULL; + dl_resource_forget(parquet_reader_release, impl); + + parquet_reader_release(impl); +} + +static const FormatReaderOps parquet_reader_ops = { + parquet_reader_next_batch, + parquet_reader_close +}; + +/* + * Which row groups this fragment covers. A fragment is a range rather than a + * whole file so that one large file can be read by several segments at once; + * an empty range is legal and reads nothing. + */ +static DlErrCode +parquet_row_groups(const Fragment *fragment, int total, + std::vector *row_groups) +{ + int first = fragment->first_row_group; + int count = fragment->n_row_groups; + + /* + * The last test is written as a subtraction because the addition it + * replaces overflows: first + INT_MAX wraps negative, passes the check, and + * the loop below then builds a two-billion-element vector out of a range + * that should have been rejected. + */ + if (first < 0 || first > total || count < 0 || count > total - first) + { + char message[160]; + + snprintf(message, sizeof(message), + "row groups %d..%d were asked for from a file that has %d", + first, count > 0 ? first + count - 1 : first, total); + dl_error_set(DL_ERR_INVALID_OPTION, "open a Parquet file", NULL, message); + return DL_ERR_INVALID_OPTION; + } + + if (count == 0) + count = total - first; + + for (int i = 0; i < count; i++) + row_groups->push_back(first + i); + + return DL_OK; +} + +DlErrCode +parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, + const RowGroupFilterSet *filters, FormatReader **out) +{ + DlErrCode result = DL_OK; + + if (out == NULL) + return DL_ARG_ERROR("open_reader"); + *out = NULL; + + if (fragment == NULL || fragment->path == NULL) + return DL_ARG_ERROR("open_reader"); + + /* + * Statistics-based row group pruning is not implemented. Accepting the + * filters and ignoring them would still give the right rows, so nothing + * would fail -- which is exactly why it is refused instead: a caller that + * believed the pruning had happened would have no way to find out. + */ + if (filters != NULL) + { + dl_error_set(DL_ERR_NOT_SUPPORTED, "open a Parquet file", NULL, + "row group filtering is not implemented yet"); + return DL_ERR_NOT_SUPPORTED; + } + + DL_ABI_GUARD_BEGIN + { + std::unique_ptr impl(new ParquetReader()); + arrow::MemoryPool *pool = arrow::default_memory_pool(); + std::vector row_groups; + std::vector columns; + DlErrCode rc; + + parquet::arrow::FileReaderBuilder builder; + parquet::ArrowReaderProperties properties; + + arrow::Result> file = + arrow::io::ReadableFile::Open(fragment->path, pool); + + if (!file.ok()) + return DlArrowStatus(file.status(), "open a Parquet file"); + impl->file = *file; + + arrow::Status status = builder.Open(impl->file); + + if (!status.ok()) + return DlArrowStatus(status, "open a Parquet file"); + + /* The same batch size the write side accumulates to. */ + properties.set_batch_size(iceberg_batch_rows); + + /* + * A backend is not a thread pool. Arrow will read column chunks in + * parallel if asked, and a worker thread that hits an error has no way + * to report it through PostgreSQL's error handling, so this reads on + * the thread it was called on. + */ + properties.set_use_threads(false); + + status = builder.memory_pool(pool)->properties(properties) + ->Build(&impl->reader); + + if (!status.ok()) + return DlArrowStatus(status, "open a Parquet file"); + + rc = parquet_row_groups(fragment, impl->reader->num_row_groups(), + &row_groups); + if (rc != DL_OK) + return rc; + + if (projection != NULL && projection->ncolumns > 0) + columns.assign(projection->columns, + projection->columns + projection->ncolumns); + else + { + std::shared_ptr schema; + + status = impl->reader->GetSchema(&schema); + if (!status.ok()) + return DlArrowStatus(status, "read a Parquet schema"); + + for (int i = 0; i < schema->num_fields(); i++) + columns.push_back(i); + } + + /* + * Arrow 21 deprecates this in favour of a Result-returning one that + * Arrow 9 does not have, so whoever raises the floor past 21 gets a + * warning here and a version guard to write -- the one around + * FileWriter::Open in parquet_write.cpp is the shape of it. + */ + status = impl->reader->GetRecordBatchReader(row_groups, columns, + &impl->batches); + if (!status.ok()) + return DlArrowStatus(status, "open a Parquet batch reader"); + + impl->base.ops = &parquet_reader_ops; + impl->base.impl = impl.get(); + + /* + * The file is open from here, so this is the last thing that may fail: + * past it, nothing can lose track of the descriptor. + */ + if (!dl_resource_remember(parquet_reader_release, impl.get())) + { + dl_error_set(DL_ERR_INTERNAL, "open_reader", NULL, + "could not record the open file for cleanup"); + return DL_ERR_INTERNAL; + } + + *out = &impl.release()->base; + } + DL_ABI_GUARD_END(result, "open_reader"); + + return result; +} diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp new file mode 100644 index 00000000000..e5c00b9b043 --- /dev/null +++ b/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp @@ -0,0 +1,451 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parquet_write.cpp + * Writing a Parquet file. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/parquet/parquet_write.cpp + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "format/arrow_support.h" + +#include "common/dl_resource.h" +#include "common/dl_wrappers.h" +#include "format/parquet/parquet_internal.h" + +struct ParquetWriter +{ + FormatWriter base; + std::string path; + std::shared_ptr schema; + std::shared_ptr sink; + std::unique_ptr writer; + + /* + * A row group is written whole, so the batches that go into one are held + * until there are enough of them. This is not a buffer we chose to add: + * Parquet cannot begin a row group it does not have, and the alternative + * -- one row group per batch -- would produce files whose row groups are + * a thousand rows, where a reader expects something nearer a million and + * pays a seek for each one. + */ + std::vector> pending; + int64_t pending_rows; + int64_t row_group_size; +}; + +/* + * Releases a batch unless something already has. The C data interface clears + * the callback when ownership moves, so this is a no-op on the path where Arrow + * took the batch, and the release on every other path. + */ +class ParquetReleaseBatch +{ +public: + explicit ParquetReleaseBatch(struct ArrowArray *batch) : batch_(batch) {} + ~ParquetReleaseBatch() + { + if (batch_ != nullptr && batch_->release != nullptr) + batch_->release(batch_); + } + +private: + struct ArrowArray *batch_; +}; + +/* + * Gives up on the file being written. The sink is closed first and the writer + * left to its destructor: a Parquet writer writes the footer when it closes, + * and against a sink that is already closed it cannot -- which is what stops a + * complete, valid, truncated file appearing at the path if the unlink does not + * take. What it leaves then has no footer, so nothing can read it, and that is + * why the unlink's result is not worth reporting. + */ +static void +parquet_discard(ParquetWriter *impl) +{ + if (impl->sink != nullptr) + (void) impl->sink->Close(); + + (void) unlink(impl->path.c_str()); +} + +/* + * What the resource owner calls if nothing else did. A file that got this far + * was never finished, so it is discarded rather than left: the same thing + * abort() does, and for the same reason. C linkage because a C function + * pointer is what it is handed to. + */ +extern "C" void +parquet_writer_release(void *arg) +{ + DL_CLEANUP_GUARD_BEGIN + { + std::unique_ptr impl(static_cast(arg)); + + parquet_discard(impl.get()); + } + DL_CLEANUP_GUARD_END; +} + +/* + * Discards the file unless the scope it guards clears it. finish() has to get + * rid of a file it could not complete on every way out, and one of those ways + * is an exception that the guard around it turns into an error code -- past any + * cleanup written as a statement. + */ +class ParquetDiscardOnFailure +{ +public: + explicit ParquetDiscardOnFailure(ParquetWriter *writer) : writer_(writer) {} + ~ParquetDiscardOnFailure() + { + if (writer_ != nullptr) + parquet_discard(writer_); + } + void Keep() { writer_ = nullptr; } + +private: + ParquetWriter *writer_; +}; + +static arrow::Status +parquet_flush_row_group(ParquetWriter *impl) +{ + if (impl->pending.empty()) + return arrow::Status::OK(); + + ARROW_ASSIGN_OR_RAISE(std::shared_ptr table, + arrow::Table::FromRecordBatches(impl->schema, + impl->pending)); + + impl->pending.clear(); + impl->pending_rows = 0; + + return impl->writer->WriteTable(*table, impl->row_group_size); +} + +static DlErrCode +parquet_writer_write_batch(FormatWriter *writer, struct ArrowArray *batch) +{ + DlErrCode result = DL_OK; + + if (batch == NULL) + return DL_ARG_ERROR("write_batch"); + + /* + * The interface promises the batch is consumed whether or not the write + * succeeds, and that has to hold for the ways out that are not a return: + * importing allocates before it takes ownership, so it can throw with the + * batch still live. On the ordinary path the import has already cleared + * the callback and this does nothing. + */ + ParquetReleaseBatch release_batch(batch); + + if (writer == NULL) + return DL_ARG_ERROR("write_batch"); + + DL_ABI_GUARD_BEGIN + { + ParquetWriter *impl = static_cast(writer->impl); + + arrow::Result> imported = + arrow::ImportRecordBatch(batch, impl->schema); + + if (!imported.ok()) + return DlArrowStatus(imported.status(), "import an Arrow batch"); + + int64_t rows = (*imported)->num_rows(); + + /* + * Flush before the batch that would take the group past its size, not + * after. Flushing afterwards leaves a remainder that WriteTable emits + * as a second, tiny row group -- so a batch size that does not divide + * the row group size would produce exactly the file of many small row + * groups this buffering exists to avoid. + */ + if (impl->pending_rows > 0 && + impl->pending_rows + rows > impl->row_group_size) + { + arrow::Status status = parquet_flush_row_group(impl); + + if (!status.ok()) + return DlArrowStatus(status, "write a Parquet row group"); + } + + impl->pending_rows += rows; + impl->pending.push_back(*imported); + } + DL_ABI_GUARD_END(result, "write_batch"); + + return result; +} + +static DlErrCode +parquet_writer_bytes_written(FormatWriter *writer, int64_t *out) +{ + DlErrCode result = DL_OK; + + if (writer == NULL || out == NULL) + return DL_ARG_ERROR("bytes_written"); + + DL_ABI_GUARD_BEGIN + { + ParquetWriter *impl = static_cast(writer->impl); + + /* + * What has reached the file, which trails what has been handed over: + * a row group is written whole, so the batches waiting for one are + * not in this number. The layer that rolls files reads it to decide + * when a file is big enough, and the undercount costs it one row group + * of overshoot -- the tolerance the interface is written with. + */ + arrow::Result position = impl->sink->Tell(); + + if (!position.ok()) + return DlArrowStatus(position.status(), "measure a Parquet file"); + + *out = *position; + } + DL_ABI_GUARD_END(result, "bytes_written"); + + return result; +} + +static DlErrCode +parquet_writer_finish(FormatWriter **writer, FileMeta **meta) +{ + DlErrCode result = DL_OK; + + if (writer == NULL || *writer == NULL) + return DL_ARG_ERROR("finish_writer"); + + if (meta != NULL) + *meta = NULL; /* what a commit needs is not collected yet */ + + DL_ABI_GUARD_BEGIN + { + /* + * Taking ownership here is what makes "consumed either way" true even + * of the paths that leave through an exception: the caller's handle is + * cleared before anything that could fail. + */ + std::unique_ptr impl( + static_cast((*writer)->impl)); + ParquetDiscardOnFailure discard(impl.get()); + + *writer = NULL; + dl_resource_forget(parquet_writer_release, impl.get()); + + /* + * Closing the writer is what writes the footer, so a failure here + * leaves an unreadable file behind and has to be reported -- this is + * the one place in the writer's interface where close-time errors can + * still reach a caller. + */ + arrow::Status status = parquet_flush_row_group(impl.get()); + + if (status.ok()) + status = impl->writer->Close(); + if (status.ok()) + status = impl->sink->Close(); + + if (!status.ok()) + return DlArrowStatus(status, "finish a Parquet file"); + + discard.Keep(); + } + DL_ABI_GUARD_END(result, "finish_writer"); + + return result; +} + +static void +parquet_writer_abort(FormatWriter **writer) +{ + if (writer == NULL || *writer == NULL) + return; + + /* Cleared first; see the note in parquet_reader_close(). */ + ParquetWriter *impl = static_cast((*writer)->impl); + + *writer = NULL; + dl_resource_forget(parquet_writer_release, impl); + + /* + * A file that was never finished has no footer, so nothing can read it and + * leaving it behind only costs space and confusion. Failures are ignored: + * this runs while an error is already being handled. + */ + parquet_writer_release(impl); +} + +static const FormatWriterOps parquet_writer_ops = { + parquet_writer_write_batch, + parquet_writer_bytes_written, + parquet_writer_finish, + parquet_writer_abort +}; + +static DlErrCode +parquet_compression(const char *name, arrow::Compression::type *out) +{ + std::string requested(name); + char message[128]; + + if (requested == "none" || requested == "uncompressed") + *out = arrow::Compression::UNCOMPRESSED; + else if (requested == "snappy") + *out = arrow::Compression::SNAPPY; + else if (requested == "gzip") + *out = arrow::Compression::GZIP; + else if (requested == "zstd") + *out = arrow::Compression::ZSTD; + else + { + snprintf(message, sizeof(message), + "\"%s\" is not a compression this build can write", name); + dl_error_set(DL_ERR_INVALID_OPTION, "open a Parquet file", NULL, message); + return DL_ERR_INVALID_OPTION; + } + + return DL_OK; +} + +DlErrCode +parquet_open_writer(const char *path, void *tupdesc_arg, + const WriterOptions *options, FormatWriter **out) +{ + DlErrCode result = DL_OK; + + if (out == NULL) + return DL_ARG_ERROR("open_writer"); + *out = NULL; + + if (path == NULL || tupdesc_arg == NULL) + return DL_ARG_ERROR("open_writer"); + + DL_ABI_GUARD_BEGIN + { + std::unique_ptr impl(new ParquetWriter()); + arrow::MemoryPool *pool = arrow::default_memory_pool(); + parquet::WriterProperties::Builder properties; + arrow::Compression::type compression = arrow::Compression::SNAPPY; + DlErrCode rc; + + impl->path = path; + impl->schema = DlArrowSchemaFromTupleDesc((TupleDesc) tupdesc_arg); + if (impl->schema == nullptr) + return DL_ERR_NOT_SUPPORTED; /* detail already recorded */ + + if (options != NULL && options->compression != NULL) + { + rc = parquet_compression(options->compression, &compression); + if (rc != DL_OK) + return rc; + } + properties.compression(compression); + + if (options != NULL && options->row_group_size > 0) + properties.max_row_group_length(options->row_group_size); + + std::shared_ptr built = properties.build(); + + /* Whether it was asked for or left to Parquet, this is the size. */ + impl->row_group_size = built->max_row_group_length(); + impl->pending_rows = 0; + + arrow::Result> sink = + arrow::io::FileOutputStream::Open(path); + + if (!sink.ok()) + return DlArrowStatus(sink.status(), "create a Parquet file"); + impl->sink = *sink; + + /* + * The Arrow schema is deliberately not stored in the file's metadata. + * With it, reading back would restore the types from our own note + * rather than from Parquet's, and a round trip would agree with itself + * no matter what it had written; without it, what comes back is what + * any other reader of the file sees. + */ + /* + * Arrow 11 deprecated the form that returns its writer through an out + * parameter. Both spellings have to be here because the versions this + * builds against range from 9 to 17 depending on the distribution, and + * the older one warns on the newer Arrow rather than failing -- which is + * the kind of warning that stops being read. + */ +#if ARROW_VERSION_MAJOR >= 11 + arrow::Result> writer = + parquet::arrow::FileWriter::Open(*impl->schema, pool, impl->sink, + built, + parquet::default_arrow_writer_properties()); + arrow::Status status = writer.status(); + + if (status.ok()) + impl->writer = std::move(*writer); +#else + arrow::Status status = + parquet::arrow::FileWriter::Open(*impl->schema, pool, impl->sink, + built, + parquet::default_arrow_writer_properties(), + &impl->writer); +#endif + + if (!status.ok()) + { + (void) impl->sink->Close(); + unlink(path); + return DlArrowStatus(status, "create a Parquet file"); + } + + impl->base.ops = &parquet_writer_ops; + impl->base.impl = impl.get(); + + /* Last thing that may fail; see parquet_open_reader(). */ + if (!dl_resource_remember(parquet_writer_release, impl.get())) + { + parquet_discard(impl.get()); + dl_error_set(DL_ERR_INTERNAL, "open_writer", NULL, + "could not record the open file for cleanup"); + return DL_ERR_INTERNAL; + } + + *out = &impl.release()->base; + } + DL_ABI_GUARD_END(result, "open_writer"); + + return result; +} diff --git a/contrib/datalake_fdw/src/test/datalake_fdw_test.c b/contrib/datalake_fdw/src/test/datalake_fdw_test.c new file mode 100644 index 00000000000..1d2c2946d11 --- /dev/null +++ b/contrib/datalake_fdw/src/test/datalake_fdw_test.c @@ -0,0 +1,411 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * datalake_fdw_test.c + * The format layer, reachable from SQL. + * + * A data file is written and read by the access method, which is not finished; + * until it is, there is no way to run the format layer in a real backend, and + * "it compiles" would be the only thing anyone could say about it. These two + * functions are that way in: they write the result of a query to a file and + * read a file back as rows, so a round trip is an ordinary SQL statement. + * + * They are a separate extension because they are not part of what this module + * offers -- installing datalake_fdw does not put them in the database. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/test/datalake_fdw_test.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "executor/spi.h" +#include "funcapi.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/tuplestore.h" + +#include "am_iceberg/pg_iceberg_guc.h" +#include "common/dl_err.h" +#include "format/arrow_builder.h" +#include "format/arrow_decode.h" +#include "format/format.h" + +PG_FUNCTION_INFO_V1(datalake_parquet_write); +PG_FUNCTION_INFO_V1(datalake_parquet_read); + +static const FormatRoutine * +parquet_routine(void) +{ + const FormatRoutine *routine = GetFormatRoutine("parquet"); + + /* + * Only reachable from a build that dropped the format, so what it can say + * is whatever the registry recorded -- guessing at a reason here would be + * a message that outlives the thing it describes. + */ + if (routine == NULL) + dl_error_report(ERROR, DL_ERR_NOT_SUPPORTED, "get_format"); + + return routine; +} + +/* + * Hands one batch to the writer. The batch is consumed either way, so there is + * nothing left to release when this reports a failure. + */ +static void +write_one_batch(FormatWriter *writer, DlArrowBuilder builder) +{ + struct ArrowArray batch; + DlErrCode rc; + + rc = dl_arrow_builder_flush(builder, &batch); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "build_batch"); + + rc = writer->ops->write_batch(writer, &batch); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "write_batch"); +} + +/* + * datalake_parquet_write(path, query, row_group_size) -> rows written + * + * The rows the query returns are written to `path` as Parquet. A row group + * size of zero leaves the format's own default in place; anything else also + * becomes the number of rows per batch, because a row group is closed at a + * batch boundary and the option would otherwise be rounded away by a batch size + * that does not divide by it. + */ +Datum +datalake_parquet_write(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *query = text_to_cstring(PG_GETARG_TEXT_PP(1)); + int32 row_group_size = PG_GETARG_INT32(2); + const FormatRoutine *routine = parquet_routine(); + WriterOptions options = {0}; + FormatWriter *volatile open_writer = NULL; + DlArrowBuilder volatile open_builder = NULL; + long batch_rows = iceberg_batch_rows; + int64 written = 0; + MemoryContext row_context; + + /* + * Bounded above as well as below, and by the same number as + * iceberg.batch_rows: a row group is held in memory until it is complete, + * so an unbounded one asks the writer to buffer the whole result set. + */ + if (row_group_size < 0 || row_group_size > DL_MAX_ROW_GROUP_ROWS) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("row group size must be between 0 and %d", + DL_MAX_ROW_GROUP_ROWS))); + + options.row_group_size = row_group_size; + if (row_group_size > 0 && row_group_size < batch_rows) + batch_rows = row_group_size; + + if (SPI_connect() != SPI_OK_CONNECT) + elog(ERROR, "SPI_connect failed"); + + /* + * Detoasting a value allocates, and the copy is dead as soon as it has been + * appended. Without a context of its own, a wide table would hold every + * copy it ever made until the function returned. + */ + row_context = AllocSetContextCreate(CurrentMemoryContext, + "datalake_parquet_write", + ALLOCSET_DEFAULT_SIZES); + + PG_TRY(); + { + SPIPlanPtr plan; + Portal portal; + TupleDesc tupdesc = NULL; + FormatWriter *writer = NULL; + DlArrowBuilder builder = NULL; + Datum *values = NULL; + bool *nulls = NULL; + DlErrCode rc; + + plan = SPI_prepare(query, 0, NULL); + if (plan == NULL) + elog(ERROR, "SPI_prepare failed: %s", + SPI_result_code_string(SPI_result)); + + portal = SPI_cursor_open(NULL, plan, NULL, NULL, true); + + for (;;) + { + MemoryContext oldcontext; + uint64 i; + + SPI_cursor_fetch(portal, true, batch_rows); + + if (SPI_tuptable == NULL) + elog(ERROR, "the query did not return a result set"); + + /* + * The descriptor is only available once something has been + * fetched, and the writer needs it before the first row can be + * appended -- so the file is created here rather than before the + * loop. The copy outlives SPI_freetuptable(), which frees the + * descriptor along with the rows it described. + */ + if (tupdesc == NULL) + { + tupdesc = CreateTupleDescCopy(SPI_tuptable->tupdesc); + + rc = routine->open_writer(path, tupdesc, &options, &writer); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "open_writer"); + open_writer = writer; + + rc = dl_arrow_builder_open(tupdesc, &builder); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "open_builder"); + open_builder = builder; + + values = palloc(tupdesc->natts * sizeof(Datum)); + nulls = palloc(tupdesc->natts * sizeof(bool)); + } + + if (SPI_processed == 0) + break; + + oldcontext = MemoryContextSwitchTo(row_context); + + for (i = 0; i < SPI_processed; i++) + { + HeapTuple tuple = SPI_tuptable->vals[i]; + int attno; + + CHECK_FOR_INTERRUPTS(); + + for (attno = 0; attno < tupdesc->natts; attno++) + { + Form_pg_attribute attr = TupleDescAttr(tupdesc, attno); + bool isnull; + Datum value = SPI_getbinval(tuple, SPI_tuptable->tupdesc, + attno + 1, &isnull); + + /* + * The Arrow side runs as C++ and must not allocate, so a + * value that is compressed or stored out of line is + * expanded here, where failing to do so is an ordinary + * error rather than an exception crossing an ABI. + */ + if (!isnull && attr->attlen == -1) + value = PointerGetDatum(PG_DETOAST_DATUM_PACKED(value)); + + values[attno] = value; + nulls[attno] = isnull; + } + + rc = dl_arrow_builder_append(builder, values, nulls, + tupdesc->natts); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "append_row"); + + written++; + } + + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(row_context); + + write_one_batch(writer, builder); + SPI_freetuptable(SPI_tuptable); + } + + SPI_cursor_close(portal); + + /* + * A query that returned nothing still produces a file, with the schema + * and no rows: an empty file is a fact about the query, and a missing + * one would be a fact about this function. + */ + rc = writer->ops->finish(&writer, NULL); + open_writer = NULL; /* consumed, whether or not it succeeded */ + if (rc != DL_OK) + dl_error_report(ERROR, rc, "finish_writer"); + + dl_arrow_builder_close(&builder); + open_builder = NULL; + } + PG_CATCH(); + { + DlArrowBuilder builder = open_builder; + FormatWriter *writer = open_writer; + + if (builder != NULL) + dl_arrow_builder_close(&builder); + if (writer != NULL) + writer->ops->abort(&writer); + + PG_RE_THROW(); + } + PG_END_TRY(); + + SPI_finish(); + + PG_RETURN_INT64(written); +} + +/* + * datalake_parquet_read(path, first_row_group, n_row_groups) -> setof record + * + * The column definition list says what the caller expects the file to hold, and + * is checked against the file's own schema rather than assumed: reading an + * Arrow column as the wrong PostgreSQL type would produce values, just not the + * ones in the file. + * + * The row group arguments are the unit a scan is divided at. Reading 0..0 and + * then 1..1 has to produce exactly what reading the whole file does, which is + * the property a scan spread across segments will depend on. + */ +Datum +datalake_parquet_read(PG_FUNCTION_ARGS) +{ + char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + const FormatRoutine *routine = parquet_routine(); + FormatReader *volatile open_reader = NULL; + struct ArrowArray *batch = palloc0(sizeof(struct ArrowArray)); + struct ArrowSchema *schema = palloc0(sizeof(struct ArrowSchema)); + Fragment fragment = {0}; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + Datum *values; + bool *nulls; + FormatReader *reader = NULL; + MemoryContext row_context; + DlErrCode rc; + + fragment.path = path; + fragment.first_row_group = PG_GETARG_INT32(1); + fragment.n_row_groups = PG_GETARG_INT32(2); + + InitMaterializedSRF(fcinfo, MAT_SRF_USE_EXPECTED_DESC); + tupdesc = rsinfo->setDesc; + tupstore = rsinfo->setResult; + + values = palloc(tupdesc->natts * sizeof(Datum)); + nulls = palloc(tupdesc->natts * sizeof(bool)); + + /* + * Every text and bytea decoded out of a batch is a copy, and tuplestore + * copies it again. A materialize-mode function is called once, so the + * caller's per-tuple context is not reset until it returns -- without a + * context of its own, reading a large file would hold a second copy of all + * of it until then. + */ + row_context = AllocSetContextCreate(CurrentMemoryContext, + "datalake_parquet_read", + ALLOCSET_DEFAULT_SIZES); + + rc = routine->open_reader(&fragment, NULL, NULL, &reader); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "open_reader"); + open_reader = reader; + + PG_TRY(); + { + for (;;) + { + MemoryContext oldcontext; + bool eof; + int64 row; + int attno; + + CHECK_FOR_INTERRUPTS(); + + rc = reader->ops->next_batch(reader, batch, schema, &eof); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "next_batch"); + if (eof) + break; + + if (schema->n_children != tupdesc->natts) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("the file does not have the number of columns the query expects"), + errdetail("The file has %lld, the query expects %d.", + (long long) schema->n_children, + tupdesc->natts))); + + for (attno = 0; attno < tupdesc->natts; attno++) + { + rc = dl_arrow_decode_check(schema->children[attno], + TupleDescAttr(tupdesc, attno)->atttypid); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "check_column"); + } + + oldcontext = MemoryContextSwitchTo(row_context); + + for (row = 0; row < batch->length; row++) + { + CHECK_FOR_INTERRUPTS(); + + for (attno = 0; attno < tupdesc->natts; attno++) + { + rc = dl_arrow_decode_value(batch->children[attno], row, + TupleDescAttr(tupdesc, attno)->atttypid, + TupleDescAttr(tupdesc, attno)->atttypmod, + &values[attno], &nulls[attno]); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "decode_value"); + } + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + + MemoryContextSwitchTo(oldcontext); + MemoryContextReset(row_context); + + /* Releasing the batch releases the columns under it. */ + batch->release(batch); + schema->release(schema); + } + + reader->ops->close(&reader); + open_reader = NULL; + MemoryContextDelete(row_context); + } + PG_CATCH(); + { + FormatReader *failed = open_reader; + + if (batch->release != NULL) + batch->release(batch); + if (schema->release != NULL) + schema->release(schema); + if (failed != NULL) + failed->ops->close(&failed); + + PG_RE_THROW(); + } + PG_END_TRY(); + + return (Datum) 0; +} diff --git a/contrib/datalake_fdw/test/automation/README.md b/contrib/datalake_fdw/test/automation/README.md index f6586ed0e5c..470dba525cb 100644 --- a/contrib/datalake_fdw/test/automation/README.md +++ b/contrib/datalake_fdw/test/automation/README.md @@ -56,13 +56,16 @@ scripts/test/ category runners scripts/utils/ shared shell helpers sqlrepo/smoke/ one directory per category iceberg_am/ DDL, refusals and privileges -- no external service + format_parquet/ a table through a local Parquet file and back ``` -`sqlrepo/smoke/iceberg_am` holds the cases pg_regress runs; the module's -`Makefile` points `--inputdir` here, so `make installcheck` from the module -directory and `make test` from this one run the same cases. They live here -rather than in a `sql/` directory of their own so that there is one place to look -for test material. +Each of these holds cases pg_regress runs, and pg_regress takes one +`--inputdir`, so each category is a run of its own: the module's `Makefile` has +`installcheck` for `iceberg_am` and `installcheck-format-parquet` for the other, +and hangs the second off the first so that asking for `installcheck` gets both. +`make test` from this directory runs the same cases. They live here rather than +in a `sql/` directory of their own so that there is one place to look for test +material. ## What arrives with the metadata engine diff --git a/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh b/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh index ce707a1fed3..9273c425753 100755 --- a/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh +++ b/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh @@ -37,9 +37,10 @@ dl_load_config # category:services -- an empty service list means "no external dependency" CATEGORY_SERVICES=" iceberg_am: +format_parquet: " -categories="${CATEGORIES:-iceberg_am}" +categories="${CATEGORIES:-iceberg_am format_parquet}" services_for() { @@ -74,10 +75,18 @@ service_is_available() run_iceberg_am() { # These cases are expected-output cases, so pg_regress runs them; the module - # Makefile already points it at sqlrepo/smoke/iceberg_am. + # Makefile already points it at sqlrepo/smoke/iceberg_am. That target also + # runs format_parquet, so running both categories here runs it twice -- + # which is what "make test CATEGORIES=format_parquet" has to keep working. make -C "$module_dir" USE_PGXS=1 installcheck } +run_format_parquet() +{ + # pg_regress takes one --inputdir, so each category is a run of its own. + make -C "$module_dir" USE_PGXS=1 installcheck-format-parquet +} + failed=0 skipped=0 ran=0 @@ -105,6 +114,7 @@ for category in $categories; do dl_info "RUN $category" case "$category" in iceberg_am) run_iceberg_am ;; + format_parquet) run_format_parquet ;; *) dl_warn "category \"$category\" has no runner"; false ;; esac diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out new file mode 100644 index 00000000000..6f649fd1cf4 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out @@ -0,0 +1,250 @@ +-- Parquet: every type a lake table can store, written to a file and read back, +-- and the row group range that a scan will one day be divided at. +SET client_min_messages = warning; +DROP VIEW IF EXISTS dlparq_roundtrip, dlparq_split; +DROP TABLE IF EXISTS dlparq_src, dlparq_pairs, dlparq_unsupported CASCADE; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +-- A timestamptz is printed in the session's zone, so without this the output +-- would depend on where the test ran rather than on what the file holds. +SET TimeZone = 'UTC'; +-- Fixed file names rather than a unique one per run: the writer truncates, so a +-- run reuses what the last one left instead of adding to it. Nothing reachable +-- from SQL can remove a file, so unique names would accumulate forever. +\set roundtrip_file '/tmp/datalake_fdw_regress_roundtrip.parquet' +\set split_file '/tmp/datalake_fdw_regress_split.parquet' +\set empty_file '/tmp/datalake_fdw_regress_empty.parquet' +\set batch_file '/tmp/datalake_fdw_regress_batch.parquet' +\set missing_file '/tmp/datalake_fdw_regress_does_not_exist.parquet' +CREATE TABLE dlparq_src ( + c_bool boolean, + c_int2 smallint, + c_int4 integer, + c_int8 bigint, + c_float4 real, + c_float8 double precision, + c_text text, + c_varchar varchar(16), + c_bpchar char(5), + c_bytea bytea, + c_date date, + c_ts timestamp, + c_tstz timestamptz +) DISTRIBUTED RANDOMLY; +-- The dates and timestamps are chosen around both epochs: PostgreSQL counts +-- from 2000-01-01 and Arrow from 1970-01-01, and a value on either side of +-- 1970 is what tells a wrong shift from a right one. A row of nulls is here +-- because a validity bitmap that is never exercised is a bitmap that has not +-- been tested. +INSERT INTO dlparq_src VALUES + (true, 1, 100, 1000, 1.5, 2.5, + 'hello', 'varchar', 'abc', '\x0102'::bytea, + '1970-01-01', '1970-01-01 00:00:00', '1970-01-01 00:00:00+00'), + (false, -2, -200, -2000, -1.5, -2.5, + 'a longer string with 中文', 'x', '', '\x'::bytea, + '2000-01-01', '2000-01-01 12:34:56.789012', '2000-01-01 12:34:56.789012+00'), + (true, 32767, 2147483647, 9223372036854775807, 3.25, 1e300, + '', 'z', 'zzzzz', '\xdeadbeef'::bytea, + '1969-12-31', '1969-12-31 23:59:59.999999', '1969-12-31 23:59:59.999999+00'), + (NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL); +SELECT datalake_parquet_write(:'roundtrip_file', + 'SELECT * FROM dlparq_src') AS rows_written; + rows_written +-------------- + 4 +(1 row) + +-- The column definition list is what the caller claims the file holds; it is +-- checked against the file's own schema, not assumed. A view so that the list +-- is written once. +CREATE VIEW dlparq_roundtrip AS + SELECT * FROM datalake_parquet_read(:'roundtrip_file') AS t ( + c_bool boolean, + c_int2 smallint, + c_int4 integer, + c_int8 bigint, + c_float4 real, + c_float8 double precision, + c_text text, + c_varchar varchar(16), + c_bpchar char(5), + c_bytea bytea, + c_date date, + c_ts timestamp, + c_tstz timestamptz); +SELECT * FROM dlparq_roundtrip ORDER BY c_int4; + c_bool | c_int2 | c_int4 | c_int8 | c_float4 | c_float8 | c_text | c_varchar | c_bpchar | c_bytea | c_date | c_ts | c_tstz +--------+--------+------------+---------------------+----------+----------+---------------------------+-----------+----------+------------+------------+---------------------------------+------------------------------------- + f | -2 | -200 | -2000 | -1.5 | -2.5 | a longer string with 中文 | x | | \x | 01-01-2000 | Sat Jan 01 12:34:56.789012 2000 | Sat Jan 01 12:34:56.789012 2000 UTC + t | 1 | 100 | 1000 | 1.5 | 2.5 | hello | varchar | abc | \x0102 | 01-01-1970 | Thu Jan 01 00:00:00 1970 | Thu Jan 01 00:00:00 1970 UTC + t | 32767 | 2147483647 | 9223372036854775807 | 3.25 | 1e+300 | | z | zzzzz | \xdeadbeef | 12-31-1969 | Wed Dec 31 23:59:59.999999 1969 | Wed Dec 31 23:59:59.999999 1969 UTC + | | | | | | | | | | | | +(4 rows) + +-- Both directions: one way only says what the file lost, and a file with a row +-- nobody wrote is just as wrong as one missing a row somebody did. +SELECT count(*) AS differences +FROM ((TABLE dlparq_src EXCEPT ALL TABLE dlparq_roundtrip) + UNION ALL + (TABLE dlparq_roundtrip EXCEPT ALL TABLE dlparq_src)) d; + differences +------------- + 0 +(1 row) + +-- The comparison above cannot see the padding of a char(n): bpchar equality +-- ignores trailing spaces, so a value that came back three characters long +-- would still have compared equal to the five it went in as. +SELECT octet_length(c_bpchar) AS bpchar_bytes, + octet_length(c_text) AS text_bytes, + octet_length(c_bytea) AS bytea_bytes +FROM dlparq_roundtrip ORDER BY 1, 2, 3; + bpchar_bytes | text_bytes | bytea_bytes +--------------+------------+------------- + 5 | 0 | 4 + 5 | 5 | 2 + 5 | 27 | 0 + | | +(4 rows) + +-- Row groups are the unit a scan is divided at, so reading the parts has to add +-- up to reading the whole -- no row seen twice, none missed. +CREATE TABLE dlparq_pairs AS + SELECT i AS k, 'v' || i AS v FROM generate_series(1, 6) i + DISTRIBUTED RANDOMLY; +SELECT datalake_parquet_write(:'split_file', + 'SELECT k, v FROM dlparq_pairs ORDER BY k', + 2) AS rows_written; + rows_written +-------------- + 6 +(1 row) + +CREATE VIEW dlparq_split AS + SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int, v text); +SELECT * FROM dlparq_split ORDER BY k; + k | v +---+---- + 1 | v1 + 2 | v2 + 3 | v3 + 4 | v4 + 5 | v5 + 6 | v6 +(6 rows) + +SELECT 0 AS first_row_group, * FROM datalake_parquet_read(:'split_file', 0, 1) + AS t (k int, v text) +UNION ALL +SELECT 1, * FROM datalake_parquet_read(:'split_file', 1, 1) AS t (k int, v text) +UNION ALL +SELECT 2, * FROM datalake_parquet_read(:'split_file', 2, 1) AS t (k int, v text) +ORDER BY 1, 2; + first_row_group | k | v +-----------------+---+---- + 0 | 1 | v1 + 0 | 2 | v2 + 1 | 3 | v3 + 1 | 4 | v4 + 2 | 5 | v5 + 2 | 6 | v6 +(6 rows) + +-- Reading from a row group on is the same as reading each of them. +SELECT count(*) AS rows_from_the_second_on +FROM datalake_parquet_read(:'split_file', 1) AS t (k int, v text); + rows_from_the_second_on +------------------------- + 4 +(1 row) + +-- A query that returns nothing still produces a file: an empty file is a fact +-- about the query, a missing one would be a fact about the writer. +SELECT datalake_parquet_write(:'empty_file', + 'SELECT k, v FROM dlparq_pairs WHERE false') AS rows_written; + rows_written +-------------- + 0 +(1 row) + +SELECT count(*) AS rows_read +FROM datalake_parquet_read(:'empty_file') AS t (k int, v text); + rows_read +----------- + 0 +(1 row) + +-- iceberg.batch_rows is how many rows cross the boundary at a time, and both +-- halves read it. Everything above fits in one batch, which leaves the loops +-- on both sides running exactly once; at two rows a batch the reader iterates +-- and the writer accumulates several batches into the one row group. +SET iceberg.batch_rows = 2; +SELECT count(*) AS rows_in_batches_of_two FROM dlparq_split; + rows_in_batches_of_two +------------------------ + 6 +(1 row) + +SELECT datalake_parquet_write(:'batch_file', + 'SELECT k, v FROM dlparq_pairs ORDER BY k') AS rows_written; + rows_written +-------------- + 6 +(1 row) + +SELECT * FROM datalake_parquet_read(:'batch_file') AS t (k int, v text) ORDER BY k; + k | v +---+---- + 1 | v1 + 2 | v2 + 3 | v3 + 4 | v4 + 5 | v5 + 6 | v6 +(6 rows) + +RESET iceberg.batch_rows; +-- Refusals. Each of these would otherwise be a wrong answer rather than an +-- error: a column silently dropped, a value reinterpreted, a short read. +CREATE TABLE dlparq_unsupported (k int, n numeric) DISTRIBUTED RANDOMLY; +SELECT datalake_parquet_write(:'roundtrip_file', + 'SELECT * FROM dlparq_unsupported'); +ERROR: iceberg: open_writer failed +DETAIL: arrow schema: column "n" has a type that lake tables cannot store yet +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k bigint, v text); +ERROR: iceberg: check_column failed +DETAIL: decode an Arrow column: a column stored as Arrow type "i" cannot be read as bigint +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int); +ERROR: the file does not have the number of columns the query expects +DETAIL: The file has 2, the query expects 1. +SELECT * FROM datalake_parquet_read(:'split_file', 9, 1) AS t (k int, v text); +ERROR: iceberg: open_reader failed +DETAIL: open a Parquet file: row groups 9..9 were asked for from a file that has 3 +-- A row group range that only overflowed arithmetic would let through: the +-- count is rejected rather than turned into a two-billion-entry list. +SELECT * FROM datalake_parquet_read(:'split_file', 1, 2147483647) AS t (k int, v text); +ERROR: iceberg: open_reader failed +DETAIL: open a Parquet file: row groups 1..2147483647 were asked for from a file that has 3 +SELECT datalake_parquet_write(:'split_file', 'SELECT 1', -1); +ERROR: row group size must be between 0 and 1048576 +SELECT datalake_parquet_write(:'split_file', 'SELECT 1', 2000000000); +ERROR: row group size must be between 0 and 1048576 +-- PostgreSQL's timestamp range runs about 34 years past the last instant Arrow +-- can hold as microseconds from 1970. Writing one of those has to be refused, +-- because the shift would wrap and the value would land 292000 years before the +-- epoch with the write reporting success. +SELECT datalake_parquet_write(:'batch_file', + $$SELECT '294250-01-01 00:00:00'::timestamp$$); +ERROR: iceberg: append_row failed +DETAIL: append a value to an Arrow array: timestamp is too far in the future to be written to a data file (Invalid) +-- Arrow words this one, and its wording is not ours to depend on. +\set VERBOSITY terse +SELECT * FROM datalake_parquet_read(:'missing_file') AS t (k int, v text); +ERROR: iceberg: open_reader failed +\set VERBOSITY default +SET client_min_messages = warning; +DROP VIEW dlparq_roundtrip, dlparq_split; +DROP TABLE dlparq_src, dlparq_pairs, dlparq_unsupported; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql new file mode 100644 index 00000000000..5b709d20921 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql @@ -0,0 +1,174 @@ +-- Parquet: every type a lake table can store, written to a file and read back, +-- and the row group range that a scan will one day be divided at. + +SET client_min_messages = warning; +DROP VIEW IF EXISTS dlparq_roundtrip, dlparq_split; +DROP TABLE IF EXISTS dlparq_src, dlparq_pairs, dlparq_unsupported CASCADE; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +-- A timestamptz is printed in the session's zone, so without this the output +-- would depend on where the test ran rather than on what the file holds. +SET TimeZone = 'UTC'; + +-- Fixed file names rather than a unique one per run: the writer truncates, so a +-- run reuses what the last one left instead of adding to it. Nothing reachable +-- from SQL can remove a file, so unique names would accumulate forever. +\set roundtrip_file '/tmp/datalake_fdw_regress_roundtrip.parquet' +\set split_file '/tmp/datalake_fdw_regress_split.parquet' +\set empty_file '/tmp/datalake_fdw_regress_empty.parquet' +\set batch_file '/tmp/datalake_fdw_regress_batch.parquet' +\set missing_file '/tmp/datalake_fdw_regress_does_not_exist.parquet' + +CREATE TABLE dlparq_src ( + c_bool boolean, + c_int2 smallint, + c_int4 integer, + c_int8 bigint, + c_float4 real, + c_float8 double precision, + c_text text, + c_varchar varchar(16), + c_bpchar char(5), + c_bytea bytea, + c_date date, + c_ts timestamp, + c_tstz timestamptz +) DISTRIBUTED RANDOMLY; + +-- The dates and timestamps are chosen around both epochs: PostgreSQL counts +-- from 2000-01-01 and Arrow from 1970-01-01, and a value on either side of +-- 1970 is what tells a wrong shift from a right one. A row of nulls is here +-- because a validity bitmap that is never exercised is a bitmap that has not +-- been tested. +INSERT INTO dlparq_src VALUES + (true, 1, 100, 1000, 1.5, 2.5, + 'hello', 'varchar', 'abc', '\x0102'::bytea, + '1970-01-01', '1970-01-01 00:00:00', '1970-01-01 00:00:00+00'), + (false, -2, -200, -2000, -1.5, -2.5, + 'a longer string with 中文', 'x', '', '\x'::bytea, + '2000-01-01', '2000-01-01 12:34:56.789012', '2000-01-01 12:34:56.789012+00'), + (true, 32767, 2147483647, 9223372036854775807, 3.25, 1e300, + '', 'z', 'zzzzz', '\xdeadbeef'::bytea, + '1969-12-31', '1969-12-31 23:59:59.999999', '1969-12-31 23:59:59.999999+00'), + (NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL); + +SELECT datalake_parquet_write(:'roundtrip_file', + 'SELECT * FROM dlparq_src') AS rows_written; + +-- The column definition list is what the caller claims the file holds; it is +-- checked against the file's own schema, not assumed. A view so that the list +-- is written once. +CREATE VIEW dlparq_roundtrip AS + SELECT * FROM datalake_parquet_read(:'roundtrip_file') AS t ( + c_bool boolean, + c_int2 smallint, + c_int4 integer, + c_int8 bigint, + c_float4 real, + c_float8 double precision, + c_text text, + c_varchar varchar(16), + c_bpchar char(5), + c_bytea bytea, + c_date date, + c_ts timestamp, + c_tstz timestamptz); + +SELECT * FROM dlparq_roundtrip ORDER BY c_int4; + +-- Both directions: one way only says what the file lost, and a file with a row +-- nobody wrote is just as wrong as one missing a row somebody did. +SELECT count(*) AS differences +FROM ((TABLE dlparq_src EXCEPT ALL TABLE dlparq_roundtrip) + UNION ALL + (TABLE dlparq_roundtrip EXCEPT ALL TABLE dlparq_src)) d; + +-- The comparison above cannot see the padding of a char(n): bpchar equality +-- ignores trailing spaces, so a value that came back three characters long +-- would still have compared equal to the five it went in as. +SELECT octet_length(c_bpchar) AS bpchar_bytes, + octet_length(c_text) AS text_bytes, + octet_length(c_bytea) AS bytea_bytes +FROM dlparq_roundtrip ORDER BY 1, 2, 3; + +-- Row groups are the unit a scan is divided at, so reading the parts has to add +-- up to reading the whole -- no row seen twice, none missed. +CREATE TABLE dlparq_pairs AS + SELECT i AS k, 'v' || i AS v FROM generate_series(1, 6) i + DISTRIBUTED RANDOMLY; + +SELECT datalake_parquet_write(:'split_file', + 'SELECT k, v FROM dlparq_pairs ORDER BY k', + 2) AS rows_written; + +CREATE VIEW dlparq_split AS + SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int, v text); + +SELECT * FROM dlparq_split ORDER BY k; + +SELECT 0 AS first_row_group, * FROM datalake_parquet_read(:'split_file', 0, 1) + AS t (k int, v text) +UNION ALL +SELECT 1, * FROM datalake_parquet_read(:'split_file', 1, 1) AS t (k int, v text) +UNION ALL +SELECT 2, * FROM datalake_parquet_read(:'split_file', 2, 1) AS t (k int, v text) +ORDER BY 1, 2; + +-- Reading from a row group on is the same as reading each of them. +SELECT count(*) AS rows_from_the_second_on +FROM datalake_parquet_read(:'split_file', 1) AS t (k int, v text); + +-- A query that returns nothing still produces a file: an empty file is a fact +-- about the query, a missing one would be a fact about the writer. +SELECT datalake_parquet_write(:'empty_file', + 'SELECT k, v FROM dlparq_pairs WHERE false') AS rows_written; +SELECT count(*) AS rows_read +FROM datalake_parquet_read(:'empty_file') AS t (k int, v text); + +-- iceberg.batch_rows is how many rows cross the boundary at a time, and both +-- halves read it. Everything above fits in one batch, which leaves the loops +-- on both sides running exactly once; at two rows a batch the reader iterates +-- and the writer accumulates several batches into the one row group. +SET iceberg.batch_rows = 2; +SELECT count(*) AS rows_in_batches_of_two FROM dlparq_split; +SELECT datalake_parquet_write(:'batch_file', + 'SELECT k, v FROM dlparq_pairs ORDER BY k') AS rows_written; +SELECT * FROM datalake_parquet_read(:'batch_file') AS t (k int, v text) ORDER BY k; +RESET iceberg.batch_rows; + +-- Refusals. Each of these would otherwise be a wrong answer rather than an +-- error: a column silently dropped, a value reinterpreted, a short read. +CREATE TABLE dlparq_unsupported (k int, n numeric) DISTRIBUTED RANDOMLY; +SELECT datalake_parquet_write(:'roundtrip_file', + 'SELECT * FROM dlparq_unsupported'); + +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k bigint, v text); +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int); +SELECT * FROM datalake_parquet_read(:'split_file', 9, 1) AS t (k int, v text); + +-- A row group range that only overflowed arithmetic would let through: the +-- count is rejected rather than turned into a two-billion-entry list. +SELECT * FROM datalake_parquet_read(:'split_file', 1, 2147483647) AS t (k int, v text); + +SELECT datalake_parquet_write(:'split_file', 'SELECT 1', -1); +SELECT datalake_parquet_write(:'split_file', 'SELECT 1', 2000000000); + +-- PostgreSQL's timestamp range runs about 34 years past the last instant Arrow +-- can hold as microseconds from 1970. Writing one of those has to be refused, +-- because the shift would wrap and the value would land 292000 years before the +-- epoch with the write reporting success. +SELECT datalake_parquet_write(:'batch_file', + $$SELECT '294250-01-01 00:00:00'::timestamp$$); + +-- Arrow words this one, and its wording is not ours to depend on. +\set VERBOSITY terse +SELECT * FROM datalake_parquet_read(:'missing_file') AS t (k int, v text); +\set VERBOSITY default + +SET client_min_messages = warning; +DROP VIEW dlparq_roundtrip, dlparq_split; +DROP TABLE dlparq_src, dlparq_pairs, dlparq_unsupported; +RESET client_min_messages; From c75f6e94c7083f1902fde20d0077d63c11679aa5 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Sat, 12 Sep 2026 16:19:19 +0800 Subject: [PATCH 2/9] datalake_fdw: address the Parquet review, field ids and tracked memory Review of #1951, plus what the same pass turned up. The framework is what this settles; type coverage beyond it is left to separate issues. Memory. Every Arrow allocation now goes through one pool (format/arrow_memory_pool.cpp) that reserves with the vmem tracker before allocating and releases after freeing, so statement_mem, the resource group and gp_vmem_protect_limit see Arrow's memory as they see palloc's. The reserve runs under HOLD_INTERRUPTS, because the tracker can elog(ERROR) on its way to saying no and a longjmp out of Arrow's C++ frames is undefined behaviour; pre_buffer is switched off by name, because from Arrow 13 it defaults on and allocates from I/O threads the tracker cannot see. All five pool sites use it, the two hidden defaults included. A refusal is DL_ERR_OUT_OF_MEMORY, ERRCODE_OUT_OF_MEMORY. Files. The writer creates its file with O_CREAT|O_EXCL and hands the descriptor to Arrow, so a failed write can no longer truncate and then delete a file that was already at the path; every failure path goes through parquet_discard, which only ever removes what this writer created. Compression names are lower-cased and asked of Arrow in three steps, so the message says whether Arrow, Parquet or this build is what refused. Columns. A table's columns are matched to a file's by Iceberg field id: the writer stamps PARQUET:field_id into the Parquet schema, ProjectionSet names field ids in output order (ABI 2), a field the file lacks reads as a null-typed column, and an id-less column can never be matched. The reader also accepts the promotions the spec allows, int32 as bigint and float as double precision. Dropped attributes are skipped in the schema, the batch builder and the writer. Types. time and uuid are added; large_utf8 and large_binary are read; INT96 timestamps are coerced to microseconds. Dictionary-encoded columns are refused rather than decoded as their indexes. Both sides require a UTF8 database, and the reader verifies every string before it becomes a text. varchar(n) and char(n) are refused, with numeric and everything else the format cannot store, at CREATE TABLE, through one function the DDL hook and the writer both ask (format/format_types.h); LIKE is refused because its columns are resolved after the hook has run. Smaller: dl_resource.c uses lib/ilist.h; the format name is compared without regard to case like every other option value; the test functions gain compression and field_ids arguments and check PG_NARGS() so that a stale extension definition errors instead of crashing. Left for later, each with an issue: NUMERIC, millisecond and nanosecond timestamp columns, name mapping for files without field ids, and the CI build image. --- contrib/datalake_fdw/Makefile | 1 + .../datalake_fdw/datalake_fdw_test--1.0.sql | 12 +- .../src/am_iceberg/pg_iceberg_extensible.c | 64 ++++ contrib/datalake_fdw/src/common/dl_err.c | 4 + contrib/datalake_fdw/src/common/dl_err.h | 3 + contrib/datalake_fdw/src/common/dl_resource.c | 33 +- .../datalake_fdw/src/format/arrow_builder.cpp | 58 +++- .../datalake_fdw/src/format/arrow_decode.c | 309 ++++++++++++++---- .../datalake_fdw/src/format/arrow_decode.h | 34 +- .../src/format/arrow_memory_pool.cpp | 227 +++++++++++++ .../src/format/arrow_memory_pool.h | 53 +++ .../datalake_fdw/src/format/arrow_support.cpp | 142 ++++++-- .../datalake_fdw/src/format/arrow_support.h | 40 ++- contrib/datalake_fdw/src/format/format.h | 35 +- .../datalake_fdw/src/format/format_registry.c | 12 +- .../datalake_fdw/src/format/format_types.h | 64 ++++ .../src/format/parquet/parquet_read.cpp | 184 ++++++++++- .../src/format/parquet/parquet_write.cpp | 123 +++++-- .../datalake_fdw/src/test/datalake_fdw_test.c | 105 +++++- .../expected/parquet_roundtrip.out | 231 ++++++++++--- .../format_parquet/sql/parquet_roundtrip.sql | 130 ++++++-- .../iceberg_am/expected/iceberg_am_reject.out | 53 ++- .../iceberg_am/sql/iceberg_am_reject.sql | 36 +- 23 files changed, 1654 insertions(+), 299 deletions(-) create mode 100644 contrib/datalake_fdw/src/format/arrow_memory_pool.cpp create mode 100644 contrib/datalake_fdw/src/format/arrow_memory_pool.h create mode 100644 contrib/datalake_fdw/src/format/format_types.h diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile index 0653764581c..a9ad29cd33e 100644 --- a/contrib/datalake_fdw/Makefile +++ b/contrib/datalake_fdw/Makefile @@ -40,6 +40,7 @@ OBJS = \ src/meta/engine_stub/stub_engine.o \ src/format/format_registry.o \ src/format/arrow_support.o \ + src/format/arrow_memory_pool.o \ src/format/arrow_builder.o \ src/format/arrow_decode.o \ src/format/parquet/parquet_format.o \ diff --git a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql index bb76a5023ba..9171dbcd86f 100644 --- a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql +++ b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql @@ -38,14 +38,18 @@ */ CREATE FUNCTION datalake_parquet_write(path text, query text, - row_group_size int DEFAULT 0) + row_group_size int DEFAULT 0, + compression text DEFAULT '') RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; -REVOKE EXECUTE ON FUNCTION datalake_parquet_write(text, text, int) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_parquet_write(text, text, int, text) FROM PUBLIC; +-- field_ids names, for each column of the definition list, the Iceberg field +-- id it is read from; empty reads the file as it is, column for column. CREATE FUNCTION datalake_parquet_read(path text, first_row_group int DEFAULT 0, - n_row_groups int DEFAULT 0) + n_row_groups int DEFAULT 0, + field_ids int[] DEFAULT '{}') RETURNS SETOF record AS 'MODULE_PATHNAME' LANGUAGE C STRICT EXECUTE ON COORDINATOR; -REVOKE EXECUTE ON FUNCTION datalake_parquet_read(text, int, int) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_parquet_read(text, int, int, int[]) FROM PUBLIC; diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c index 26e1f7a68c5..9eec658aa4f 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c @@ -47,13 +47,16 @@ #include "common/backend_registry.h" #include "fmgr.h" #include "foreign/foreign.h" +#include "format/format_types.h" #include "meta/iceberg_meta_engine.h" #include "meta/meta_engine_init.h" #include "miscadmin.h" #include "nodes/makefuncs.h" #include "nodes/parsenodes.h" +#include "parser/parse_type.h" #include "storage/lmgr.h" #include "tcop/utility.h" +#include "utils/builtins.h" #include "utils/fmgroids.h" #include "utils/syscache.h" @@ -85,6 +88,7 @@ static void unlock_create_servers(Oid catalog_srvid, Oid volume_srvid, LOCKMODE lockmode); static void validate_create_binding(const char *catalog_name, const char *volume_name); +static void check_iceberg_columns(CreateStmt *stmt); static void prepare_iceberg_create(CreateStmt *stmt); static void reject_utility_mode_ddl(const char *subject) pg_attribute_noreturn(); static void reject_targeted_operation(const char *operation); @@ -582,6 +586,64 @@ validate_create_binding(const char *catalog_name, const char *volume_name) pg_iceberg_check_server_usage(volume_server->serverid); } +/* + * Every column has to be one a data file can hold, and CREATE TABLE is the + * moment to say so: a table accepted here and refused at its first write is a + * table nothing can ever be put in, and ALTER TABLE cannot reach it either. + * The rule is the format layer's own, asked through the one function both + * sides use, so that what the DDL accepts and what the writer stores cannot + * drift apart. + */ +static void +check_iceberg_columns(CreateStmt *stmt) +{ + ListCell *lc; + + foreach(lc, stmt->tableElts) + { + Node *element = (Node *) lfirst(lc); + + if (IsA(element, ColumnDef)) + { + ColumnDef *coldef = (ColumnDef *) element; + Type type; + Oid typid; + int32 typmod; + const char *refusal; + + /* + * missing_ok, because "serial" and its relatives are not types: + * the parser turns them into an integer column and a sequence + * later, and the integer is fine. Anything else that does not + * resolve is refused by the parser, in its usual words. + */ + type = LookupTypeName(NULL, coldef->typeName, &typmod, true); + if (type == NULL) + continue; + typid = typeTypeId(type); + ReleaseSysCache(type); + + refusal = dl_format_type_refusal(typid, typmod); + if (refusal != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("iceberg: column \"%s\" has type %s, which %s", + coldef->colname, + format_type_with_typemod(typid, typmod), + refusal))); + } + else if (IsA(element, TableLikeClause)) + { + /* + * The columns LIKE copies are resolved by the parser after this + * hook has run, so nothing here can see their types; refusing the + * clause is what keeps an unchecked one out. + */ + pg_iceberg_not_supported("LIKE"); + } + } +} + static void prepare_iceberg_create(CreateStmt *stmt) { @@ -632,6 +694,8 @@ prepare_iceberg_create(CreateStmt *stmt) if (stmt->tablespacename != NULL) pg_iceberg_not_supported("TABLESPACE"); + check_iceberg_columns(stmt); + if (Gp_role != GP_ROLE_EXECUTE) { distributed_by = makeNode(DistributedBy); diff --git a/contrib/datalake_fdw/src/common/dl_err.c b/contrib/datalake_fdw/src/common/dl_err.c index 44f32cf307b..ce0d94a6618 100644 --- a/contrib/datalake_fdw/src/common/dl_err.c +++ b/contrib/datalake_fdw/src/common/dl_err.c @@ -70,6 +70,8 @@ dl_error_sqlstate(DlErrCode code) return ERRCODE_DUPLICATE_TABLE; case DL_ERR_IO: return ERRCODE_IO_ERROR; + case DL_ERR_OUT_OF_MEMORY: + return ERRCODE_OUT_OF_MEMORY; } return ERRCODE_INTERNAL_ERROR; @@ -152,6 +154,8 @@ dl_err_message(DlErrCode code) return "I/O error"; case DL_ERR_INTERNAL: return "internal error"; + case DL_ERR_OUT_OF_MEMORY: + return "out of memory"; } return "unknown error"; diff --git a/contrib/datalake_fdw/src/common/dl_err.h b/contrib/datalake_fdw/src/common/dl_err.h index 7d0e652ed4d..7e27ab28545 100644 --- a/contrib/datalake_fdw/src/common/dl_err.h +++ b/contrib/datalake_fdw/src/common/dl_err.h @@ -55,6 +55,9 @@ typedef enum DlErrCode { DL_ERR_ALREADY_EXISTS, DL_ERR_IO, DL_ERR_INTERNAL, + DL_ERR_OUT_OF_MEMORY, /* the memory limit, not the machine: Arrow's + * allocations are reserved with the vmem + * tracker, and this is what it refused */ } DlErrCode; #define DL_ERR_FIELD_LEN 128 diff --git a/contrib/datalake_fdw/src/common/dl_resource.c b/contrib/datalake_fdw/src/common/dl_resource.c index 691180c2407..776880bae31 100644 --- a/contrib/datalake_fdw/src/common/dl_resource.c +++ b/contrib/datalake_fdw/src/common/dl_resource.c @@ -30,6 +30,7 @@ #include +#include "lib/ilist.h" #include "storage/ipc.h" #include "utils/resowner.h" @@ -37,7 +38,7 @@ typedef struct DlResourceEntry { - struct DlResourceEntry *next; + dlist_node node; ResourceOwner owner; DlResourceRelease release; void *arg; @@ -45,9 +46,10 @@ typedef struct DlResourceEntry /* * There are a handful of these at a time -- one per open data file -- so a list - * walked linearly is the whole structure needed. + * walked linearly is the whole structure needed. The server's own intrusive + * list, as PAX uses it for the same job. */ -static DlResourceEntry *dl_resources; +static dlist_head dl_resources = DLIST_STATIC_INIT(dl_resources); /* * malloc rather than palloc: this outlives the memory context that was current @@ -58,7 +60,7 @@ static void dl_resource_release_callback(ResourceReleasePhase phase, bool isCommit, bool isTopLevel, void *arg) { - DlResourceEntry **link; + dlist_mutable_iter iter; /* * After locks, so that anything the release path might touch is still @@ -69,16 +71,12 @@ dl_resource_release_callback(ResourceReleasePhase phase, bool isCommit, if (phase != RESOURCE_RELEASE_AFTER_LOCKS || proc_exit_inprogress) return; - link = &dl_resources; - while (*link != NULL) + dlist_foreach_modify(iter, &dl_resources) { - DlResourceEntry *entry = *link; + DlResourceEntry *entry = dlist_container(DlResourceEntry, node, iter.cur); if (entry->owner != CurrentResourceOwner) - { - link = &entry->next; continue; - } /* * Reaching here on a commit means the owner released nothing: the @@ -89,7 +87,7 @@ dl_resource_release_callback(ResourceReleasePhase phase, bool isCommit, if (isCommit) elog(WARNING, "datalake_fdw leaked a resource: %p", entry->arg); - *link = entry->next; + dlist_delete(&entry->node); entry->release(entry->arg); free(entry); } @@ -112,8 +110,7 @@ dl_resource_remember(DlResourceRelease release, void *arg) entry->owner = CurrentResourceOwner; entry->release = release; entry->arg = arg; - entry->next = dl_resources; - dl_resources = entry; + dlist_push_tail(&dl_resources, &entry->node); return true; } @@ -121,19 +118,17 @@ dl_resource_remember(DlResourceRelease release, void *arg) void dl_resource_forget(DlResourceRelease release, void *arg) { - DlResourceEntry **link = &dl_resources; + dlist_mutable_iter iter; - while (*link != NULL) + dlist_foreach_modify(iter, &dl_resources) { - DlResourceEntry *entry = *link; + DlResourceEntry *entry = dlist_container(DlResourceEntry, node, iter.cur); if (entry->release == release && entry->arg == arg) { - *link = entry->next; + dlist_delete(&entry->node); free(entry); return; } - - link = &entry->next; } } diff --git a/contrib/datalake_fdw/src/format/arrow_builder.cpp b/contrib/datalake_fdw/src/format/arrow_builder.cpp index 3dd68f4006a..34183b950ae 100644 --- a/contrib/datalake_fdw/src/format/arrow_builder.cpp +++ b/contrib/datalake_fdw/src/format/arrow_builder.cpp @@ -37,12 +37,14 @@ #include "common/dl_resource.h" #include "common/dl_wrappers.h" #include "format/arrow_builder.h" +#include "format/arrow_memory_pool.h" extern "C" { #include "catalog/pg_type.h" #include "utils/date.h" #include "utils/timestamp.h" +#include "utils/uuid.h" #include "varatt.h" } @@ -58,6 +60,15 @@ extern "C" struct DlArrowBuilderData { std::shared_ptr schema; + int natts; /* of the descriptor, dropped ones included */ + + /* + * One entry per field of the schema, which is one per live attribute: the + * attribute's position in the descriptor, its type, and its builder. The + * descriptor and the schema disagree about positions as soon as a column + * has been dropped, and this is where that is reconciled. + */ + std::vector attnos; std::vector types; std::vector> builders; int64_t nrows; @@ -94,7 +105,6 @@ dl_append_datum(arrow::ArrayBuilder *builder, Oid atttypid, Datum value) case TEXTOID: case VARCHAROID: - case BPCHAROID: { struct varlena *v = (struct varlena *) DatumGetPointer(value); @@ -127,6 +137,11 @@ dl_append_datum(arrow::ArrayBuilder *builder, Oid atttypid, Datum value) ->Append(date + DL_EPOCH_DELTA_DAYS); } + /* Microseconds since midnight on both sides; nothing to shift. */ + case TIMEOID: + return static_cast(builder) + ->Append(DatumGetTimeADT(value)); + case TIMESTAMPOID: case TIMESTAMPTZOID: { @@ -155,6 +170,10 @@ dl_append_datum(arrow::ArrayBuilder *builder, Oid atttypid, Datum value) ->Append(ts + DL_EPOCH_DELTA_USECS); } + case UUIDOID: + return static_cast(builder) + ->Append(DatumGetUUIDP(value)->data); + default: /* @@ -196,25 +215,40 @@ dl_arrow_builder_open(void *tupdesc_arg, DlArrowBuilder *out) TupleDesc tupdesc = (TupleDesc) tupdesc_arg; std::unique_ptr builder(new DlArrowBuilderData()); - builder->schema = DlArrowSchemaFromTupleDesc(tupdesc); + /* + * The field ids do not matter to a batch -- they travel with the + * writer's schema, which was built from the same descriptor -- so the + * default numbering is fine here. + */ + builder->schema = DlArrowSchemaFromTupleDesc(tupdesc, nullptr); if (builder->schema == nullptr) return DL_ERR_NOT_SUPPORTED; /* detail already recorded */ + builder->natts = tupdesc->natts; builder->nrows = 0; + builder->attnos.reserve(tupdesc->natts); builder->types.reserve(tupdesc->natts); builder->builders.reserve(tupdesc->natts); - for (int i = 0; i < tupdesc->natts; i++) + for (int attno = 0; attno < tupdesc->natts; attno++) { + Form_pg_attribute attr = TupleDescAttr(tupdesc, attno); std::unique_ptr column; - arrow::Status status = arrow::MakeBuilder(arrow::default_memory_pool(), - builder->schema->field(i)->type(), - &column); + arrow::Status status; + + /* The schema skipped it; see DlArrowSchemaFromTupleDesc(). */ + if (attr->attisdropped) + continue; + status = arrow::MakeBuilder(DlArrowMemoryPool(), + builder->schema->field( + (int) builder->builders.size())->type(), + &column); if (!status.ok()) return DlArrowStatus(status, "create an Arrow array builder"); - builder->types.push_back(TupleDescAttr(tupdesc, i)->atttypid); + builder->attnos.push_back(attno); + builder->types.push_back(attr->atttypid); builder->builders.push_back(std::move(column)); } @@ -245,7 +279,8 @@ dl_arrow_builder_append(DlArrowBuilder builder, const Datum *values, if (builder == NULL || values == NULL || nulls == NULL) return DL_ARG_ERROR("append_row"); - if (nvalues != (int) builder->builders.size()) + /* A row is as wide as the descriptor, dropped attributes included. */ + if (nvalues != builder->natts) { dl_error_set(DL_ERR_INTERNAL, "append an Arrow row", NULL, "the row has a different number of columns than the batch"); @@ -254,12 +289,13 @@ dl_arrow_builder_append(DlArrowBuilder builder, const Datum *values, DL_ABI_GUARD_BEGIN { - for (int i = 0; i < nvalues; i++) + for (size_t i = 0; i < builder->builders.size(); i++) { - arrow::Status status = nulls[i] + int attno = builder->attnos[i]; + arrow::Status status = nulls[attno] ? builder->builders[i]->AppendNull() : dl_append_datum(builder->builders[i].get(), builder->types[i], - values[i]); + values[attno]); if (!status.ok()) return DlArrowStatus(status, "append a value to an Arrow array"); diff --git a/contrib/datalake_fdw/src/format/arrow_decode.c b/contrib/datalake_fdw/src/format/arrow_decode.c index 6574eeb2a9b..c733eb0b318 100644 --- a/contrib/datalake_fdw/src/format/arrow_decode.c +++ b/contrib/datalake_fdw/src/format/arrow_decode.c @@ -31,13 +31,17 @@ #include #include "catalog/pg_type.h" +#include "mb/pg_wchar.h" #include "utils/builtins.h" #include "utils/date.h" #include "utils/fmgrprotos.h" +#include "utils/memutils.h" #include "utils/timestamp.h" +#include "utils/uuid.h" #include "varatt.h" #include "format/arrow_decode.h" +#include "format/format_types.h" /* * The same shift as in arrow_builder.cpp, in the other direction: PostgreSQL @@ -47,10 +51,12 @@ #define DL_EPOCH_DELTA_USECS (((int64) DL_EPOCH_DELTA_DAYS) * USECS_PER_DAY) /* - * Arrow spells its types as a short string. Only the ones a column of ours can - * be stored as are listed; anything else is a file we did not write, or one - * written by a version that knows more types than this one. + * Arrow spells its types as a short string. Listed are the ones a column of + * ours can be stored as, plus the ones it can be read from without loss; + * anything else is a file we did not write, or one written by a version that + * knows more types than this one. */ +#define DL_ARROW_FORMAT_NULL "n" #define DL_ARROW_FORMAT_BOOL "b" #define DL_ARROW_FORMAT_INT16 "s" #define DL_ARROW_FORMAT_INT32 "i" @@ -58,31 +64,73 @@ #define DL_ARROW_FORMAT_FLOAT32 "f" #define DL_ARROW_FORMAT_FLOAT64 "g" #define DL_ARROW_FORMAT_UTF8 "u" +#define DL_ARROW_FORMAT_LARGE_UTF8 "U" #define DL_ARROW_FORMAT_BINARY "z" +#define DL_ARROW_FORMAT_LARGE_BINARY "Z" #define DL_ARROW_FORMAT_DATE32 "tdD" +#define DL_ARROW_FORMAT_TIME64_US "ttu" +#define DL_ARROW_FORMAT_UUID "w:16" /* fixed-size binary of 16 */ /* A timestamp is "tsu:" followed by the time zone, which may be empty. */ #define DL_ARROW_FORMAT_TIMESTAMP_US "tsu:" static DlErrCode -dl_arrow_decode_refuse(const char *arrow_format, Oid atttypid) +dl_arrow_decode_refuse(const char *arrow_format, Oid atttypid, int32 atttypmod) { char message[256]; snprintf(message, sizeof(message), "a column stored as Arrow type \"%s\" cannot be read as %s", arrow_format == NULL ? "" : arrow_format, - format_type_be(atttypid)); + format_type_with_typemod(atttypid, atttypmod)); dl_error_set(DL_ERR_NOT_SUPPORTED, "decode an Arrow column", NULL, message); return DL_ERR_NOT_SUPPORTED; } +static bool +dl_arrow_format_in(const char *format, const char *const *accepted) +{ + for (; *accepted != NULL; accepted++) + { + if (strcmp(format, *accepted) == 0) + return true; + } + + return false; +} + DlErrCode -dl_arrow_decode_check(const struct ArrowSchema *field, Oid atttypid) +dl_arrow_decode_check(const struct ArrowSchema *field, Oid atttypid, + int32 atttypmod) { + /* + * The type's own storage first, then what it can be read from without + * loss. Iceberg lets a column be promoted from int to long and from float + * to double, and a file written before the promotion holds the old type; + * the values fit, so they are read. Nothing narrows. + */ + static const char *const bool_formats[] = {DL_ARROW_FORMAT_BOOL, NULL}; + static const char *const int2_formats[] = {DL_ARROW_FORMAT_INT16, NULL}; + static const char *const int4_formats[] = {DL_ARROW_FORMAT_INT32, + DL_ARROW_FORMAT_INT16, NULL}; + static const char *const int8_formats[] = {DL_ARROW_FORMAT_INT64, + DL_ARROW_FORMAT_INT32, DL_ARROW_FORMAT_INT16, NULL}; + static const char *const float4_formats[] = {DL_ARROW_FORMAT_FLOAT32, NULL}; + static const char *const float8_formats[] = {DL_ARROW_FORMAT_FLOAT64, + DL_ARROW_FORMAT_FLOAT32, NULL}; + static const char *const text_formats[] = {DL_ARROW_FORMAT_UTF8, + DL_ARROW_FORMAT_LARGE_UTF8, NULL}; + static const char *const bytea_formats[] = {DL_ARROW_FORMAT_BINARY, + DL_ARROW_FORMAT_LARGE_BINARY, NULL}; + static const char *const date_formats[] = {DL_ARROW_FORMAT_DATE32, NULL}; + static const char *const time_formats[] = {DL_ARROW_FORMAT_TIME64_US, NULL}; + static const char *const uuid_formats[] = {DL_ARROW_FORMAT_UUID, NULL}; + const char *format; - const char *expected; + const char *refusal; + const char *const *accepted; + char message[256]; if (field == NULL || field->format == NULL) { @@ -93,36 +141,98 @@ dl_arrow_decode_check(const struct ArrowSchema *field, Oid atttypid) format = field->format; + /* + * A type this module could not have stored is not one it reads either, and + * the rule about modifiers is the one CREATE TABLE applies: a varchar(n) + * column cannot exist in a lake table, so a file cannot be read as one. + */ + refusal = dl_format_type_refusal(atttypid, atttypmod); + if (refusal != NULL) + { + snprintf(message, sizeof(message), "a column cannot be read as %s, which %s", + format_type_with_typemod(atttypid, atttypmod), refusal); + dl_error_set(DL_ERR_NOT_SUPPORTED, "decode an Arrow column", NULL, message); + return DL_ERR_NOT_SUPPORTED; + } + + /* + * A dictionary-encoded column describes its indexes in the format string + * and keeps its values in a separate array. The indexes are integers, so + * without this a dictionary of strings would pass as an integer column and + * every value read would be a position in a table nobody looked at. Arrow + * restores this encoding from a note pyarrow leaves in the file whenever + * pandas wrote a categorical, so it is not a rare file. + */ + if (field->dictionary != NULL) + { + snprintf(message, sizeof(message), + "a dictionary-encoded column cannot be read as %s", + format_type_with_typemod(atttypid, atttypmod)); + dl_error_set(DL_ERR_NOT_SUPPORTED, "decode an Arrow column", NULL, message); + return DL_ERR_NOT_SUPPORTED; + } + + /* + * A column of the null type has no values, only nulls, and every type can + * hold those. It is what the reader produces for a field the file does + * not have -- a column added to the table after the file was written. + */ + if (strcmp(format, DL_ARROW_FORMAT_NULL) == 0) + return DL_OK; + switch (atttypid) { case BOOLOID: - expected = DL_ARROW_FORMAT_BOOL; + accepted = bool_formats; break; case INT2OID: - expected = DL_ARROW_FORMAT_INT16; + accepted = int2_formats; break; case INT4OID: - expected = DL_ARROW_FORMAT_INT32; + accepted = int4_formats; break; case INT8OID: - expected = DL_ARROW_FORMAT_INT64; + accepted = int8_formats; break; case FLOAT4OID: - expected = DL_ARROW_FORMAT_FLOAT32; + accepted = float4_formats; break; case FLOAT8OID: - expected = DL_ARROW_FORMAT_FLOAT64; + accepted = float8_formats; break; + case TEXTOID: case VARCHAROID: - case BPCHAROID: - expected = DL_ARROW_FORMAT_UTF8; + + /* + * The file's strings are UTF-8. Into a database of any other + * encoding they would have to be converted, and until they are, + * copying the bytes would make text values no function of the + * server can interpret. + */ + if (GetDatabaseEncoding() != PG_UTF8) + { + snprintf(message, sizeof(message), + "a lake table's strings are UTF-8, and this database's " + "encoding is %s", GetDatabaseEncodingName()); + dl_error_set(DL_ERR_NOT_SUPPORTED, "decode an Arrow column", NULL, + message); + return DL_ERR_NOT_SUPPORTED; + } + accepted = text_formats; break; + case BYTEAOID: - expected = DL_ARROW_FORMAT_BINARY; + accepted = bytea_formats; break; case DATEOID: - expected = DL_ARROW_FORMAT_DATE32; + accepted = date_formats; + break; + case TIMEOID: + accepted = time_formats; + break; + case UUIDOID: + accepted = uuid_formats; break; case TIMESTAMPOID: @@ -132,7 +242,7 @@ dl_arrow_decode_check(const struct ArrowSchema *field, Oid atttypid) size_t prefix_len = strlen(DL_ARROW_FORMAT_TIMESTAMP_US); if (strncmp(format, DL_ARROW_FORMAT_TIMESTAMP_US, prefix_len) != 0) - return dl_arrow_decode_refuse(format, atttypid); + return dl_arrow_decode_refuse(format, atttypid, atttypmod); /* * Arrow stores a zoned timestamp as the instant in UTC and @@ -144,17 +254,17 @@ dl_arrow_decode_check(const struct ArrowSchema *field, Oid atttypid) */ zone = format + prefix_len; if ((zone[0] != '\0') != (atttypid == TIMESTAMPTZOID)) - return dl_arrow_decode_refuse(format, atttypid); + return dl_arrow_decode_refuse(format, atttypid, atttypmod); return DL_OK; } default: - return dl_arrow_decode_refuse(format, atttypid); + return dl_arrow_decode_refuse(format, atttypid, atttypmod); } - if (strcmp(format, expected) != 0) - return dl_arrow_decode_refuse(format, atttypid); + if (!dl_arrow_format_in(format, accepted)) + return dl_arrow_decode_refuse(format, atttypid, atttypmod); return DL_OK; } @@ -198,26 +308,75 @@ dl_arrow_out_of_range(const char *what) } /* - * A variable-length value: an offsets buffer of int32 and one run of bytes. - * Both text and bytea are laid out this way, and differ only in the header the - * copy gets. + * An integer of whichever width the column has, widened. Only reached for a + * width dl_arrow_decode_check() accepted for the target type, so nothing here + * can truncate. */ -static void -dl_arrow_varlen(const struct ArrowArray *column, int64_t row, - const char **data, int32 *length) +static int64 +dl_arrow_integer(const struct ArrowArray *column, const char *format, + int64_t row) +{ + switch (format[0]) + { + case 's': + return DL_ARROW_VALUES(column, int16)[row]; + case 'i': + return DL_ARROW_VALUES(column, int32)[row]; + default: + return DL_ARROW_VALUES(column, int64)[row]; + } +} + +/* + * A variable-length value: an offsets buffer and one run of bytes. Text and + * bytea are laid out this way and differ only in the header the copy gets; the + * "large" forms differ only in the width of the offsets, which is what lets a + * single column exceed 2 GB -- and what lets a single value exceed what a + * PostgreSQL varlena can hold, which is why the length is checked. + */ +static DlErrCode +dl_arrow_varlen(const struct ArrowArray *column, const char *format, + int64_t row, const char **data, int64 *length) { - const int32 *offsets = DL_ARROW_VALUES(column, int32); const char *bytes = (const char *) column->buffers[2]; + int64 start; + int64 end; + + if (format[0] == 'U' || format[0] == 'Z') + { + const int64 *offsets = DL_ARROW_VALUES(column, int64); + + start = offsets[row]; + end = offsets[row + 1]; + } + else + { + const int32 *offsets = DL_ARROW_VALUES(column, int32); - *data = bytes + offsets[row]; - *length = offsets[row + 1] - offsets[row]; + start = offsets[row]; + end = offsets[row + 1]; + } + + if (end - start > (int64) (MaxAllocSize - VARHDRSZ)) + { + dl_error_set(DL_ERR_INVALID_OPTION, "decode an Arrow column", NULL, + "the file holds a value longer than a PostgreSQL value can be"); + return DL_ERR_INVALID_OPTION; + } + + *data = bytes + start; + *length = end - start; + + return DL_OK; } DlErrCode -dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, - Oid atttypid, int32 atttypmod, - Datum *value, bool *isnull) +dl_arrow_decode_value(const struct ArrowSchema *field, + const struct ArrowArray *column, int64_t row, + Oid atttypid, Datum *value, bool *isnull) { + const char *format = field->format; + *value = (Datum) 0; *isnull = true; @@ -228,6 +387,13 @@ dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, return DL_ERR_INTERNAL; } + /* + * The null type has no buffers at all, not even a validity bitmap, so it + * is answered before anything looks for one. + */ + if (strcmp(format, DL_ARROW_FORMAT_NULL) == 0) + return DL_OK; + if (dl_arrow_value_is_null(column, row)) return DL_OK; @@ -246,57 +412,60 @@ dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, } case INT2OID: - *value = Int16GetDatum(DL_ARROW_VALUES(column, int16)[row]); + *value = Int16GetDatum((int16) dl_arrow_integer(column, format, row)); return DL_OK; case INT4OID: - *value = Int32GetDatum(DL_ARROW_VALUES(column, int32)[row]); + *value = Int32GetDatum((int32) dl_arrow_integer(column, format, row)); return DL_OK; case INT8OID: - *value = Int64GetDatum(DL_ARROW_VALUES(column, int64)[row]); + *value = Int64GetDatum(dl_arrow_integer(column, format, row)); return DL_OK; case FLOAT4OID: *value = Float4GetDatum(DL_ARROW_VALUES(column, float)[row]); return DL_OK; case FLOAT8OID: - *value = Float8GetDatum(DL_ARROW_VALUES(column, double)[row]); + if (format[0] == 'f') + *value = Float8GetDatum((double) DL_ARROW_VALUES(column, float)[row]); + else + *value = Float8GetDatum(DL_ARROW_VALUES(column, double)[row]); return DL_OK; case TEXTOID: case VARCHAROID: - case BPCHAROID: { const char *data; - int32 length; + int64 length; + DlErrCode rc; - dl_arrow_varlen(column, row, &data, &length); - *value = PointerGetDatum(cstring_to_text_with_len(data, length)); + rc = dl_arrow_varlen(column, format, row, &data, &length); + if (rc != DL_OK) + return rc; /* - * The file records the bytes and nothing about the length the - * column was declared with, so the value is put through the - * same coercion an inserted one would be: char(n) comes back - * padded to n, and a value too long for a varchar(n) is an - * error rather than something the executor has to meet later. + * PostgreSQL's own input paths verify every string before it + * becomes a text, and this is an input path: the file says its + * strings are UTF-8, and a file this module did not write may + * be lying. An invalid sequence is refused here, where it can + * be named, rather than met later by whichever function first + * walks the characters. */ - if (atttypmod >= 0 && atttypid == BPCHAROID) - *value = DirectFunctionCall3(bpchar, *value, - Int32GetDatum(atttypmod), - BoolGetDatum(false)); - else if (atttypmod >= 0 && atttypid == VARCHAROID) - *value = DirectFunctionCall3(varchar, *value, - Int32GetDatum(atttypmod), - BoolGetDatum(false)); + pg_verifymbstr(data, (int) length, false); + *value = PointerGetDatum(cstring_to_text_with_len(data, (int) length)); return DL_OK; } case BYTEAOID: { const char *data; - int32 length; + int64 length; bytea *result; + DlErrCode rc; + + rc = dl_arrow_varlen(column, format, row, &data, &length); + if (rc != DL_OK) + return rc; - dl_arrow_varlen(column, row, &data, &length); result = (bytea *) palloc(VARHDRSZ + length); SET_VARSIZE(result, VARHDRSZ + length); memcpy(VARDATA(result), data, length); @@ -326,6 +495,18 @@ dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, return DL_OK; } + case TIMEOID: + { + int64 micros = DL_ARROW_VALUES(column, int64)[row]; + + /* PostgreSQL admits 24:00:00, so the upper bound is inclusive. */ + if (micros < 0 || micros > USECS_PER_DAY) + return dl_arrow_out_of_range("time"); + + *value = TimeADTGetDatum(micros); + return DL_OK; + } + case TIMESTAMPOID: case TIMESTAMPTZOID: { @@ -343,6 +524,18 @@ dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, return DL_OK; } + case UUIDOID: + { + /* Fixed-size binary: the values buffer is rows of 16 bytes. */ + const uint8 *bytes = ((const uint8 *) column->buffers[1]) + + (column->offset + row) * UUID_LEN; + pg_uuid_t *uuid = (pg_uuid_t *) palloc(sizeof(pg_uuid_t)); + + memcpy(uuid->data, bytes, UUID_LEN); + *value = UUIDPGetDatum(uuid); + return DL_OK; + } + default: /* @@ -350,6 +543,6 @@ dl_arrow_decode_value(const struct ArrowArray *column, int64_t row, * switch does not list. */ *isnull = true; - return dl_arrow_decode_refuse(NULL, atttypid); + return dl_arrow_decode_refuse(format, atttypid, -1); } } diff --git a/contrib/datalake_fdw/src/format/arrow_decode.h b/contrib/datalake_fdw/src/format/arrow_decode.h index 184a1edff2a..72034de957d 100644 --- a/contrib/datalake_fdw/src/format/arrow_decode.h +++ b/contrib/datalake_fdw/src/format/arrow_decode.h @@ -20,12 +20,6 @@ * arrow_decode.h * PostgreSQL values out of an Arrow batch. * - * The read half of the boundary the format layer is built on, and the mirror of - * arrow_builder.h. This side is C: turning a column into Datums means - * allocating text and bytea, an allocation can fail, and a failure in - * PostgreSQL unwinds with longjmp -- which is safe here and would not be if it - * had to pass through C++ frames on the way out. - * * It reads the buffers of the Arrow C data interface directly rather than * handing them back to Arrow, which keeps the read path free of C++ and makes * it a real check on what our own writer exports. @@ -49,27 +43,31 @@ * Called once per column per batch: the answer depends only on the schema, and * checking it per value would be the same answer several million times. * + * What is accepted is what the type can hold without loss: its own Arrow type, + * the narrower ones Iceberg lets a column be promoted from -- an int column + * read as bigint, a float as double precision -- and Arrow's null type, which + * is what a column the file does not have comes back as. The modifier is + * judged by the same rule CREATE TABLE applies, so a varchar(n) is refused + * here for the same reason it could not have been created. + * * `field` is one child of the batch's schema. */ extern DlErrCode dl_arrow_decode_check(const struct ArrowSchema *field, - Oid atttypid); + Oid atttypid, int32 atttypmod); /* * One value. Only valid for a column dl_arrow_decode_check() accepted, which - * is what lets this trust the buffer layout instead of re-deriving it. - * - * Values that point at memory -- text, bytea -- are copied into the current - * memory context, because the batch is released long before the tuples built - * from it are done with. + * is what lets this trust the buffer layout instead of re-deriving it -- and + * `field` is how it knows which of the accepted layouts this column has. * - * `atttypmod` is the modifier the column was declared with, or -1. A file this - * module did not write has no idea what it was, so a char(n) in it need not be - * padded to n and a varchar(n) need not be within n; without applying it, a - * value that breaks the type's own rules would reach the executor. + * Values that point at memory -- text, bytea, uuid -- are copied into the + * current memory context, because the batch is released long before the + * tuples built from it are done with. A string is verified to be what the + * file claims, UTF-8, before it becomes a text: the file is not ours. */ -extern DlErrCode dl_arrow_decode_value(const struct ArrowArray *column, +extern DlErrCode dl_arrow_decode_value(const struct ArrowSchema *field, + const struct ArrowArray *column, int64_t row, Oid atttypid, - int32 atttypmod, Datum *value, bool *isnull); #endif /* DL_ARROW_DECODE_H */ diff --git a/contrib/datalake_fdw/src/format/arrow_memory_pool.cpp b/contrib/datalake_fdw/src/format/arrow_memory_pool.cpp new file mode 100644 index 00000000000..2e83cc7d710 --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_memory_pool.cpp @@ -0,0 +1,227 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_memory_pool.cpp + * An Arrow memory pool whose bytes the vmem tracker knows about. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_memory_pool.cpp + * + *------------------------------------------------------------------------- + */ + +#include + +#include +#include +#include + +#include "format/arrow_memory_pool.h" + +extern "C" +{ +#include "postgres.h" + +#include "miscadmin.h" +#include "utils/vmem_tracker.h" +} + +namespace +{ + +const char * +dl_vmem_failure(MemoryAllocationStatus status) +{ + switch (status) + { + case MemoryFailure_VmemExhausted: + return "the segment's memory limit (gp_vmem_protect_limit) is exhausted"; + case MemoryFailure_QueryMemoryExhausted: + return "the query's memory limit is exhausted"; + case MemoryFailure_ResourceGroupMemoryExhausted: + return "the resource group's memory limit is exhausted"; + case MemoryFailure_SystemMemoryExhausted: + return "the system is out of memory"; + default: + return "the memory could not be reserved"; + } +} + +/* + * Reserves with the tracker before allocating and releases after freeing, so + * that what the tracker counts is never less than what Arrow holds. + * + * ProxyMemoryPool rather than MemoryPool as the base: the statistics methods + * are pure virtual in some Arrow versions and absent in others, and the proxy + * implements whichever set its version has. Only the three that move memory + * are overridden, and those changed signature at Arrow 11, which is what the + * version guard is about. + */ +class DlTrackedMemoryPool : public arrow::ProxyMemoryPool +{ +public: + explicit DlTrackedMemoryPool(arrow::MemoryPool *pool) + : arrow::ProxyMemoryPool(pool) + { + } + +#if ARROW_VERSION_MAJOR >= 11 + using arrow::MemoryPool::Allocate; + using arrow::MemoryPool::Reallocate; + using arrow::MemoryPool::Free; + + arrow::Status Allocate(int64_t size, int64_t alignment, + uint8_t **out) override + { + return TrackedAllocate(size, [&]() { + return arrow::ProxyMemoryPool::Allocate(size, alignment, out); + }); + } + + arrow::Status Reallocate(int64_t old_size, int64_t new_size, + int64_t alignment, uint8_t **ptr) override + { + return TrackedReallocate(old_size, new_size, [&]() { + return arrow::ProxyMemoryPool::Reallocate(old_size, new_size, + alignment, ptr); + }); + } + + void Free(uint8_t *buffer, int64_t size, int64_t alignment) override + { + arrow::ProxyMemoryPool::Free(buffer, size, alignment); + Release(size); + } +#else + arrow::Status Allocate(int64_t size, uint8_t **out) override + { + return TrackedAllocate(size, [&]() { + return arrow::ProxyMemoryPool::Allocate(size, out); + }); + } + + arrow::Status Reallocate(int64_t old_size, int64_t new_size, + uint8_t **ptr) override + { + return TrackedReallocate(old_size, new_size, [&]() { + return arrow::ProxyMemoryPool::Reallocate(old_size, new_size, ptr); + }); + } + + void Free(uint8_t *buffer, int64_t size) override + { + arrow::ProxyMemoryPool::Free(buffer, size); + Release(size); + } +#endif + + std::string backend_name() const override + { + return "vmem-tracked " + arrow::ProxyMemoryPool::backend_name(); + } + +private: + /* + * The tracker may decide, on the way to saying no, that this session is + * the one to cancel, or that a pending interrupt should be serviced now; + * either is an elog(ERROR), and a longjmp from here would go through + * Arrow's C++ frames, which is undefined behaviour. Holding interrupts + * turns both into "not now": the tracker then only returns a status, and + * the interrupt is taken at the next CHECK_FOR_INTERRUPTS() in C code, + * which every loop that calls into Arrow has. + */ + static arrow::Status + Reserve(int64_t bytes) + { + MemoryAllocationStatus status; + + if (bytes <= 0) + return arrow::Status::OK(); + + HOLD_INTERRUPTS(); + status = VmemTracker_ReserveVmem(bytes); + RESUME_INTERRUPTS(); + + if (status != MemoryAllocation_Success) + return arrow::Status::OutOfMemory("could not reserve ", bytes, + " bytes for Arrow: ", + dl_vmem_failure(status)); + + return arrow::Status::OK(); + } + + static void + Release(int64_t bytes) + { + if (bytes > 0) + VmemTracker_ReleaseVmem(bytes); + } + + template + static arrow::Status + TrackedAllocate(int64_t size, Allocate allocate) + { + arrow::Status status = Reserve(size); + + if (!status.ok()) + return status; + + status = allocate(); + if (!status.ok()) + Release(size); + + return status; + } + + /* + * Growth is reserved before the move and shrinkage released after it, the + * same order gp_realloc() uses: a failed realloc then leaves the tracker + * where it was on both paths. + */ + template + static arrow::Status + TrackedReallocate(int64_t old_size, int64_t new_size, Reallocate reallocate) + { + int64_t growth = new_size - old_size; + arrow::Status status = Reserve(growth); + + if (!status.ok()) + return status; + + status = reallocate(); + if (!status.ok()) + { + Release(growth); + return status; + } + + Release(-growth); + return status; + } +}; + +} /* namespace */ + +arrow::MemoryPool * +DlArrowMemoryPool(void) +{ + static DlTrackedMemoryPool pool(arrow::default_memory_pool()); + + return &pool; +} diff --git a/contrib/datalake_fdw/src/format/arrow_memory_pool.h b/contrib/datalake_fdw/src/format/arrow_memory_pool.h new file mode 100644 index 00000000000..6087a926403 --- /dev/null +++ b/contrib/datalake_fdw/src/format/arrow_memory_pool.h @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * arrow_memory_pool.h + * The allocator every Arrow allocation in this module goes through. + * + * Arrow allocates from its own pool, not from palloc, so nothing it holds is + * visible to the server's memory accounting on its own: a writer buffering a + * row group, or a reader decoding one, could push a segment past its limit + * while looking small to the resource manager. This pool reserves every byte + * with the vmem tracker before handing it out, so statement_mem, the resource + * group and gp_vmem_protect_limit see Arrow's memory as they see palloc's. + * + * Every place that would name arrow::default_memory_pool() names this instead + * -- including the two that would otherwise get it by default, the Parquet + * ReaderProperties and the WriterProperties builder. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/arrow_memory_pool.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_ARROW_MEMORY_POOL_H +#define DL_ARROW_MEMORY_POOL_H + +#include + +/* + * The one pool. It wraps Arrow's default and only adds the accounting, so + * what it hands out is aligned and freed exactly as the default's would be. + * Only for the backend's own thread: the vmem tracker is not thread-safe, which + * is one more reason the readers here never let Arrow use a thread pool. + */ +extern arrow::MemoryPool *DlArrowMemoryPool(void); + +#endif /* DL_ARROW_MEMORY_POOL_H */ diff --git a/contrib/datalake_fdw/src/format/arrow_support.cpp b/contrib/datalake_fdw/src/format/arrow_support.cpp index 8addade807f..a4e951eeb9f 100644 --- a/contrib/datalake_fdw/src/format/arrow_support.cpp +++ b/contrib/datalake_fdw/src/format/arrow_support.cpp @@ -32,6 +32,8 @@ * Abs, Min and Max as macros, and a template header has no way to defend * itself against them. */ +#include +#include #include #include @@ -42,6 +44,8 @@ extern "C" { #include "catalog/pg_type.h" +#include "mb/pg_wchar.h" +#include "utils/uuid.h" } DlErrCode @@ -65,6 +69,9 @@ DlArrowStatus(const arrow::Status &status, const char *operation) case arrow::StatusCode::KeyError: code = DL_ERR_INVALID_OPTION; break; + case arrow::StatusCode::OutOfMemory: + code = DL_ERR_OUT_OF_MEMORY; + break; default: code = DL_ERR_INTERNAL; break; @@ -94,15 +101,14 @@ DlArrowTypeForPgType(Oid atttypid) return arrow::float64(); /* - * All three of PostgreSQL's string types are one Arrow type: the - * length limit is a constraint PostgreSQL enforces before a value - * reaches us, and Parquet has nowhere to record it. A char(n) - * value arrives already padded, so what is written is what - * PostgreSQL stores. + * Both of PostgreSQL's unbounded string types are the one string + * type a lake table has. The bounded forms -- varchar(n), char(n) + * -- are refused by dl_format_type_refusal() rather than mapped: + * Parquet has nowhere to record the bound, so a file written by + * anything else could hold values that break it. */ case TEXTOID: case VARCHAROID: - case BPCHAROID: return arrow::utf8(); case BYTEAOID: @@ -110,6 +116,13 @@ DlArrowTypeForPgType(Oid atttypid) case DATEOID: return arrow::date32(); + /* + * PostgreSQL keeps a time as microseconds since midnight, which is + * exactly what an Iceberg time is; no epoch to shift. + */ + case TIMEOID: + return arrow::time64(arrow::TimeUnit::MICRO); + /* * PostgreSQL keeps both timestamp types in microseconds, so * microseconds is the unit that loses nothing. timestamptz is a @@ -122,45 +135,94 @@ DlArrowTypeForPgType(Oid atttypid) case TIMESTAMPTZOID: return arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"); + /* + * An Iceberg uuid is 16 bytes, and so is PostgreSQL's -- the same + * 16 bytes, in the same order. + */ + case UUIDOID: + return arrow::fixed_size_binary(UUID_LEN); + default: return nullptr; } } +extern "C" const char * +dl_format_type_refusal(Oid typid, int32 typmod) +{ + if (typid == BPCHAROID) + return "is padded to a declared length, and a lake table has no way to " + "store that; use text"; + + if (typid == VARCHAROID && typmod >= 0) + return "has a length limit, and a lake table has no way to store one; " + "use text or varchar without a length"; + + if (DlArrowTypeForPgType(typid) == nullptr) + return "cannot be stored in a lake table"; + + return nullptr; +} + std::shared_ptr -DlArrowSchemaFromTupleDesc(TupleDesc tupdesc) +DlArrowSchemaFromTupleDesc(TupleDesc tupdesc, const int32_t *field_ids) { std::vector> fields; + char message[256]; + int32_t next_field_id = 1; + + /* + * Parquet defines its string type as UTF-8 and its column names likewise, + * and every other reader of the file will take them as such. Bytes in any + * other encoding would be written under a label that lies about them, so + * the question is settled once, here, for the whole descriptor. + */ + if (GetDatabaseEncoding() != PG_UTF8) + { + snprintf(message, sizeof(message), + "a lake table stores strings and column names as UTF-8, and " + "this database's encoding is %s", GetDatabaseEncodingName()); + dl_error_set(DL_ERR_NOT_SUPPORTED, "arrow schema", nullptr, message); + return nullptr; + } fields.reserve(tupdesc->natts); for (int i = 0; i < tupdesc->natts; i++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, i); + const char *refusal; + int32_t field_id; /* - * A dropped column has no type to write and no name worth recording. - * Leaving a placeholder in the file would keep column positions - * aligned, but nothing reads such a file yet, so refusing is the - * answer that cannot be silently wrong. + * A dropped column is a tombstone in the descriptor, not a column of + * the table: it has no name and no type, and the file has no place + * for it. Skipping it is what lets a table stay writable after ALTER + * TABLE DROP COLUMN; the batch builder skips the same attributes. */ if (attr->attisdropped) + continue; + + refusal = dl_format_type_refusal(attr->atttypid, attr->atttypmod); + if (refusal != nullptr) { - dl_error_set(DL_ERR_NOT_SUPPORTED, "arrow schema", nullptr, - "a dropped column cannot be written to a data file"); + snprintf(message, sizeof(message), "column \"%s\" has a type that %s", + NameStr(attr->attname), refusal); + dl_error_set(DL_ERR_NOT_SUPPORTED, "arrow schema", nullptr, message); return nullptr; } - std::shared_ptr type = DlArrowTypeForPgType(attr->atttypid); - - if (type == nullptr) + /* + * Parquet's bridge writes a negative id as no id at all, silently. A + * caller that hands one over has a bug, and a file whose columns can + * never be matched is not the way to find out about it. + */ + field_id = field_ids != nullptr ? field_ids[i] : next_field_id++; + if (field_id < 0) { - std::string message = std::string("column \"") + - NameStr(attr->attname) + "\" has a type that lake tables " - "cannot store yet"; - - dl_error_set(DL_ERR_NOT_SUPPORTED, "arrow schema", nullptr, - message.c_str()); + snprintf(message, sizeof(message), "column \"%s\" has no field id", + NameStr(attr->attname)); + dl_error_set(DL_ERR_INTERNAL, "arrow schema", nullptr, message); return nullptr; } @@ -171,9 +233,41 @@ DlArrowSchemaFromTupleDesc(TupleDesc tupdesc) * turn any later relaxation of the constraint into a write failure * against files already on disk. */ - fields.push_back(arrow::field(NameStr(attr->attname), type, - /* nullable */ true)); + fields.push_back(arrow::field(NameStr(attr->attname), + DlArrowTypeForPgType(attr->atttypid), + /* nullable */ true, + arrow::key_value_metadata( + {DL_ARROW_FIELD_ID_KEY}, + {std::to_string(field_id)}))); } return arrow::schema(fields); } + +int32_t +DlArrowFieldId(const arrow::Field &field) +{ + const std::shared_ptr &metadata = field.metadata(); + int key; + const char *text; + char *end; + long id; + + if (metadata == nullptr) + return -1; + + key = metadata->FindKey(DL_ARROW_FIELD_ID_KEY); + if (key < 0) + return -1; + + /* + * Parquet's bridge wrote this from an int, so anything that is not one is + * not a field id, whatever else it might be. + */ + text = metadata->value(key).c_str(); + id = strtol(text, &end, 10); + if (end == text || *end != '\0' || id < 0 || id > INT32_MAX) + return -1; + + return (int32_t) id; +} diff --git a/contrib/datalake_fdw/src/format/arrow_support.h b/contrib/datalake_fdw/src/format/arrow_support.h index d79b5934ddc..8be4c41cc20 100644 --- a/contrib/datalake_fdw/src/format/arrow_support.h +++ b/contrib/datalake_fdw/src/format/arrow_support.h @@ -18,13 +18,8 @@ * under the License. * * arrow_support.h - * What every Arrow-facing translation unit in this module needs: how a - * PostgreSQL column type is stored, and how an Arrow failure is reported. - * - * The type mapping is in one place because the writer, the builder that feeds - * it and the reader that decodes what comes back all have to agree on it, and a - * disagreement between them would show up as wrong values rather than as an - * error. + * The type mapping and the error translation shared by the Arrow-facing + * parts of this module. * * IDENTIFICATION * contrib/datalake_fdw/src/format/arrow_support.h @@ -35,6 +30,7 @@ #ifndef DL_ARROW_SUPPORT_H #define DL_ARROW_SUPPORT_H +#include #include #include @@ -49,6 +45,15 @@ extern "C" #include "access/tupdesc.h" } +#include "format/format_types.h" + +/* + * The metadata key under which Parquet's Arrow bridge carries a column's field + * id in both directions: a field written with it gets the id in the Parquet + * schema, and a field read from a Parquet schema that has one carries it here. + */ +#define DL_ARROW_FIELD_ID_KEY "PARQUET:field_id" + /* * Turns an Arrow status into this module's error code, recording what Arrow * said -- its own class and message are the only thing that makes a failure in @@ -60,16 +65,25 @@ extern DlErrCode DlArrowStatus(const arrow::Status &status, const char *operatio /* * The Arrow type a column of this PostgreSQL type is stored as, or a null - * pointer when the type has no mapping yet. Callers report the refusal - * themselves, because only they know which column it was about. + * pointer when the type has no mapping. The type alone: whether a modifier + * makes the column unstorable is dl_format_type_refusal()'s question, and + * callers report a refusal themselves, because only they know which column it + * was about. */ extern std::shared_ptr DlArrowTypeForPgType(Oid atttypid); /* - * The whole descriptor. Returns a null pointer and records which column was - * the problem in the error detail: a type with no mapping, or a dropped column, - * which has no type to write and which nothing reads a file for yet. + * The whole descriptor, minus its dropped attributes, each field carrying the + * Iceberg field id `field_ids` gives for that attribute -- or, when that is + * null, its 1-based position among the live columns. Returns a null pointer + * and records which column was the problem in the error detail: a type or a + * modifier no data file can hold, or a database encoding other than UTF8, + * which no data file can hold either. */ -extern std::shared_ptr DlArrowSchemaFromTupleDesc(TupleDesc tupdesc); +extern std::shared_ptr DlArrowSchemaFromTupleDesc(TupleDesc tupdesc, + const int32_t *field_ids); + +/* The field id a field read from a Parquet file carries, or -1 when none. */ +extern int32_t DlArrowFieldId(const arrow::Field &field); #endif /* DL_ARROW_SUPPORT_H */ diff --git a/contrib/datalake_fdw/src/format/format.h b/contrib/datalake_fdw/src/format/format.h index db793eb9689..436240aa5f5 100644 --- a/contrib/datalake_fdw/src/format/format.h +++ b/contrib/datalake_fdw/src/format/format.h @@ -78,14 +78,23 @@ typedef struct Fragment } Fragment; /* - * The columns to materialise, as 0-based indexes into the file schema. A NULL - * set, or one with no columns, means every column: "read nothing" is not a - * projection anyone asks for, so it is not worth a second way to say "all". + * The columns to materialise, named by Iceberg field id, in the order the batch + * is to present them. A file's columns are matched by the field id each of + * them carries -- never by position and never by name: an older file lacks the + * columns added since, a renamed column keeps its id, and a file written by + * something else may order its columns as it likes. A field id the file does + * not have comes back as a column of Arrow's null type, every value NULL, + * which is what the Iceberg spec says a column added after the file was written + * holds. A column of the file that carries no field id can never be matched. + * + * A NULL set means every column the file has, in the file's order. That is + * for reading a file on its own terms -- the test functions do -- and not for + * reading a table, whose columns the file may not agree with. */ typedef struct ProjectionSet { - const int *columns; - int ncolumns; + const int32_t *field_ids; + int nfields; } ProjectionSet; /* @@ -99,6 +108,15 @@ typedef struct WriterOptions { const char *compression; /* format-defined name; NULL for the default */ int64_t row_group_size; /* rows per row group; 0 for the default */ + + /* + * The Iceberg field id of each attribute of the descriptor the writer is + * opened with, dropped attributes included (and ignored), so that the + * array is indexed the way the descriptor is. NULL numbers the live + * columns 1..n in order, which is what a new table's ids are; a table that + * has evolved has to say what its ids are. + */ + const int32_t *field_ids; } WriterOptions; typedef struct RowGroupFilterSet RowGroupFilterSet; @@ -149,8 +167,11 @@ typedef struct FormatWriterOps { } FormatWriterOps; struct FormatWriter { const FormatWriterOps *ops; void *impl; }; -/* Bumped when an existing field changes meaning; appending does not need it. */ -#define DL_FORMAT_ABI_VERSION 1 +/* + * Bumped when an existing field changes meaning; appending does not need it. + * 2: ProjectionSet names field ids rather than positions in the file. + */ +#define DL_FORMAT_ABI_VERSION 2 typedef struct FormatRoutine { uint32_t abi_version, struct_size; /* same prefix-compat semantics as meta engine */ diff --git a/contrib/datalake_fdw/src/format/format_registry.c b/contrib/datalake_fdw/src/format/format_registry.c index 44bf9ec0732..f6d07131ecd 100644 --- a/contrib/datalake_fdw/src/format/format_registry.c +++ b/contrib/datalake_fdw/src/format/format_registry.c @@ -26,9 +26,7 @@ *------------------------------------------------------------------------- */ -#include -#include -#include +#include "postgres.h" #include "common/dl_err.h" #include "format/format.h" @@ -36,8 +34,10 @@ /* * Parquet is the only format so far. A name that reaches here came from a - * table option, so an unknown one is an ordinary mistake and the caller has to - * be able to say which name it was -- returning a bare NULL would leave every + * table option, and is compared the way the other option values of this module + * are -- without regard to case, so that 'Parquet' is not a second, unknown + * format. An unknown one is an ordinary mistake and the caller has to be able + * to say which name it was -- returning a bare NULL would leave every * caller to write that message again, and get it wrong differently. The name * goes into the error detail, so a caller that reports DL_ERR_NOT_SUPPORTED * gets it without knowing this function exists. @@ -47,7 +47,7 @@ GetFormatRoutine(const char *format) { char message[128]; - if (format != NULL && strcmp(format, "parquet") == 0) + if (format != NULL && pg_strcasecmp(format, "parquet") == 0) return GetParquetFormatRoutine(); snprintf(message, sizeof(message), diff --git a/contrib/datalake_fdw/src/format/format_types.h b/contrib/datalake_fdw/src/format/format_types.h new file mode 100644 index 00000000000..8c8a97b0d06 --- /dev/null +++ b/contrib/datalake_fdw/src/format/format_types.h @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * format_types.h + * Which PostgreSQL column types a lake table can hold. + * + * The answer belongs to the format layer, which is what has to store the + * values; but the place to give it is CREATE TABLE, before a table exists that + * can never be written to. This is the one function both ask, so that they + * cannot disagree. It is plain C so that the access method can call it + * without seeing Arrow. + * + * postgres.h must be included before this header. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/format_types.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_FORMAT_TYPES_H +#define DL_FORMAT_TYPES_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * NULL when a column of this type, with this modifier, can be stored in a data + * file; otherwise a static sentence saying why not, written to follow the name + * of the type -- "type X, which ". The type is not named here because + * naming it means a catalog lookup, and one caller is on the far side of the + * C++ boundary where nothing may allocate; the callers know the type. + * + * A length limit is refused, not stored: varchar(n) and char(n) have nowhere + * to record n in a lake table, whose only string type is an unbounded UTF-8 + * string, and a limit that lives in the catalog but not in the data would make + * a file written by anything else able to violate the declared type. char(n) + * is refused for its padding as well, which no other reader would strip. + */ +extern const char *dl_format_type_refusal(Oid typid, int32 typmod); + +#ifdef __cplusplus +} +#endif + +#endif /* DL_FORMAT_TYPES_H */ diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp index 9c245672fbe..c5432fd2292 100644 --- a/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp +++ b/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp @@ -26,20 +26,25 @@ *------------------------------------------------------------------------- */ +#include #include +#include #include +#include #include #include #include #include #include +#include #include "format/arrow_support.h" #include "am_iceberg/pg_iceberg_guc.h" #include "common/dl_resource.h" #include "common/dl_wrappers.h" +#include "format/arrow_memory_pool.h" #include "format/parquet/parquet_internal.h" struct ParquetReader @@ -48,6 +53,15 @@ struct ParquetReader std::shared_ptr file; std::unique_ptr reader; std::shared_ptr batches; + + /* + * How a batch the file yields becomes the batch the caller asked for: one + * entry per requested field, holding that field's position among the + * columns read, or -1 for a field the file does not have. Empty when the + * caller asked for the file as it is, and batches pass straight through. + */ + std::vector output_columns; + std::shared_ptr output_schema; }; static DlErrCode @@ -76,7 +90,43 @@ parquet_reader_next_batch(FormatReader *reader, struct ArrowArray *out, return DL_OK; } - return DlArrowStatus(arrow::ExportRecordBatch(*batch, out, schema), + if (impl->output_columns.empty()) + return DlArrowStatus(arrow::ExportRecordBatch(*batch, out, schema), + "export a Parquet batch"); + + std::vector> columns; + + columns.reserve(impl->output_columns.size()); + + for (int source : impl->output_columns) + { + if (source >= 0) + { + columns.push_back(batch->column(source)); + continue; + } + + /* + * A field the file does not have: NULL in every row, which is what + * the Iceberg spec says a column added after the file was written + * holds. The null type carries no buffers, so this costs nothing + * per row. + */ + arrow::Result> nulls = + arrow::MakeArrayOfNull(arrow::null(), batch->num_rows(), + DlArrowMemoryPool()); + + if (!nulls.ok()) + return DlArrowStatus(nulls.status(), "read a Parquet batch"); + + columns.push_back(*nulls); + } + + std::shared_ptr projected = + arrow::RecordBatch::Make(impl->output_schema, batch->num_rows(), + std::move(columns)); + + return DlArrowStatus(arrow::ExportRecordBatch(*projected, out, schema), "export a Parquet batch"); } DL_ABI_GUARD_END(result, "next_batch"); @@ -161,6 +211,82 @@ parquet_row_groups(const Fragment *fragment, int total, return DL_OK; } +/* + * Which of the file's columns to read, and how to lay them out for the caller. + * + * Matching is by field id -- see ProjectionSet -- and never by position or by + * name. The columns are read in file order whatever order they were asked for + * in, because file order is the order Parquet's reader hands them back in; + * output_columns is what then puts them in the caller's order, and supplies + * the fields the file does not have. + */ +static DlErrCode +parquet_project(ParquetReader *impl, const arrow::Schema &file_schema, + const ProjectionSet *projection, std::vector *columns) +{ + std::map file_column_by_id; + std::vector> fields; + char message[160]; + + for (int i = 0; i < file_schema.num_fields(); i++) + { + int32_t field_id = DlArrowFieldId(*file_schema.field(i)); + + /* A column that carries no id can never be matched to anything. */ + if (field_id < 0) + continue; + + /* + * Two columns with one id is a file nothing can read by id. Taking + * either would be guessing about data. + */ + if (!file_column_by_id.emplace(field_id, i).second) + { + snprintf(message, sizeof(message), + "the file has two columns with field id %d", (int) field_id); + dl_error_set(DL_ERR_INVALID_OPTION, "open a Parquet file", NULL, message); + return DL_ERR_INVALID_OPTION; + } + } + + /* The file columns to read: ascending, each once. */ + for (int k = 0; k < projection->nfields; k++) + { + auto found = file_column_by_id.find(projection->field_ids[k]); + + if (found != file_column_by_id.end()) + columns->push_back(found->second); + } + std::sort(columns->begin(), columns->end()); + columns->erase(std::unique(columns->begin(), columns->end()), columns->end()); + + fields.reserve(projection->nfields); + impl->output_columns.reserve(projection->nfields); + + for (int k = 0; k < projection->nfields; k++) + { + int32_t field_id = projection->field_ids[k]; + auto found = file_column_by_id.find(field_id); + + if (found == file_column_by_id.end()) + { + impl->output_columns.push_back(-1); + fields.push_back(arrow::field("field_id_" + std::to_string(field_id), + arrow::null())); + continue; + } + + impl->output_columns.push_back( + (int) (std::lower_bound(columns->begin(), columns->end(), + found->second) - columns->begin())); + fields.push_back(file_schema.field(found->second)); + } + + impl->output_schema = arrow::schema(fields); + + return DL_OK; +} + DlErrCode parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, const RowGroupFilterSet *filters, FormatReader **out) @@ -190,7 +316,8 @@ parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, DL_ABI_GUARD_BEGIN { std::unique_ptr impl(new ParquetReader()); - arrow::MemoryPool *pool = arrow::default_memory_pool(); + arrow::MemoryPool *pool = DlArrowMemoryPool(); + std::shared_ptr file_schema; std::vector row_groups; std::vector columns; DlErrCode rc; @@ -205,7 +332,13 @@ parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, return DlArrowStatus(file.status(), "open a Parquet file"); impl->file = *file; - arrow::Status status = builder.Open(impl->file); + /* + * The properties are passed so that the pool is: left to its default + * argument, Open() would decode Parquet pages out of Arrow's own pool, + * unseen by the memory accounting the module's pool exists for. + */ + arrow::Status status = builder.Open(impl->file, + parquet::ReaderProperties(pool)); if (!status.ok()) return DlArrowStatus(status, "open a Parquet file"); @@ -217,9 +350,31 @@ parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, * A backend is not a thread pool. Arrow will read column chunks in * parallel if asked, and a worker thread that hits an error has no way * to report it through PostgreSQL's error handling, so this reads on - * the thread it was called on. + * the thread it was called on. Pre-buffering is the other way Arrow + * moves work onto its I/O threads -- and from Arrow 13 it is on by + * default -- so it is switched off by name rather than left to a + * default that changes between the versions this builds against. */ properties.set_use_threads(false); + properties.set_pre_buffer(false); + + /* + * Spark, Hive and Impala wrote timestamps as INT96 for years, and + * Arrow surfaces those as nanoseconds unless told otherwise. A + * PostgreSQL timestamp is microseconds, so that is what they are read + * as: INT96 has no unit of its own to lose, and nanoseconds would + * confine the values to 1677..2262 -- the "end of time" dates a + * warehouse keeps as sentinels lie outside that, and Arrow wraps them + * silently rather than refusing. + * + * One caveat, Arrow's rather than ours: INT96 is a day number plus the + * nanoseconds into that day, and Arrow's microsecond conversion takes + * the second part as non-negative, which is what the writers above + * store. pyarrow's deprecated INT96 writer stores a negative one for + * instants before 1970, and those come back wrong -- from pyarrow + * itself with the same setting, as from here. + */ + properties.set_coerce_int96_timestamp_unit(arrow::TimeUnit::MICRO); status = builder.memory_pool(pool)->properties(properties) ->Build(&impl->reader); @@ -232,18 +387,19 @@ parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, if (rc != DL_OK) return rc; - if (projection != NULL && projection->ncolumns > 0) - columns.assign(projection->columns, - projection->columns + projection->ncolumns); + status = impl->reader->GetSchema(&file_schema); + if (!status.ok()) + return DlArrowStatus(status, "read a Parquet schema"); + + if (projection != NULL && projection->nfields > 0) + { + rc = parquet_project(impl.get(), *file_schema, projection, &columns); + if (rc != DL_OK) + return rc; + } else { - std::shared_ptr schema; - - status = impl->reader->GetSchema(&schema); - if (!status.ok()) - return DlArrowStatus(status, "read a Parquet schema"); - - for (int i = 0; i < schema->num_fields(); i++) + for (int i = 0; i < file_schema->num_fields(); i++) columns.push_back(i); } diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp index e5c00b9b043..ea60301df97 100644 --- a/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp +++ b/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp @@ -26,22 +26,35 @@ *------------------------------------------------------------------------- */ +#include +#include #include +#include #include #include -#include #include +#include +#include + #include #include #include +#include #include #include +#include #include "format/arrow_support.h" +extern "C" +{ +#include "common/file_perm.h" +} + #include "common/dl_resource.h" #include "common/dl_wrappers.h" +#include "format/arrow_memory_pool.h" #include "format/parquet/parquet_internal.h" struct ParquetWriter @@ -91,6 +104,9 @@ class ParquetReleaseBatch * complete, valid, truncated file appearing at the path if the unlink does not * take. What it leaves then has no footer, so nothing can read it, and that is * why the unlink's result is not worth reporting. + * + * The file is ours to delete: parquet_open_writer() created it with O_EXCL, so + * nothing was at the path before, and nothing this removes was anyone else's. */ static void parquet_discard(ParquetWriter *impl) @@ -318,28 +334,55 @@ static const FormatWriterOps parquet_writer_ops = { parquet_writer_abort }; +/* + * The name arrives as a user typed it into a table option, so case is not + * meaning. Arrow is asked about it rather than a list kept here, because two + * of the three answers depend on things this file cannot know: Parquet's + * specification admits a subset of the codecs Arrow names, and the Arrow this + * module is linked against was built with a subset of those. Asked in that + * order, so that the message says which of the three it was. + */ static DlErrCode parquet_compression(const char *name, arrow::Compression::type *out) { - std::string requested(name); - char message[128]; - - if (requested == "none" || requested == "uncompressed") - *out = arrow::Compression::UNCOMPRESSED; - else if (requested == "snappy") - *out = arrow::Compression::SNAPPY; - else if (requested == "gzip") - *out = arrow::Compression::GZIP; - else if (requested == "zstd") - *out = arrow::Compression::ZSTD; - else + std::string requested(name); + char message[160]; + + for (char &c : requested) + c = (char) tolower((unsigned char) c); + + /* PostgreSQL's word for it; Arrow's is the long one. */ + if (requested == "none") + requested = "uncompressed"; + + arrow::Result codec = + arrow::util::Codec::GetCompressionType(requested); + + if (!codec.ok()) { snprintf(message, sizeof(message), - "\"%s\" is not a compression this build can write", name); + "\"%s\" is not a compression Arrow knows", name); dl_error_set(DL_ERR_INVALID_OPTION, "open a Parquet file", NULL, message); return DL_ERR_INVALID_OPTION; } + if (!parquet::IsCodecSupported(*codec)) + { + snprintf(message, sizeof(message), + "\"%s\" is not a compression a Parquet file can use", name); + dl_error_set(DL_ERR_INVALID_OPTION, "open a Parquet file", NULL, message); + return DL_ERR_INVALID_OPTION; + } + + if (!arrow::util::Codec::IsAvailable(*codec)) + { + snprintf(message, sizeof(message), + "\"%s\" is not a compression this build of Arrow can write", name); + dl_error_set(DL_ERR_INVALID_OPTION, "open a Parquet file", NULL, message); + return DL_ERR_INVALID_OPTION; + } + + *out = *codec; return DL_OK; } @@ -359,13 +402,15 @@ parquet_open_writer(const char *path, void *tupdesc_arg, DL_ABI_GUARD_BEGIN { std::unique_ptr impl(new ParquetWriter()); - arrow::MemoryPool *pool = arrow::default_memory_pool(); + arrow::MemoryPool *pool = DlArrowMemoryPool(); parquet::WriterProperties::Builder properties; arrow::Compression::type compression = arrow::Compression::SNAPPY; DlErrCode rc; + int fd; impl->path = path; - impl->schema = DlArrowSchemaFromTupleDesc((TupleDesc) tupdesc_arg); + impl->schema = DlArrowSchemaFromTupleDesc((TupleDesc) tupdesc_arg, + options != NULL ? options->field_ids : nullptr); if (impl->schema == nullptr) return DL_ERR_NOT_SUPPORTED; /* detail already recorded */ @@ -377,6 +422,9 @@ parquet_open_writer(const char *path, void *tupdesc_arg, } properties.compression(compression); + /* Left alone, the builder would take Arrow's default pool, silently. */ + properties.memory_pool(pool); + if (options != NULL && options->row_group_size > 0) properties.max_row_group_length(options->row_group_size); @@ -386,11 +434,45 @@ parquet_open_writer(const char *path, void *tupdesc_arg, impl->row_group_size = built->max_row_group_length(); impl->pending_rows = 0; + /* + * Created here rather than by Arrow. Arrow's path form opens with + * O_TRUNC, which would empty a file that was already there -- and this + * writer deletes the file it holds whenever it cannot finish it, so a + * truncated file would then be a deleted one. With O_EXCL the kernel + * answers "did I create this", and the writer only ever deletes what + * it created. A lake's data file names are unique by construction, so + * a path that exists is a mistake, and refusing it is right anyway. + */ + fd = open(path, O_WRONLY | O_CREAT | O_EXCL, pg_file_create_mode); + if (fd < 0) + { + int saved_errno = errno; + std::string message; + + if (saved_errno == EEXIST) + { + message = std::string("\"") + path + "\" already exists"; + dl_error_set(DL_ERR_ALREADY_EXISTS, "create a Parquet file", NULL, + message.c_str()); + return DL_ERR_ALREADY_EXISTS; + } + + message = std::string("could not create \"") + path + "\": " + + strerror(saved_errno); + dl_error_set(DL_ERR_IO, "create a Parquet file", NULL, message.c_str()); + return DL_ERR_IO; + } + + /* From here the file exists and is ours, so every failure discards it. */ arrow::Result> sink = - arrow::io::FileOutputStream::Open(path); + arrow::io::FileOutputStream::Open(fd); if (!sink.ok()) + { + (void) close(fd); /* Arrow took nothing */ + parquet_discard(impl.get()); return DlArrowStatus(sink.status(), "create a Parquet file"); + } impl->sink = *sink; /* @@ -398,7 +480,9 @@ parquet_open_writer(const char *path, void *tupdesc_arg, * With it, reading back would restore the types from our own note * rather than from Parquet's, and a round trip would agree with itself * no matter what it had written; without it, what comes back is what - * any other reader of the file sees. + * any other reader of the file sees. The field ids are not part of + * that note: Parquet's bridge writes them into the Parquet schema + * itself, where every reader finds them. */ /* * Arrow 11 deprecated the form that returns its writer through an out @@ -426,8 +510,7 @@ parquet_open_writer(const char *path, void *tupdesc_arg, if (!status.ok()) { - (void) impl->sink->Close(); - unlink(path); + parquet_discard(impl.get()); return DlArrowStatus(status, "create a Parquet file"); } diff --git a/contrib/datalake_fdw/src/test/datalake_fdw_test.c b/contrib/datalake_fdw/src/test/datalake_fdw_test.c index 1d2c2946d11..e6c96f0a57a 100644 --- a/contrib/datalake_fdw/src/test/datalake_fdw_test.c +++ b/contrib/datalake_fdw/src/test/datalake_fdw_test.c @@ -37,8 +37,10 @@ #include "postgres.h" +#include "catalog/pg_type.h" #include "executor/spi.h" #include "funcapi.h" +#include "utils/array.h" #include "utils/builtins.h" #include "utils/memutils.h" #include "utils/tuplestore.h" @@ -52,6 +54,24 @@ PG_FUNCTION_INFO_V1(datalake_parquet_write); PG_FUNCTION_INFO_V1(datalake_parquet_read); +/* + * The SQL declaration and the C function have to agree on the argument list, + * and nothing checks that they do: a database where the extension was created + * from an older datalake_fdw_test--1.0.sql hands over fewer arguments than the + * function reads, and reading one that is not there is a crash. This turns + * that into an error naming the fix. + */ +static void +check_nargs(FunctionCallInfo fcinfo, int expected) +{ + if (PG_NARGS() != expected) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("datalake_fdw_test is out of date: the function was declared with %d arguments and expects %d", + PG_NARGS(), expected), + errhint("DROP EXTENSION datalake_fdw_test and CREATE it again."))); +} + static const FormatRoutine * parquet_routine(void) { @@ -88,21 +108,25 @@ write_one_batch(FormatWriter *writer, DlArrowBuilder builder) } /* - * datalake_parquet_write(path, query, row_group_size) -> rows written + * datalake_parquet_write(path, query, row_group_size, compression) -> rows written * * The rows the query returns are written to `path` as Parquet. A row group * size of zero leaves the format's own default in place; anything else also * becomes the number of rows per batch, because a row group is closed at a * batch boundary and the option would otherwise be rounded away by a batch size - * that does not divide by it. + * that does not divide by it. An empty compression name means the format's + * default: the function is STRICT, so NULL cannot be the way to say that. + * + * The columns are given field ids 1..n, which is what a new table's would be. */ Datum datalake_parquet_write(PG_FUNCTION_ARGS) { - char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); - char *query = text_to_cstring(PG_GETARG_TEXT_PP(1)); - int32 row_group_size = PG_GETARG_INT32(2); - const FormatRoutine *routine = parquet_routine(); + char *path; + char *query; + int32 row_group_size; + char *compression; + const FormatRoutine *routine; WriterOptions options = {0}; FormatWriter *volatile open_writer = NULL; DlArrowBuilder volatile open_builder = NULL; @@ -110,6 +134,13 @@ datalake_parquet_write(PG_FUNCTION_ARGS) int64 written = 0; MemoryContext row_context; + check_nargs(fcinfo, 4); + path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + query = text_to_cstring(PG_GETARG_TEXT_PP(1)); + row_group_size = PG_GETARG_INT32(2); + compression = text_to_cstring(PG_GETARG_TEXT_PP(3)); + routine = parquet_routine(); + /* * Bounded above as well as below, and by the same number as * iceberg.batch_rows: a row group is held in memory until it is complete, @@ -122,6 +153,8 @@ datalake_parquet_write(PG_FUNCTION_ARGS) DL_MAX_ROW_GROUP_ROWS))); options.row_group_size = row_group_size; + options.compression = compression[0] != '\0' ? compression : NULL; + options.field_ids = NULL; if (row_group_size > 0 && row_group_size < batch_rows) batch_rows = row_group_size; @@ -272,13 +305,20 @@ datalake_parquet_write(PG_FUNCTION_ARGS) } /* - * datalake_parquet_read(path, first_row_group, n_row_groups) -> setof record + * datalake_parquet_read(path, first_row_group, n_row_groups, field_ids) + * -> setof record * * The column definition list says what the caller expects the file to hold, and * is checked against the file's own schema rather than assumed: reading an * Arrow column as the wrong PostgreSQL type would produce values, just not the * ones in the file. * + * With an empty field id list the file is read as it is, every column in the + * file's order, and the definition list has to match it column for column. + * With one, each entry names the Iceberg field id the corresponding column of + * the definition list is to be read from, in the way a table's columns are + * matched to a data file's; an id the file does not have reads as NULL. + * * The row group arguments are the unit a scan is divided at. Reading 0..0 and * then 1..1 has to produce exactly what reading the whole file does, which is * the property a scan spread across segments will depend on. @@ -286,13 +326,19 @@ datalake_parquet_write(PG_FUNCTION_ARGS) Datum datalake_parquet_read(PG_FUNCTION_ARGS) { - char *path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + char *path; + ArrayType *field_id_array; ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; - const FormatRoutine *routine = parquet_routine(); + const FormatRoutine *routine; FormatReader *volatile open_reader = NULL; struct ArrowArray *batch = palloc0(sizeof(struct ArrowArray)); struct ArrowSchema *schema = palloc0(sizeof(struct ArrowSchema)); Fragment fragment = {0}; + ProjectionSet projection = {0}; + const ProjectionSet *projection_arg = NULL; + Datum *field_id_datums; + bool *field_id_nulls; + int nfield_ids; TupleDesc tupdesc; Tuplestorestate *tupstore; Datum *values; @@ -301,6 +347,11 @@ datalake_parquet_read(PG_FUNCTION_ARGS) MemoryContext row_context; DlErrCode rc; + check_nargs(fcinfo, 4); + path = text_to_cstring(PG_GETARG_TEXT_PP(0)); + field_id_array = PG_GETARG_ARRAYTYPE_P(3); + routine = parquet_routine(); + fragment.path = path; fragment.first_row_group = PG_GETARG_INT32(1); fragment.n_row_groups = PG_GETARG_INT32(2); @@ -312,6 +363,33 @@ datalake_parquet_read(PG_FUNCTION_ARGS) values = palloc(tupdesc->natts * sizeof(Datum)); nulls = palloc(tupdesc->natts * sizeof(bool)); + deconstruct_array(field_id_array, INT4OID, sizeof(int32), true, TYPALIGN_INT, + &field_id_datums, &field_id_nulls, &nfield_ids); + if (nfield_ids > 0) + { + int32 *field_ids = palloc(nfield_ids * sizeof(int32)); + int i; + + if (nfield_ids != tupdesc->natts) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("the field id list names %d columns, the column definition list has %d", + nfield_ids, tupdesc->natts))); + + for (i = 0; i < nfield_ids; i++) + { + if (field_id_nulls[i]) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("a field id cannot be null"))); + field_ids[i] = DatumGetInt32(field_id_datums[i]); + } + + projection.field_ids = field_ids; + projection.nfields = nfield_ids; + projection_arg = &projection; + } + /* * Every text and bytea decoded out of a batch is a copy, and tuplestore * copies it again. A materialize-mode function is called once, so the @@ -323,7 +401,7 @@ datalake_parquet_read(PG_FUNCTION_ARGS) "datalake_parquet_read", ALLOCSET_DEFAULT_SIZES); - rc = routine->open_reader(&fragment, NULL, NULL, &reader); + rc = routine->open_reader(&fragment, projection_arg, NULL, &reader); if (rc != DL_OK) dl_error_report(ERROR, rc, "open_reader"); open_reader = reader; @@ -356,7 +434,8 @@ datalake_parquet_read(PG_FUNCTION_ARGS) for (attno = 0; attno < tupdesc->natts; attno++) { rc = dl_arrow_decode_check(schema->children[attno], - TupleDescAttr(tupdesc, attno)->atttypid); + TupleDescAttr(tupdesc, attno)->atttypid, + TupleDescAttr(tupdesc, attno)->atttypmod); if (rc != DL_OK) dl_error_report(ERROR, rc, "check_column"); } @@ -369,9 +448,9 @@ datalake_parquet_read(PG_FUNCTION_ARGS) for (attno = 0; attno < tupdesc->natts; attno++) { - rc = dl_arrow_decode_value(batch->children[attno], row, + rc = dl_arrow_decode_value(schema->children[attno], + batch->children[attno], row, TupleDescAttr(tupdesc, attno)->atttypid, - TupleDescAttr(tupdesc, attno)->atttypmod, &values[attno], &nulls[attno]); if (rc != DL_OK) dl_error_report(ERROR, rc, "decode_value"); diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out index 6f649fd1cf4..23e3d0c4c6a 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/expected/parquet_roundtrip.out @@ -1,22 +1,29 @@ --- Parquet: every type a lake table can store, written to a file and read back, --- and the row group range that a scan will one day be divided at. +-- Parquet: every type a lake table can store, written to a file and read back; +-- the row group range that a scan will one day be divided at; and the field +-- ids that a table's columns are matched to a file's by. SET client_min_messages = warning; DROP VIEW IF EXISTS dlparq_roundtrip, dlparq_split; -DROP TABLE IF EXISTS dlparq_src, dlparq_pairs, dlparq_unsupported CASCADE; +DROP TABLE IF EXISTS dlparq_src, dlparq_pairs, dlparq_unsupported, dlparq_bounded CASCADE; CREATE EXTENSION IF NOT EXISTS datalake_fdw; CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; RESET client_min_messages; -- A timestamptz is printed in the session's zone, so without this the output -- would depend on where the test ran rather than on what the file holds. SET TimeZone = 'UTC'; --- Fixed file names rather than a unique one per run: the writer truncates, so a --- run reuses what the last one left instead of adding to it. Nothing reachable --- from SQL can remove a file, so unique names would accumulate forever. +-- Fixed file names, removed before the run rather than overwritten during it: +-- the writer refuses a path that already exists, so what the last run left has +-- to go first. COPY TO PROGRAM runs on the coordinator, which is where these +-- functions read and write. \set roundtrip_file '/tmp/datalake_fdw_regress_roundtrip.parquet' \set split_file '/tmp/datalake_fdw_regress_split.parquet' +\set swapped_file '/tmp/datalake_fdw_regress_swapped.parquet' \set empty_file '/tmp/datalake_fdw_regress_empty.parquet' \set batch_file '/tmp/datalake_fdw_regress_batch.parquet' +\set gzip_file '/tmp/datalake_fdw_regress_gzip.parquet' \set missing_file '/tmp/datalake_fdw_regress_does_not_exist.parquet' +COPY (SELECT 1) TO PROGRAM 'rm -f /tmp/datalake_fdw_regress_*.parquet'; +-- varchar without a length, because a length is what a lake table cannot hold; +-- see the refusals at the end. CREATE TABLE dlparq_src ( c_bool boolean, c_int2 smallint, @@ -25,30 +32,34 @@ CREATE TABLE dlparq_src ( c_float4 real, c_float8 double precision, c_text text, - c_varchar varchar(16), - c_bpchar char(5), + c_varchar varchar, c_bytea bytea, c_date date, + c_time time, c_ts timestamp, - c_tstz timestamptz + c_tstz timestamptz, + c_uuid uuid ) DISTRIBUTED RANDOMLY; -- The dates and timestamps are chosen around both epochs: PostgreSQL counts -- from 2000-01-01 and Arrow from 1970-01-01, and a value on either side of --- 1970 is what tells a wrong shift from a right one. A row of nulls is here --- because a validity bitmap that is never exercised is a bitmap that has not --- been tested. +-- 1970 is what tells a wrong shift from a right one. 24:00:00 is the one time +-- PostgreSQL admits past the end of the day. A row of nulls is here because a +-- validity bitmap that is never exercised is a bitmap that has not been tested. INSERT INTO dlparq_src VALUES (true, 1, 100, 1000, 1.5, 2.5, - 'hello', 'varchar', 'abc', '\x0102'::bytea, - '1970-01-01', '1970-01-01 00:00:00', '1970-01-01 00:00:00+00'), + 'hello', 'varchar', '\x0102'::bytea, + '1970-01-01', '00:00:00', '1970-01-01 00:00:00', '1970-01-01 00:00:00+00', + '00000000-0000-0000-0000-000000000000'), (false, -2, -200, -2000, -1.5, -2.5, - 'a longer string with 中文', 'x', '', '\x'::bytea, - '2000-01-01', '2000-01-01 12:34:56.789012', '2000-01-01 12:34:56.789012+00'), + 'a longer string with 中文', 'x', '\x'::bytea, + '2000-01-01', '12:34:56.789012', '2000-01-01 12:34:56.789012', '2000-01-01 12:34:56.789012+00', + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), (true, 32767, 2147483647, 9223372036854775807, 3.25, 1e300, - '', 'z', 'zzzzz', '\xdeadbeef'::bytea, - '1969-12-31', '1969-12-31 23:59:59.999999', '1969-12-31 23:59:59.999999+00'), + '', 'z', '\xdeadbeef'::bytea, + '1969-12-31', '24:00:00', '1969-12-31 23:59:59.999999', '1969-12-31 23:59:59.999999+00', + 'ffffffff-ffff-ffff-ffff-ffffffffffff'), (NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); SELECT datalake_parquet_write(:'roundtrip_file', 'SELECT * FROM dlparq_src') AS rows_written; rows_written @@ -68,19 +79,20 @@ CREATE VIEW dlparq_roundtrip AS c_float4 real, c_float8 double precision, c_text text, - c_varchar varchar(16), - c_bpchar char(5), + c_varchar varchar, c_bytea bytea, c_date date, + c_time time, c_ts timestamp, - c_tstz timestamptz); + c_tstz timestamptz, + c_uuid uuid); SELECT * FROM dlparq_roundtrip ORDER BY c_int4; - c_bool | c_int2 | c_int4 | c_int8 | c_float4 | c_float8 | c_text | c_varchar | c_bpchar | c_bytea | c_date | c_ts | c_tstz ---------+--------+------------+---------------------+----------+----------+---------------------------+-----------+----------+------------+------------+---------------------------------+------------------------------------- - f | -2 | -200 | -2000 | -1.5 | -2.5 | a longer string with 中文 | x | | \x | 01-01-2000 | Sat Jan 01 12:34:56.789012 2000 | Sat Jan 01 12:34:56.789012 2000 UTC - t | 1 | 100 | 1000 | 1.5 | 2.5 | hello | varchar | abc | \x0102 | 01-01-1970 | Thu Jan 01 00:00:00 1970 | Thu Jan 01 00:00:00 1970 UTC - t | 32767 | 2147483647 | 9223372036854775807 | 3.25 | 1e+300 | | z | zzzzz | \xdeadbeef | 12-31-1969 | Wed Dec 31 23:59:59.999999 1969 | Wed Dec 31 23:59:59.999999 1969 UTC - | | | | | | | | | | | | + c_bool | c_int2 | c_int4 | c_int8 | c_float4 | c_float8 | c_text | c_varchar | c_bytea | c_date | c_time | c_ts | c_tstz | c_uuid +--------+--------+------------+---------------------+----------+----------+---------------------------+-----------+------------+------------+-----------------+---------------------------------+-------------------------------------+-------------------------------------- + f | -2 | -200 | -2000 | -1.5 | -2.5 | a longer string with 中文 | x | \x | 01-01-2000 | 12:34:56.789012 | Sat Jan 01 12:34:56.789012 2000 | Sat Jan 01 12:34:56.789012 2000 UTC | a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11 + t | 1 | 100 | 1000 | 1.5 | 2.5 | hello | varchar | \x0102 | 01-01-1970 | 00:00:00 | Thu Jan 01 00:00:00 1970 | Thu Jan 01 00:00:00 1970 UTC | 00000000-0000-0000-0000-000000000000 + t | 32767 | 2147483647 | 9223372036854775807 | 3.25 | 1e+300 | | z | \xdeadbeef | 12-31-1969 | 24:00:00 | Wed Dec 31 23:59:59.999999 1969 | Wed Dec 31 23:59:59.999999 1969 UTC | ffffffff-ffff-ffff-ffff-ffffffffffff + | | | | | | | | | | | | | (4 rows) -- Both directions: one way only says what the file lost, and a file with a row @@ -94,19 +106,17 @@ FROM ((TABLE dlparq_src EXCEPT ALL TABLE dlparq_roundtrip) 0 (1 row) --- The comparison above cannot see the padding of a char(n): bpchar equality --- ignores trailing spaces, so a value that came back three characters long --- would still have compared equal to the five it went in as. -SELECT octet_length(c_bpchar) AS bpchar_bytes, - octet_length(c_text) AS text_bytes, +-- The bytes as well as the values: a string that came back re-encoded would +-- still compare equal. +SELECT octet_length(c_text) AS text_bytes, octet_length(c_bytea) AS bytea_bytes -FROM dlparq_roundtrip ORDER BY 1, 2, 3; - bpchar_bytes | text_bytes | bytea_bytes ---------------+------------+------------- - 5 | 0 | 4 - 5 | 5 | 2 - 5 | 27 | 0 - | | +FROM dlparq_roundtrip ORDER BY 1, 2; + text_bytes | bytea_bytes +------------+------------- + 0 | 4 + 5 | 2 + 27 | 0 + | (4 rows) -- Row groups are the unit a scan is divided at, so reading the parts has to add @@ -160,6 +170,76 @@ FROM datalake_parquet_read(:'split_file', 1) AS t (k int, v text); 4 (1 row) +-- A table's columns are matched to a file's by field id, never by position. +-- The writer numbered these 1 (k) and 2 (v); asking for them the other way +-- round gives them the other way round, and asking for an id the file does not +-- have -- a column added to the table after the file was written -- gives NULL. +SELECT * FROM datalake_parquet_read(:'split_file', 0, 0, '{2, 1, 3}') + AS t (v text, k int, added int) ORDER BY k; + v | k | added +----+---+------- + v1 | 1 | + v2 | 2 | + v3 | 3 | + v4 | 4 | + v5 | 5 | + v6 | 6 | +(6 rows) + +SELECT * FROM datalake_parquet_read(:'split_file', 0, 0, '{2}') + AS t (v text) ORDER BY v; + v +---- + v1 + v2 + v3 + v4 + v5 + v6 +(6 rows) + +-- Nothing the file has: as many rows as the file, all NULL. +SELECT count(*) AS rows_of_nothing, count(x) AS values_of_nothing +FROM datalake_parquet_read(:'split_file', 0, 0, '{9}') AS t (x int); + rows_of_nothing | values_of_nothing +-----------------+------------------- + 6 | 0 +(1 row) + +-- The ids follow the data that was written, not the names: a file written with +-- the columns the other way round holds v under 1 and k under 2. +SELECT datalake_parquet_write(:'swapped_file', + 'SELECT v, k FROM dlparq_pairs ORDER BY k') AS rows_written; + rows_written +-------------- + 6 +(1 row) + +SELECT * FROM datalake_parquet_read(:'swapped_file', 0, 0, '{2, 1}') + AS t (k int, v text) ORDER BY k; + k | v +---+---- + 1 | v1 + 2 | v2 + 3 | v3 + 4 | v4 + 5 | v5 + 6 | v6 +(6 rows) + +-- What a column can be read as without loss: its own type, and the wider one +-- Iceberg lets it be promoted to. +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k bigint, v text) ORDER BY k; + k | v +---+---- + 1 | v1 + 2 | v2 + 3 | v3 + 4 | v4 + 5 | v5 + 6 | v6 +(6 rows) + -- A query that returns nothing still produces a file: an empty file is a fact -- about the query, a missing one would be a fact about the writer. SELECT datalake_parquet_write(:'empty_file', @@ -206,19 +286,55 @@ SELECT * FROM datalake_parquet_read(:'batch_file') AS t (k int, v text) ORDER BY (6 rows) RESET iceberg.batch_rows; +-- The compression is asked of Arrow, so the name is spelled the way Arrow +-- spells it, in whatever case the option was typed. +SELECT datalake_parquet_write(:'gzip_file', + 'SELECT k, v FROM dlparq_pairs ORDER BY k', + 0, 'GZIP') AS rows_written; + rows_written +-------------- + 6 +(1 row) + +SELECT count(*) AS rows_read_gzip +FROM datalake_parquet_read(:'gzip_file') AS t (k int, v text); + rows_read_gzip +---------------- + 6 +(1 row) + -- Refusals. Each of these would otherwise be a wrong answer rather than an -- error: a column silently dropped, a value reinterpreted, a short read. CREATE TABLE dlparq_unsupported (k int, n numeric) DISTRIBUTED RANDOMLY; -SELECT datalake_parquet_write(:'roundtrip_file', +SELECT datalake_parquet_write(:'missing_file', 'SELECT * FROM dlparq_unsupported'); ERROR: iceberg: open_writer failed -DETAIL: arrow schema: column "n" has a type that lake tables cannot store yet -SELECT * FROM datalake_parquet_read(:'split_file') AS t (k bigint, v text); +DETAIL: arrow schema: column "n" has a type that cannot be stored in a lake table +-- A length limit has nowhere to live in the file, and padding would be written +-- as data; both are refused on the way in and on the way out. +CREATE TABLE dlparq_bounded (k int, v varchar(16), c char(5)) DISTRIBUTED RANDOMLY; +SELECT datalake_parquet_write(:'missing_file', + 'SELECT k, v FROM dlparq_bounded'); +ERROR: iceberg: open_writer failed +DETAIL: arrow schema: column "v" has a type that has a length limit, and a lake table has no way to store one; use text or varchar without a length +SELECT datalake_parquet_write(:'missing_file', + 'SELECT k, c FROM dlparq_bounded'); +ERROR: iceberg: open_writer failed +DETAIL: arrow schema: column "c" has a type that is padded to a declared length, and a lake table has no way to store that; use text +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int, v varchar(3)); +ERROR: iceberg: check_column failed +DETAIL: decode an Arrow column: a column cannot be read as character varying(3), which has a length limit, and a lake table has no way to store one; use text or varchar without a length +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int, v char(3)); +ERROR: iceberg: check_column failed +DETAIL: decode an Arrow column: a column cannot be read as character(3), which is padded to a declared length, and a lake table has no way to store that; use text +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k text, v text); ERROR: iceberg: check_column failed -DETAIL: decode an Arrow column: a column stored as Arrow type "i" cannot be read as bigint +DETAIL: decode an Arrow column: a column stored as Arrow type "i" cannot be read as text SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int); ERROR: the file does not have the number of columns the query expects DETAIL: The file has 2, the query expects 1. +SELECT * FROM datalake_parquet_read(:'split_file', 0, 0, '{1}') AS t (k int, v text); +ERROR: the field id list names 1 columns, the column definition list has 2 SELECT * FROM datalake_parquet_read(:'split_file', 9, 1) AS t (k int, v text); ERROR: iceberg: open_reader failed DETAIL: open a Parquet file: row groups 9..9 were asked for from a file that has 3 @@ -227,15 +343,34 @@ DETAIL: open a Parquet file: row groups 9..9 were asked for from a file that ha SELECT * FROM datalake_parquet_read(:'split_file', 1, 2147483647) AS t (k int, v text); ERROR: iceberg: open_reader failed DETAIL: open a Parquet file: row groups 1..2147483647 were asked for from a file that has 3 -SELECT datalake_parquet_write(:'split_file', 'SELECT 1', -1); +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', -1); ERROR: row group size must be between 0 and 1048576 -SELECT datalake_parquet_write(:'split_file', 'SELECT 1', 2000000000); +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', 2000000000); ERROR: row group size must be between 0 and 1048576 +-- Compression names Arrow does not know, and one it knows but Parquet cannot +-- use. Both are refused before a file is created. +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', 0, 'deflate'); +ERROR: iceberg: open_writer failed +DETAIL: open a Parquet file: "deflate" is not a compression Arrow knows +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', 0, 'lzo'); +ERROR: iceberg: open_writer failed +DETAIL: open a Parquet file: "lzo" is not a compression a Parquet file can use +-- A path that already exists is refused whatever is there: the writer only +-- ever removes a file it created, so what was there is still there afterwards. +SELECT datalake_parquet_write(:'split_file', 'SELECT 1'); +ERROR: iceberg: open_writer failed +DETAIL: create a Parquet file: "/tmp/datalake_fdw_regress_split.parquet" already exists +SELECT count(*) AS rows_still_there FROM dlparq_split; + rows_still_there +------------------ + 6 +(1 row) + -- PostgreSQL's timestamp range runs about 34 years past the last instant Arrow -- can hold as microseconds from 1970. Writing one of those has to be refused, -- because the shift would wrap and the value would land 292000 years before the -- epoch with the write reporting success. -SELECT datalake_parquet_write(:'batch_file', +SELECT datalake_parquet_write(:'missing_file', $$SELECT '294250-01-01 00:00:00'::timestamp$$); ERROR: iceberg: append_row failed DETAIL: append a value to an Arrow array: timestamp is too far in the future to be written to a data file (Invalid) @@ -246,5 +381,5 @@ ERROR: iceberg: open_reader failed \set VERBOSITY default SET client_min_messages = warning; DROP VIEW dlparq_roundtrip, dlparq_split; -DROP TABLE dlparq_src, dlparq_pairs, dlparq_unsupported; +DROP TABLE dlparq_src, dlparq_pairs, dlparq_unsupported, dlparq_bounded; RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql index 5b709d20921..c2c91077f01 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/format_parquet/sql/parquet_roundtrip.sql @@ -1,9 +1,10 @@ --- Parquet: every type a lake table can store, written to a file and read back, --- and the row group range that a scan will one day be divided at. +-- Parquet: every type a lake table can store, written to a file and read back; +-- the row group range that a scan will one day be divided at; and the field +-- ids that a table's columns are matched to a file's by. SET client_min_messages = warning; DROP VIEW IF EXISTS dlparq_roundtrip, dlparq_split; -DROP TABLE IF EXISTS dlparq_src, dlparq_pairs, dlparq_unsupported CASCADE; +DROP TABLE IF EXISTS dlparq_src, dlparq_pairs, dlparq_unsupported, dlparq_bounded CASCADE; CREATE EXTENSION IF NOT EXISTS datalake_fdw; CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; RESET client_min_messages; @@ -12,15 +13,21 @@ RESET client_min_messages; -- would depend on where the test ran rather than on what the file holds. SET TimeZone = 'UTC'; --- Fixed file names rather than a unique one per run: the writer truncates, so a --- run reuses what the last one left instead of adding to it. Nothing reachable --- from SQL can remove a file, so unique names would accumulate forever. +-- Fixed file names, removed before the run rather than overwritten during it: +-- the writer refuses a path that already exists, so what the last run left has +-- to go first. COPY TO PROGRAM runs on the coordinator, which is where these +-- functions read and write. \set roundtrip_file '/tmp/datalake_fdw_regress_roundtrip.parquet' \set split_file '/tmp/datalake_fdw_regress_split.parquet' +\set swapped_file '/tmp/datalake_fdw_regress_swapped.parquet' \set empty_file '/tmp/datalake_fdw_regress_empty.parquet' \set batch_file '/tmp/datalake_fdw_regress_batch.parquet' +\set gzip_file '/tmp/datalake_fdw_regress_gzip.parquet' \set missing_file '/tmp/datalake_fdw_regress_does_not_exist.parquet' +COPY (SELECT 1) TO PROGRAM 'rm -f /tmp/datalake_fdw_regress_*.parquet'; +-- varchar without a length, because a length is what a lake table cannot hold; +-- see the refusals at the end. CREATE TABLE dlparq_src ( c_bool boolean, c_int2 smallint, @@ -29,31 +36,35 @@ CREATE TABLE dlparq_src ( c_float4 real, c_float8 double precision, c_text text, - c_varchar varchar(16), - c_bpchar char(5), + c_varchar varchar, c_bytea bytea, c_date date, + c_time time, c_ts timestamp, - c_tstz timestamptz + c_tstz timestamptz, + c_uuid uuid ) DISTRIBUTED RANDOMLY; -- The dates and timestamps are chosen around both epochs: PostgreSQL counts -- from 2000-01-01 and Arrow from 1970-01-01, and a value on either side of --- 1970 is what tells a wrong shift from a right one. A row of nulls is here --- because a validity bitmap that is never exercised is a bitmap that has not --- been tested. +-- 1970 is what tells a wrong shift from a right one. 24:00:00 is the one time +-- PostgreSQL admits past the end of the day. A row of nulls is here because a +-- validity bitmap that is never exercised is a bitmap that has not been tested. INSERT INTO dlparq_src VALUES (true, 1, 100, 1000, 1.5, 2.5, - 'hello', 'varchar', 'abc', '\x0102'::bytea, - '1970-01-01', '1970-01-01 00:00:00', '1970-01-01 00:00:00+00'), + 'hello', 'varchar', '\x0102'::bytea, + '1970-01-01', '00:00:00', '1970-01-01 00:00:00', '1970-01-01 00:00:00+00', + '00000000-0000-0000-0000-000000000000'), (false, -2, -200, -2000, -1.5, -2.5, - 'a longer string with 中文', 'x', '', '\x'::bytea, - '2000-01-01', '2000-01-01 12:34:56.789012', '2000-01-01 12:34:56.789012+00'), + 'a longer string with 中文', 'x', '\x'::bytea, + '2000-01-01', '12:34:56.789012', '2000-01-01 12:34:56.789012', '2000-01-01 12:34:56.789012+00', + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), (true, 32767, 2147483647, 9223372036854775807, 3.25, 1e300, - '', 'z', 'zzzzz', '\xdeadbeef'::bytea, - '1969-12-31', '1969-12-31 23:59:59.999999', '1969-12-31 23:59:59.999999+00'), + '', 'z', '\xdeadbeef'::bytea, + '1969-12-31', '24:00:00', '1969-12-31 23:59:59.999999', '1969-12-31 23:59:59.999999+00', + 'ffffffff-ffff-ffff-ffff-ffffffffffff'), (NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); SELECT datalake_parquet_write(:'roundtrip_file', 'SELECT * FROM dlparq_src') AS rows_written; @@ -70,12 +81,13 @@ CREATE VIEW dlparq_roundtrip AS c_float4 real, c_float8 double precision, c_text text, - c_varchar varchar(16), - c_bpchar char(5), + c_varchar varchar, c_bytea bytea, c_date date, + c_time time, c_ts timestamp, - c_tstz timestamptz); + c_tstz timestamptz, + c_uuid uuid); SELECT * FROM dlparq_roundtrip ORDER BY c_int4; @@ -86,13 +98,11 @@ FROM ((TABLE dlparq_src EXCEPT ALL TABLE dlparq_roundtrip) UNION ALL (TABLE dlparq_roundtrip EXCEPT ALL TABLE dlparq_src)) d; --- The comparison above cannot see the padding of a char(n): bpchar equality --- ignores trailing spaces, so a value that came back three characters long --- would still have compared equal to the five it went in as. -SELECT octet_length(c_bpchar) AS bpchar_bytes, - octet_length(c_text) AS text_bytes, +-- The bytes as well as the values: a string that came back re-encoded would +-- still compare equal. +SELECT octet_length(c_text) AS text_bytes, octet_length(c_bytea) AS bytea_bytes -FROM dlparq_roundtrip ORDER BY 1, 2, 3; +FROM dlparq_roundtrip ORDER BY 1, 2; -- Row groups are the unit a scan is divided at, so reading the parts has to add -- up to reading the whole -- no row seen twice, none missed. @@ -121,6 +131,29 @@ ORDER BY 1, 2; SELECT count(*) AS rows_from_the_second_on FROM datalake_parquet_read(:'split_file', 1) AS t (k int, v text); +-- A table's columns are matched to a file's by field id, never by position. +-- The writer numbered these 1 (k) and 2 (v); asking for them the other way +-- round gives them the other way round, and asking for an id the file does not +-- have -- a column added to the table after the file was written -- gives NULL. +SELECT * FROM datalake_parquet_read(:'split_file', 0, 0, '{2, 1, 3}') + AS t (v text, k int, added int) ORDER BY k; +SELECT * FROM datalake_parquet_read(:'split_file', 0, 0, '{2}') + AS t (v text) ORDER BY v; +-- Nothing the file has: as many rows as the file, all NULL. +SELECT count(*) AS rows_of_nothing, count(x) AS values_of_nothing +FROM datalake_parquet_read(:'split_file', 0, 0, '{9}') AS t (x int); + +-- The ids follow the data that was written, not the names: a file written with +-- the columns the other way round holds v under 1 and k under 2. +SELECT datalake_parquet_write(:'swapped_file', + 'SELECT v, k FROM dlparq_pairs ORDER BY k') AS rows_written; +SELECT * FROM datalake_parquet_read(:'swapped_file', 0, 0, '{2, 1}') + AS t (k int, v text) ORDER BY k; + +-- What a column can be read as without loss: its own type, and the wider one +-- Iceberg lets it be promoted to. +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k bigint, v text) ORDER BY k; + -- A query that returns nothing still produces a file: an empty file is a fact -- about the query, a missing one would be a fact about the writer. SELECT datalake_parquet_write(:'empty_file', @@ -139,28 +172,57 @@ SELECT datalake_parquet_write(:'batch_file', SELECT * FROM datalake_parquet_read(:'batch_file') AS t (k int, v text) ORDER BY k; RESET iceberg.batch_rows; +-- The compression is asked of Arrow, so the name is spelled the way Arrow +-- spells it, in whatever case the option was typed. +SELECT datalake_parquet_write(:'gzip_file', + 'SELECT k, v FROM dlparq_pairs ORDER BY k', + 0, 'GZIP') AS rows_written; +SELECT count(*) AS rows_read_gzip +FROM datalake_parquet_read(:'gzip_file') AS t (k int, v text); + -- Refusals. Each of these would otherwise be a wrong answer rather than an -- error: a column silently dropped, a value reinterpreted, a short read. CREATE TABLE dlparq_unsupported (k int, n numeric) DISTRIBUTED RANDOMLY; -SELECT datalake_parquet_write(:'roundtrip_file', +SELECT datalake_parquet_write(:'missing_file', 'SELECT * FROM dlparq_unsupported'); -SELECT * FROM datalake_parquet_read(:'split_file') AS t (k bigint, v text); +-- A length limit has nowhere to live in the file, and padding would be written +-- as data; both are refused on the way in and on the way out. +CREATE TABLE dlparq_bounded (k int, v varchar(16), c char(5)) DISTRIBUTED RANDOMLY; +SELECT datalake_parquet_write(:'missing_file', + 'SELECT k, v FROM dlparq_bounded'); +SELECT datalake_parquet_write(:'missing_file', + 'SELECT k, c FROM dlparq_bounded'); +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int, v varchar(3)); +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int, v char(3)); + +SELECT * FROM datalake_parquet_read(:'split_file') AS t (k text, v text); SELECT * FROM datalake_parquet_read(:'split_file') AS t (k int); +SELECT * FROM datalake_parquet_read(:'split_file', 0, 0, '{1}') AS t (k int, v text); SELECT * FROM datalake_parquet_read(:'split_file', 9, 1) AS t (k int, v text); -- A row group range that only overflowed arithmetic would let through: the -- count is rejected rather than turned into a two-billion-entry list. SELECT * FROM datalake_parquet_read(:'split_file', 1, 2147483647) AS t (k int, v text); -SELECT datalake_parquet_write(:'split_file', 'SELECT 1', -1); -SELECT datalake_parquet_write(:'split_file', 'SELECT 1', 2000000000); +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', -1); +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', 2000000000); + +-- Compression names Arrow does not know, and one it knows but Parquet cannot +-- use. Both are refused before a file is created. +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', 0, 'deflate'); +SELECT datalake_parquet_write(:'missing_file', 'SELECT 1', 0, 'lzo'); + +-- A path that already exists is refused whatever is there: the writer only +-- ever removes a file it created, so what was there is still there afterwards. +SELECT datalake_parquet_write(:'split_file', 'SELECT 1'); +SELECT count(*) AS rows_still_there FROM dlparq_split; -- PostgreSQL's timestamp range runs about 34 years past the last instant Arrow -- can hold as microseconds from 1970. Writing one of those has to be refused, -- because the shift would wrap and the value would land 292000 years before the -- epoch with the write reporting success. -SELECT datalake_parquet_write(:'batch_file', +SELECT datalake_parquet_write(:'missing_file', $$SELECT '294250-01-01 00:00:00'::timestamp$$); -- Arrow words this one, and its wording is not ours to depend on. @@ -170,5 +232,5 @@ SELECT * FROM datalake_parquet_read(:'missing_file') AS t (k int, v text); SET client_min_messages = warning; DROP VIEW dlparq_roundtrip, dlparq_split; -DROP TABLE dlparq_src, dlparq_pairs, dlparq_unsupported; +DROP TABLE dlparq_src, dlparq_pairs, dlparq_unsupported, dlparq_bounded; RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out index db5fde07cf4..d6974b509f7 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out @@ -13,7 +13,8 @@ DROP TABLE IF EXISTS dlskel_bad_tablespace, dlskel_bad_missing_server, dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, - dlskel_bad_repl, dlskel_bad_purge + dlskel_bad_repl, dlskel_bad_purge, dlskel_bad_numeric, dlskel_bad_varchar, + dlskel_bad_char, dlskel_bad_like, dlskel_types CASCADE; DROP SERVER IF EXISTS dlskel_cat CASCADE; DROP SERVER IF EXISTS dlskel_cat2 CASCADE; @@ -77,13 +78,13 @@ CREATE TABLE dlskel_r (a int, b text) WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); NOTICE: stub engine: created iceberg table "public.dlskel_r" in catalog "dlskel_cat" SELECT * FROM dlskel_r; -ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.3:40000 pid=96554) INSERT INTO dlskel_r VALUES (1, 'x'); -ERROR: iceberg: INSERT is not supported yet (seg1 172.17.0.2:40001 pid=92002) +ERROR: iceberg: INSERT is not supported yet (seg0 172.17.0.3:40000 pid=96554) UPDATE dlskel_r SET a = 1; -ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.2:40000 pid=92001) +ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.3:40000 pid=96554) DELETE FROM dlskel_r; -ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.2:40000 pid=92001) +ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.3:40000 pid=96554) COPY dlskel_r FROM stdin; ERROR: iceberg: INSERT is not supported yet CONTEXT: COPY dlskel_r, line 1 @@ -92,7 +93,7 @@ ERROR: iceberg: SELECT is not supported yet CREATE INDEX ON dlskel_r (a); ERROR: iceberg: CREATE INDEX is not supported yet SELECT * FROM dlskel_r TABLESAMPLE BERNOULLI (10); -ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.3:40000 pid=96554) -- A table from an earlier transaction takes the new-filelocator path rather -- than the access method's truncate callback, so this is the case that would -- silently report success if only the callback rejected it. @@ -122,7 +123,7 @@ SELECT count(*) AS heap_rows_kept FROM dlskel_heap; VACUUM FULL dlskel_r; ERROR: iceberg: VACUUM FULL on iceberg tables is not supported yet SELECT * FROM dlskel_r FOR UPDATE; -ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.3:40000 pid=96554) CREATE TABLE dlskel_bad_dist (a int) USING iceberg WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') @@ -166,6 +167,41 @@ CREATE TABLE dlskel_bad USING iceberg AS SELECT 1; ERROR: iceberg: CREATE TABLE AS / CREATE MATERIALIZED VIEW is not supported yet CREATE MATERIALIZED VIEW dlskel_mv USING iceberg AS SELECT 1; ERROR: iceberg: CREATE TABLE AS / CREATE MATERIALIZED VIEW is not supported yet +-- A column type a data file cannot hold is refused when the table is created, +-- not at its first write: nothing can ALTER a lake table's columns, so a table +-- accepted with one would be a table nothing can ever be put in. The rule is +-- the format layer's, asked through the same function the writer asks. +CREATE TABLE dlskel_bad_numeric (a int, n numeric) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: column "n" has type numeric, which cannot be stored in a lake table +-- A length limit has nowhere to live in a lake table, whose only string type is +-- unbounded, and a file written by anything else could break it. +CREATE TABLE dlskel_bad_varchar (a int, v varchar(16)) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: column "v" has type character varying(16), which has a length limit, and a lake table has no way to store one; use text or varchar without a length +CREATE TABLE dlskel_bad_char (a int, c char(5)) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: column "c" has type character(5), which is padded to a declared length, and a lake table has no way to store that; use text +-- LIKE copies its columns after this check has run, so it is refused rather +-- than let through unchecked. +CREATE TABLE dlskel_bad_like (LIKE dlskel_heap) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: LIKE is not supported yet +-- Every type a data file can hold, in one table; unbounded varchar included. +CREATE TABLE dlskel_types ( + c_bool boolean, c_int2 smallint, c_int4 integer, c_int8 bigint, + c_float4 real, c_float8 double precision, c_text text, c_varchar varchar, + c_bytea bytea, c_date date, c_time time, c_ts timestamp, + c_tstz timestamptz, c_uuid uuid, c_serial serial) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_types" in catalog "dlskel_cat" +DROP TABLE dlskel_types; +NOTICE: stub engine: dropped iceberg table "public.dlskel_types" from catalog "dlskel_cat", keeping data -- Converting a heap into a lake table has to be refused too: the relation is -- still a heap when the statement arrives, so the guard above does not see it. ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; @@ -417,7 +453,8 @@ DROP TABLE IF EXISTS dlskel_bad_tablespace, dlskel_bad_missing_server, dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, - dlskel_bad_repl, dlskel_bad_purge + dlskel_bad_repl, dlskel_bad_purge, dlskel_bad_numeric, dlskel_bad_varchar, + dlskel_bad_char, dlskel_bad_like, dlskel_types CASCADE; DROP SERVER IF EXISTS dlskel_cat CASCADE; DROP SERVER IF EXISTS dlskel_cat2 CASCADE; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql index 6f376f61e49..35b36780be1 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql @@ -14,7 +14,8 @@ DROP TABLE IF EXISTS dlskel_bad_tablespace, dlskel_bad_missing_server, dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, - dlskel_bad_repl, dlskel_bad_purge + dlskel_bad_repl, dlskel_bad_purge, dlskel_bad_numeric, dlskel_bad_varchar, + dlskel_bad_char, dlskel_bad_like, dlskel_types CASCADE; DROP SERVER IF EXISTS dlskel_cat CASCADE; DROP SERVER IF EXISTS dlskel_cat2 CASCADE; @@ -149,6 +150,36 @@ CREATE TABLE dlskel_bad_tablespace (a int) CREATE TABLE dlskel_bad USING iceberg AS SELECT 1; CREATE MATERIALIZED VIEW dlskel_mv USING iceberg AS SELECT 1; +-- A column type a data file cannot hold is refused when the table is created, +-- not at its first write: nothing can ALTER a lake table's columns, so a table +-- accepted with one would be a table nothing can ever be put in. The rule is +-- the format layer's, asked through the same function the writer asks. +CREATE TABLE dlskel_bad_numeric (a int, n numeric) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +-- A length limit has nowhere to live in a lake table, whose only string type is +-- unbounded, and a file written by anything else could break it. +CREATE TABLE dlskel_bad_varchar (a int, v varchar(16)) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_char (a int, c char(5)) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +-- LIKE copies its columns after this check has run, so it is refused rather +-- than let through unchecked. +CREATE TABLE dlskel_bad_like (LIKE dlskel_heap) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +-- Every type a data file can hold, in one table; unbounded varchar included. +CREATE TABLE dlskel_types ( + c_bool boolean, c_int2 smallint, c_int4 integer, c_int8 bigint, + c_float4 real, c_float8 double precision, c_text text, c_varchar varchar, + c_bytea bytea, c_date date, c_time time, c_ts timestamp, + c_tstz timestamptz, c_uuid uuid, c_serial serial) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +DROP TABLE dlskel_types; + -- Converting a heap into a lake table has to be refused too: the relation is -- still a heap when the statement arrives, so the guard above does not see it. ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; @@ -339,7 +370,8 @@ DROP TABLE IF EXISTS dlskel_bad_tablespace, dlskel_bad_missing_server, dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, - dlskel_bad_repl, dlskel_bad_purge + dlskel_bad_repl, dlskel_bad_purge, dlskel_bad_numeric, dlskel_bad_varchar, + dlskel_bad_char, dlskel_bad_like, dlskel_types CASCADE; DROP SERVER IF EXISTS dlskel_cat CASCADE; DROP SERVER IF EXISTS dlskel_cat2 CASCADE; From 9e8de385eaad39e350921c1a32ecd29a08f47eab Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:25 +0800 Subject: [PATCH 3/9] datalake_fdw: add a pluggable storage layer Where a lake table's files live was decided by a switch on the URI scheme, and only s3 and hdfs were written into it. This replaces that with a contract: a backend answers one question -- given a location and its options, which arrow::fs::FileSystem reads and writes it -- and everything else stays on this extension's side of the boundary. What lands here: - The contract, in five headers that "make install" puts under $(includedir_server)/extension/datalake_fdw/, so a backend can be built outside this tree against nothing else. It carries an ABI version, the Arrow version, and a fingerprint over the compiler major and _GLIBCXX_USE_CXX11_ABI, because shared_ptr and arrow::Result cross the boundary by value. - A registry keyed by URI scheme, found through a rendezvous variable. Registration happens during preload and in any order: the registering side pulls in datalake_fdw through load_external_function, inside a PG_TRY so an ereport cannot unwind through a plug-in's C++ frames. Each process initializes a backend at its first mount and registers the matching finalizer then, because on_exit_reset() clears what a postmaster child inherits. - The facade over that filesystem: opening, listing, deleting, turning a URI into a native path, and classifying failures into DlErrCode. Backends never re-parse a URI, and an empty listing is ruled not-found here rather than by each backend, because object storage and a filesystem disagree about what an empty prefix means. - A file backend for shared mounts, which creates with O_EXCL and, if it has to give up, removes the file this writer created and nothing else. - Credential scrubbing in one place. Every value under a key containing secret, token or password is remembered per process, and dl_error_set -- the single exit every DlErrCode error goes through -- removes it from the message, so an unexpected C++ exception and a third-party backend's own status are covered by construction. - A dltest backend, so the registration contract can be tested from inside this module. The parser now accepts file:// as well as s3://, and names what it rejects against the registry rather than a fixed list, so a scheme a plug-in registered can be written into a base_path. Three DETAIL lines in iceberg_am_reject change with it. The s3 backend is adapted to the new contract here and rewritten in the commit that follows. --- contrib/datalake_fdw/Makefile | 35 +- .../datalake_fdw/datalake_fdw_test--1.0.sql | 20 + contrib/datalake_fdw/exports.txt | 11 + .../src/am_iceberg/pg_iceberg_extensible.c | 8 + .../src/am_iceberg/pg_iceberg_options.c | 115 +++- .../src/common/backend_registry.cpp | 297 ++++++++-- .../src/common/backend_registry.h | 63 +-- .../src/common/datalake_location.h | 19 +- .../src/common/file_system_wrapper.cpp | 530 ++++++++++++++++-- .../src/common/file_system_wrapper.h | 6 +- .../src/common/local_file_system.cpp | 215 +++++++ .../src/common/local_file_system.h | 64 +++ .../src/common/s3_file_system.cpp | 261 --------- .../datalake_fdw/src/common/storage_arrow.h | 79 +++ .../datalake_fdw/src/common/storage_backend.h | 144 +++++ .../src/common/storage_backend_register.h | 52 ++ .../datalake_fdw/src/test/datalake_fdw_test.c | 336 +++++++++++ .../src/test/storage_test_backend.cpp | 72 +++ .../iceberg_am/expected/iceberg_am_reject.out | 6 +- .../storage_local/expected/storage_local.out | 128 +++++ .../smoke/storage_local/sql/storage_local.sql | 71 +++ 21 files changed, 2066 insertions(+), 466 deletions(-) create mode 100644 contrib/datalake_fdw/src/common/local_file_system.cpp create mode 100644 contrib/datalake_fdw/src/common/local_file_system.h delete mode 100644 contrib/datalake_fdw/src/common/s3_file_system.cpp create mode 100644 contrib/datalake_fdw/src/common/storage_arrow.h create mode 100644 contrib/datalake_fdw/src/common/storage_backend.h create mode 100644 contrib/datalake_fdw/src/common/storage_backend_register.h create mode 100644 contrib/datalake_fdw/src/test/storage_test_backend.cpp create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/sql/storage_local.sql diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile index a9ad29cd33e..1540cb14b23 100644 --- a/contrib/datalake_fdw/Makefile +++ b/contrib/datalake_fdw/Makefile @@ -51,10 +51,18 @@ OBJS = \ src/common/dl_option_util.o \ src/common/parser_option.o \ src/common/file_system_wrapper.o \ - src/common/s3_file_system.o \ + src/common/local_file_system.o \ src/common/backend_registry.o \ + src/test/storage_test_backend.o \ src/test/datalake_fdw_test.o +HEADERS = \ + src/common/storage_backend.h \ + src/common/storage_backend_register.h \ + src/common/datalake_location.h \ + src/common/dl_kv.h \ + src/common/dl_err.h + # libparquet is written in terms of Arrow's types, so linking one links both. PKG_CONFIG ?= pkg-config ARROW_MODULES = arrow parquet @@ -106,6 +114,8 @@ REGRESS_OPTS = --temp-config=$(srcdir)/datalake_fdw.conf \ # A second category, and pg_regress takes one --inputdir, so it is a second run. FORMAT_PARQUET_REGRESS = parquet_roundtrip FORMAT_PARQUET_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/format_parquet +STORAGE_LOCAL_REGRESS = storage_local +STORAGE_LOCAL_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_local EXTRA_CLEAN = exports_darwin.list exports.map @@ -157,7 +167,12 @@ $(shlib): $(EXPORT_LIST) # wins over the one in REGRESS_OPTS. submake and REGRESS_PREP are the same # prerequisites pgxs.mk gives its own targets, so that a parallel make cannot # start pg_regress before it has been built. -installcheck: installcheck-format-parquet +# Each category is a separate pg_regress run against the same database, so +# they have to go one at a time: "make -j" would otherwise start both, and +# two runs creating the same extension in the same database race. The +# categories are chained through their prerequisites rather than listed side +# by side, which is what keeps the order under a parallel make. +installcheck: installcheck-storage-local installcheck-format-parquet: submake $(REGRESS_PREP) $(pg_regress_installcheck) $(REGRESS_OPTS) \ @@ -165,14 +180,28 @@ installcheck-format-parquet: submake $(REGRESS_PREP) .PHONY: installcheck-format-parquet +installcheck-storage-local: submake $(REGRESS_PREP) installcheck-format-parquet + $(pg_regress_installcheck) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_LOCAL_INPUTDIR) $(STORAGE_LOCAL_REGRESS) + +.PHONY: installcheck-storage-local + # "make check" is in-tree only -- under PGXS pgxs.mk refuses the target -- and # it is the only run that supplies the temp-config that preloads this module. ifndef USE_PGXS -check: check-format-parquet +# Chained for the same reason as installcheck above, and here it matters more: +# every category would otherwise start its own temp instance in ./tmp_check. +check: check-storage-local check-format-parquet: submake $(REGRESS_PREP) $(pg_regress_check) $(REGRESS_OPTS) \ --inputdir=$(FORMAT_PARQUET_INPUTDIR) $(FORMAT_PARQUET_REGRESS) .PHONY: check-format-parquet + +check-storage-local: submake $(REGRESS_PREP) check-format-parquet + $(pg_regress_check) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_LOCAL_INPUTDIR) $(STORAGE_LOCAL_REGRESS) + +.PHONY: check-storage-local endif diff --git a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql index 9171dbcd86f..ee512382d48 100644 --- a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql +++ b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql @@ -53,3 +53,23 @@ CREATE FUNCTION datalake_parquet_read(path text, RETURNS SETOF record AS 'MODULE_PATHNAME' LANGUAGE C STRICT EXECUTE ON COORDINATOR; REVOKE EXECUTE ON FUNCTION datalake_parquet_read(text, int, int, int[]) FROM PUBLIC; + +-- Test-only storage contract functions. +CREATE FUNCTION datalake_storage_write_text(uri text, content text, + kv text[] DEFAULT NULL) +RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; +CREATE FUNCTION datalake_storage_read_text(uri text, kv text[] DEFAULT NULL) +RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; +CREATE FUNCTION datalake_storage_list(uri text, kv text[] DEFAULT NULL) +RETURNS SETOF text AS 'MODULE_PATHNAME' +LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; +CREATE FUNCTION datalake_storage_probe(scheme text) +RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; +CREATE FUNCTION datalake_storage_register_bad(kind text) +RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; + +REVOKE EXECUTE ON FUNCTION datalake_storage_write_text(text, text, text[]) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_storage_read_text(text, text[]) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_storage_list(text, text[]) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_storage_probe(text) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_storage_register_bad(text) FROM PUBLIC; diff --git a/contrib/datalake_fdw/exports.txt b/contrib/datalake_fdw/exports.txt index e89c850b512..9f6932ce60f 100644 --- a/contrib/datalake_fdw/exports.txt +++ b/contrib/datalake_fdw/exports.txt @@ -30,6 +30,7 @@ pg_finfo_iceberg_catalog_fdw_validator iceberg_catalog_fdw_validator pg_finfo_iceberg_volume_fdw_validator iceberg_volume_fdw_validator +datalake_register_storage_backend # datalake_fdw_test: not part of what this module offers, but a SQL-callable # function has to be found by name in the library like any other. @@ -37,3 +38,13 @@ pg_finfo_datalake_parquet_write datalake_parquet_write pg_finfo_datalake_parquet_read datalake_parquet_read +pg_finfo_datalake_storage_write_text +datalake_storage_write_text +pg_finfo_datalake_storage_read_text +datalake_storage_read_text +pg_finfo_datalake_storage_list +datalake_storage_list +pg_finfo_datalake_storage_probe +datalake_storage_probe +pg_finfo_datalake_storage_register_bad +datalake_storage_register_bad diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c index 9eec658aa4f..a3302905556 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c @@ -62,6 +62,8 @@ PG_MODULE_MAGIC; +extern DlErrCode datalake_register_test_storage_backend(void); + static ProcessUtility_hook_type prev_ProcessUtility_hook; static bool iceberg_is_effective_am(const char *accessMethod); @@ -1010,6 +1012,8 @@ pg_iceberg_ProcessUtility(PlannedStmt *pstmt, void _PG_init(void) { + DlErrCode test_backend_rc; + if (!process_shared_preload_libraries_in_progress) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), @@ -1021,6 +1025,10 @@ _PG_init(void) pg_iceberg_register_reloptions(); DatalakeRegisterMetaEngines(); datalake_register_storage_backends(); + test_backend_rc = datalake_register_test_storage_backend(); + if (test_backend_rc != DL_OK && test_backend_rc != DL_ERR_ALREADY_EXISTS) + dl_error_report(ERROR, test_backend_rc, + "register dltest storage backend"); prev_ProcessUtility_hook = ProcessUtility_hook; ProcessUtility_hook = pg_iceberg_ProcessUtility; diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c index 6475546223c..4d1aa2ffc95 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c @@ -61,6 +61,7 @@ static relopt_kind iceberg_relopt_kind; static char *iceberg_relopt_string(IcebergRelOptions *opts, int off); static MetaKv *defelems_to_kvs(List *options, int *n_props); static DlErrCode invalid_location(char **errdetail, char *detail); +static char *redacted_location_uri(const char *uri); static bool s3_bucket_alnum(char ch); static bool s3_bucket_char(char ch); @@ -516,6 +517,64 @@ s3_bucket_char(char ch) return s3_bucket_alnum(ch) || ch == '.' || ch == '-'; } +/* + * A form of the URI that is safe to put in an error message. + * + * A rejected URI is quoted back so the user can see what was wrong with it, + * but the parts this parser rejects are exactly the parts that carry secrets: + * a query string holds a presigned signature, userinfo holds a password. So + * those are reported as present rather than reproduced. + */ +static char * +redacted_location_uri(const char *uri) +{ + const char *authority; + const char *cut; + StringInfoData safe; + + if (uri == NULL) + return pstrdup("(null)"); + + initStringInfo(&safe); + + authority = strstr(uri, "://"); + authority = authority == NULL ? uri : authority + 3; + + /* Everything up to the authority, then the authority without userinfo. */ + appendBinaryStringInfo(&safe, uri, (int) (authority - uri)); + cut = strpbrk(authority, "/?#"); + { + Size authority_len = cut == NULL ? + strlen(authority) : (Size) (cut - authority); + const char *at = memchr(authority, '@', authority_len); + + if (at != NULL) + { + appendStringInfoString(&safe, "***@"); + appendBinaryStringInfo(&safe, at + 1, + (int) (authority_len - (at + 1 - authority))); + } + else + appendBinaryStringInfo(&safe, authority, (int) authority_len); + } + + /* Then the path, with any query or fragment named but not repeated. */ + if (cut != NULL) + { + const char *tail = strpbrk(cut, "?#"); + + if (tail == NULL) + appendStringInfoString(&safe, cut); + else + { + appendBinaryStringInfo(&safe, cut, (int) (tail - cut)); + appendStringInfoString(&safe, *tail == '?' ? "?***" : "#***"); + } + } + + return safe.data; +} + DlErrCode pg_iceberg_parse_location(const char *uri, const char *endpoint, const char *region, DatalakeLocation *out, @@ -528,7 +587,9 @@ pg_iceberg_parse_location(const char *uri, const char *endpoint, Size authority_len; Size path_len; bool is_s3; + bool is_file; Size i; + char *safe_uri; Assert(out != NULL); memset(out, 0, sizeof(*out)); @@ -539,72 +600,84 @@ pg_iceberg_parse_location(const char *uri, const char *endpoint, return invalid_location(errdetail, pstrdup("location URI is null")); + safe_uri = redacted_location_uri(uri); + scheme_end = strstr(uri, "://"); if (scheme_end == NULL) return invalid_location(errdetail, psprintf("location URI \"%s\" is missing \"://\"", - uri)); + safe_uri)); scheme_len = scheme_end - uri; is_s3 = scheme_len == strlen(DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3) && strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, scheme_len) == 0; - if (!is_s3 && - !(scheme_len == strlen(DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS) && - strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS, scheme_len) == 0)) + is_file = scheme_len == strlen("file") && + strncmp(uri, "file", scheme_len) == 0; + if (!is_s3 && !is_file) return invalid_location(errdetail, - psprintf("location URI \"%s\" has unsupported scheme; expected %s or %s", - uri, - DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, - DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS)); + psprintf("location URI \"%s\" has unsupported scheme; expected s3 or file", + safe_uri)); + /* + * These three say what is wrong without repeating the URI. A query string + * carries a presigned signature and userinfo carries a password, and an + * error message is read in a log by people the credential was not issued + * to. The caller still reports the value it was given, which is the DDL + * the user just typed, so nothing is lost in diagnosing a typo. + */ if (strchr(uri, '?') != NULL) return invalid_location(errdetail, - psprintf("location URI \"%s\" must not contain a query", - uri)); + pstrdup("location URI must not contain a query")); if (strchr(uri, '#') != NULL) return invalid_location(errdetail, - psprintf("location URI \"%s\" must not contain a fragment", - uri)); + pstrdup("location URI must not contain a fragment")); authority_start = scheme_end + 3; path_start = strchr(authority_start, '/'); authority_len = path_start == NULL ? strlen(authority_start) : (Size) (path_start - authority_start); - if (authority_len == 0) + if (is_s3 && authority_len == 0) return invalid_location(errdetail, psprintf("location URI \"%s\" has an empty authority", - uri)); + safe_uri)); + if (is_file && authority_len != 0) + return invalid_location(errdetail, + psprintf("file location URI \"%s\" must have an empty authority", + safe_uri)); + if (is_file && (path_start == NULL || path_start[0] != '/')) + return invalid_location(errdetail, + psprintf("file location URI \"%s\" must have an absolute path", + safe_uri)); if (memchr(authority_start, '@', authority_len) != NULL) return invalid_location(errdetail, - psprintf("location URI \"%s\" must not contain userinfo", - uri)); + pstrdup("location URI must not contain userinfo")); if (is_s3) { if (authority_len < 3 || authority_len > 63) return invalid_location(errdetail, psprintf("s3 bucket in location URI \"%s\" must be 3 to 63 characters", - uri)); + safe_uri)); if (!s3_bucket_alnum(authority_start[0]) || !s3_bucket_alnum(authority_start[authority_len - 1])) return invalid_location(errdetail, psprintf("s3 bucket in location URI \"%s\" must start and end with a lowercase letter or digit", - uri)); + safe_uri)); for (i = 0; i < authority_len; i++) { if (!s3_bucket_char(authority_start[i])) return invalid_location(errdetail, psprintf("s3 bucket in location URI \"%s\" contains an invalid character", - uri)); + safe_uri)); } } path_len = path_start == NULL ? 0 : strlen(path_start); - while (path_len > 0 && path_start[path_len - 1] == '/') + while (path_len > (is_file ? 1 : 0) && path_start[path_len - 1] == '/') path_len--; - out->schema_version = DATALAKE_LOCATION_SCHEMA_VERSION; + out->abi_version = DATALAKE_LOCATION_ABI_VERSION; out->scheme = pnstrdup(uri, scheme_len); out->authority = pnstrdup(authority_start, authority_len); out->path_prefix = path_len == 0 ? diff --git a/contrib/datalake_fdw/src/common/backend_registry.cpp b/contrib/datalake_fdw/src/common/backend_registry.cpp index 78fb04a5e33..9d74c0bb1fc 100644 --- a/contrib/datalake_fdw/src/common/backend_registry.cpp +++ b/contrib/datalake_fdw/src/common/backend_registry.cpp @@ -18,7 +18,7 @@ * under the License. * * backend_registry.cpp - * Registry of the storage backends, one per protocol. + * Process-local registry shared with storage backend plugins. * * IDENTIFICATION * contrib/datalake_fdw/src/common/backend_registry.cpp @@ -26,97 +26,286 @@ *------------------------------------------------------------------------- */ +#include +#include +#include + +#include +#include + +#include "common/storage_backend.h" #include "common/dl_pg_api.h" -#include +extern "C" +{ +#include "fmgr.h" +#include "miscadmin.h" +#include "storage/ipc.h" +#include "utils/memutils.h" +} #include "common/backend_registry.h" #include "common/dl_wrappers.h" -typedef struct DatalakeStorageBackend +#define DL_STORAGE_RENDEZVOUS_NAME "datalake_storage_registry_v1" +#define DL_STORAGE_V1_MIN_SIZE \ + (offsetof(DatalakeStorageBackend, finalize) + \ + sizeof(((DatalakeStorageBackend *) 0)->finalize)) + +typedef struct DatalakeStorageBackendEntry { - const char *scheme; - const struct DatalakeStorageOps *ops; -} DatalakeStorageBackend; + const DatalakeStorageBackend *backend; + bool initialized; + struct DatalakeStorageBackendEntry *next; +} DatalakeStorageBackendEntry; -/* Room for s3 and hdfs, plus space to grow without revisiting this. */ -static DatalakeStorageBackend storage_backends[4]; -static int nstorage_backends; +typedef struct DatalakeStorageRegistry +{ + DatalakeStorageBackendEntry *backends; + void *wrappers; /* reserved for a future wrapper chain */ + bool finalizer_registered; +} DatalakeStorageRegistry; -extern DlErrCode datalake_register_s3_backend(void); +extern DlErrCode datalake_register_local_backend(void); -static bool -storage_ops_are_complete(const struct DatalakeStorageOps *ops) +static DatalakeStorageRegistry * +storage_registry(bool create) { - /* - * A partially filled table would turn into a null call at the first - * operation the backend forgot, so refuse it at registration instead. - */ - return ops != NULL && - ops->fs_open != NULL && - ops->fs_close != NULL && - ops->fs_list != NULL && - ops->file_open != NULL && - ops->file_read != NULL && - ops->file_write != NULL && - ops->file_close != NULL && - ops->file_abort != NULL; + void **slot = NULL; + + /* Both calls can allocate and therefore must not longjmp through C++. */ + DL_WRAP_START; + { + slot = find_rendezvous_variable(DL_STORAGE_RENDEZVOUS_NAME); + if (create && *slot == NULL) + *slot = MemoryContextAllocZero(TopMemoryContext, + sizeof(DatalakeStorageRegistry)); + } + DL_WRAP_END; + + return slot == NULL ? NULL : + static_cast(*slot); } -DlErrCode -datalake_register_storage_backend(const char *scheme, - const struct DatalakeStorageOps *ops) +static void +storage_backends_finalize(int code, Datum arg) { - int i; + DatalakeStorageRegistry *registry = NULL; + DatalakeStorageBackendEntry *entry; - if (scheme == NULL || scheme[0] == '\0' || !storage_ops_are_complete(ops)) - return DL_ERR_INVALID_OPTION; + (void) code; + (void) arg; - for (i = 0; i < nstorage_backends; i++) + try + { + registry = storage_registry(false); + } + catch (...) { - if (strcmp(storage_backends[i].scheme, scheme) == 0) - return DL_ERR_ALREADY_EXISTS; + /* Process exit is a cleanup boundary: never throw or ereport here. */ + return; } - if (nstorage_backends >= (int) lengthof(storage_backends)) - return DL_ERR_INTERNAL; + if (registry == NULL) + return; - storage_backends[nstorage_backends].scheme = scheme; - storage_backends[nstorage_backends].ops = ops; - nstorage_backends++; + for (entry = registry->backends; entry != NULL; entry = entry->next) + { + if (entry->initialized && entry->backend->finalize != NULL) + { + try + { + DL_WRAP_START; + { + elog(DEBUG1, "datalake_fdw: finalizing storage backend \"%s\"", + entry->backend->uri_scheme); + } + DL_WRAP_END; + entry->backend->finalize(); + } + catch (...) + { + /* One plugin must not prevent the remaining finalizers. */ + } + } + entry->initialized = false; + } +} - return DL_OK; +static void +set_registration_error(const char *message) +{ + dl_error_set(DL_ERR_INVALID_OPTION, "register storage backend", NULL, + message); } -const struct DatalakeStorageOps * +extern "C" __attribute__((visibility("default"))) DlErrCode +datalake_register_storage_backend(const DatalakeStorageBackend *backend) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + DatalakeStorageRegistry *registry; + DatalakeStorageBackendEntry *entry; + char message[DL_ERR_MSG_LEN]; + + dl_error_reset(); + if (backend == NULL) + { + set_registration_error("expected a non-null storage backend, got null"); + rc = DL_ERR_INVALID_OPTION; + } + else if (backend->abi_version != DL_STORAGE_ABI_VERSION) + { + snprintf(message, sizeof(message), + "storage backend ABI version mismatch: expected %u, got %u", + DL_STORAGE_ABI_VERSION, backend->abi_version); + set_registration_error(message); + rc = DL_ERR_INVALID_OPTION; + } + else if (backend->struct_size < DL_STORAGE_V1_MIN_SIZE) + { + snprintf(message, sizeof(message), + "storage backend struct size mismatch: expected at least %zu, got %u", + (size_t) DL_STORAGE_V1_MIN_SIZE, backend->struct_size); + set_registration_error(message); + rc = DL_ERR_INVALID_OPTION; + } + else if (backend->arrow_version == NULL || + strcmp(backend->arrow_version, ARROW_VERSION_STRING) != 0) + { + snprintf(message, sizeof(message), + "storage backend Arrow version mismatch: expected \"%s\", got \"%s\"", + ARROW_VERSION_STRING, + backend->arrow_version == NULL ? "(null)" : backend->arrow_version); + set_registration_error(message); + rc = DL_ERR_INVALID_OPTION; + } + else if (backend->abi_fingerprint == NULL || + strcmp(backend->abi_fingerprint, + DL_STORAGE_ABI_FINGERPRINT) != 0) + { + snprintf(message, sizeof(message), + "storage backend ABI fingerprint mismatch: expected \"%s\", got \"%s\"", + DL_STORAGE_ABI_FINGERPRINT, + backend->abi_fingerprint == NULL ? + "(null)" : backend->abi_fingerprint); + set_registration_error(message); + rc = DL_ERR_INVALID_OPTION; + } + else if (backend->uri_scheme == NULL || backend->uri_scheme[0] == '\0' || + backend->mount == NULL) + { + set_registration_error("expected a scheme and mount function, got an incomplete storage backend"); + rc = DL_ERR_INVALID_OPTION; + } + else + { + registry = storage_registry(true); + for (entry = registry->backends; entry != NULL; entry = entry->next) + { + if (strcmp(entry->backend->uri_scheme, backend->uri_scheme) == 0) + break; + } + + if (entry != NULL) + { + snprintf(message, sizeof(message), + "storage backend scheme expected to be unique, got duplicate \"%s\"", + backend->uri_scheme); + dl_error_set(DL_ERR_ALREADY_EXISTS, + "register storage backend", NULL, message); + rc = DL_ERR_ALREADY_EXISTS; + } + else + { + DL_WRAP_START; + { + entry = static_cast( + MemoryContextAllocZero(TopMemoryContext, sizeof(*entry))); + } + DL_WRAP_END; + entry->backend = backend; + entry->next = registry->backends; + registry->backends = entry; + rc = DL_OK; + } + } + } + DL_ABI_GUARD_END(rc, "register_storage_backend"); + + return rc; +} + +const DatalakeStorageBackend * datalake_lookup_storage_backend(const char *scheme) { - int i; + DatalakeStorageRegistry *registry = storage_registry(false); + DatalakeStorageBackendEntry *entry; - if (scheme == NULL) + if (registry == NULL || scheme == NULL) return NULL; - for (i = 0; i < nstorage_backends; i++) + for (entry = registry->backends; entry != NULL; entry = entry->next) { - if (strcmp(storage_backends[i].scheme, scheme) == 0) - return storage_backends[i].ops; + if (strcmp(entry->backend->uri_scheme, scheme) == 0) + return entry->backend; } - return NULL; } +arrow::Status +datalake_initialize_storage_backend(const DatalakeStorageBackend *backend) +{ + DatalakeStorageRegistry *registry = storage_registry(false); + DatalakeStorageBackendEntry *entry; + + if (registry == NULL || backend == NULL) + return arrow::Status::Invalid("storage backend is not registered"); + + for (entry = registry->backends; entry != NULL; entry = entry->next) + { + if (entry->backend != backend) + continue; + if (entry->initialized) + return arrow::Status::OK(); + + arrow::Status status = arrow::Status::OK(); + + Assert(MyProcPid != PostmasterPid); + if (!registry->finalizer_registered) + { + DL_WRAP_START; + { + on_proc_exit(storage_backends_finalize, (Datum) 0); + } + DL_WRAP_END; + registry->finalizer_registered = true; + } + + if (backend->initialize != NULL) + status = backend->initialize(); + + if (status.ok()) + entry->initialized = true; + return status; + } + + return arrow::Status::Invalid("storage backend is not registered"); +} + extern "C" void datalake_register_storage_backends(void) { + DlErrCode rc = DL_ERR_INTERNAL; + DL_TRY { - DlErrCode rc = datalake_register_s3_backend(); - - /* Registering twice is harmless; anything else is a coding error. */ - if (rc != DL_OK && rc != DL_ERR_ALREADY_EXISTS) - ereport(ERROR, - (errmsg("datalake_fdw: could not register the s3 storage backend: %s", - dl_err_message(rc)))); + rc = datalake_register_local_backend(); } DL_CATCH_END(); + + if (rc != DL_OK && rc != DL_ERR_ALREADY_EXISTS) + dl_error_report(ERROR, rc, "register file storage backend"); } diff --git a/contrib/datalake_fdw/src/common/backend_registry.h b/contrib/datalake_fdw/src/common/backend_registry.h index 5432502ed42..3de70e6bddf 100644 --- a/contrib/datalake_fdw/src/common/backend_registry.h +++ b/contrib/datalake_fdw/src/common/backend_registry.h @@ -18,7 +18,7 @@ * under the License. * * backend_registry.h - * Registry of the storage backends, one per protocol. + * Internal access to the storage backend registry. * * IDENTIFICATION * contrib/datalake_fdw/src/common/backend_registry.h @@ -29,68 +29,19 @@ #ifndef BACKEND_REGISTRY_H #define BACKEND_REGISTRY_H -#include - -#include "common/file_system_wrapper.h" - #ifdef __cplusplus - -/* - * One storage protocol's implementation of the facade in - * common/file_system_wrapper.h. The operations mirror it one for one, so a - * backend is written against the same contract its callers see. - */ -struct DatalakeStorageOps -{ - DlErrCode (*fs_open) (const DatalakeLocation *location, - const DlKeyValue *credentials, int ncredentials, - DatalakeFileSystem *fs_out); - void (*fs_close) (DatalakeFileSystem fs); /* releases fs */ - DlErrCode (*fs_list) (DatalakeFileSystem fs, const char *prefix, - char ***names_out, int *nnames_out); - DlErrCode (*file_open) (DatalakeFileSystem fs, const char *path, - DatalakeFileMode mode, DatalakeFile *file_out); - DlErrCode (*file_read) (DatalakeFile file, void *buffer, int64_t length, - int64_t *nread); - DlErrCode (*file_write) (DatalakeFile file, const void *buffer, - int64_t length); - DlErrCode (*file_close) (DatalakeFile file); /* releases file */ - void (*file_abort) (DatalakeFile file); /* releases file */ -}; - -/* - * Every handle a backend hands out starts with this field, which is how the - * facade finds its way back to the right operations. A handle lives until a - * cleanup entry point consumes it; there is no closed-but-alive state, because - * keeping one would mean either leaking every handle or letting a backend free - * memory the facade still reads. - */ -struct DatalakeFileSystemData -{ - const struct DatalakeStorageOps *ops; -}; - -struct DatalakeFileData -{ - const struct DatalakeStorageOps *ops; -}; - -extern DlErrCode datalake_register_storage_backend(const char *scheme, - const struct DatalakeStorageOps *ops); -extern const struct DatalakeStorageOps *datalake_lookup_storage_backend(const char *scheme); - -#endif /* __cplusplus */ +#include "common/storage_backend.h" +extern const DatalakeStorageBackend *datalake_lookup_storage_backend( + const char *scheme); +extern arrow::Status datalake_initialize_storage_backend( + const DatalakeStorageBackend *backend); +#endif #ifdef __cplusplus extern "C" { #endif -/* - * Registration is an explicit call rather than a static initializer: the order - * static initializers run in a shared module is not something to depend on, - * and _PG_init is where this is meant to happen. - */ extern void datalake_register_storage_backends(void); #ifdef __cplusplus diff --git a/contrib/datalake_fdw/src/common/datalake_location.h b/contrib/datalake_fdw/src/common/datalake_location.h index 28bede6f71f..7e73bce279c 100644 --- a/contrib/datalake_fdw/src/common/datalake_location.h +++ b/contrib/datalake_fdw/src/common/datalake_location.h @@ -33,16 +33,17 @@ /* Canonical, versioned location form. URIs are parsed ONCE (options layer); * every backend receives only this struct and must never re-parse URIs. */ -typedef struct DatalakeLocation { - uint32_t schema_version; /* = 1 */ - char *scheme; /* v1 whitelist: "s3" | "hdfs" */ - char *authority; /* s3: bucket (validated); hdfs: namenode[:port] */ - char *path_prefix; /* normalized: always starts with '/', never ends with '/' - * (a bare "/" normalizes to "") */ - char *endpoint; /* optional, may be NULL */ - char *region; /* optional, may be NULL */ +typedef struct DatalakeLocation +{ + uint32_t abi_version; /* = DATALAKE_LOCATION_ABI_VERSION */ + char *scheme; /* v1 whitelist: "s3" | "file" */ + char *authority; /* s3: bucket (validated); file: empty */ + char *path_prefix; /* normalized: starts with '/', no trailing '/' + * except the file root itself */ + char *endpoint; /* optional, may be NULL */ + char *region; /* optional, may be NULL */ } DatalakeLocation; -#define DATALAKE_LOCATION_SCHEMA_VERSION 1 +#define DATALAKE_LOCATION_ABI_VERSION 1 /* Join paths as full = path_prefix + "/" + relative; relative never starts * with '/'. */ diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp index c0c8b2d3e51..d1c3bd51728 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -18,7 +18,7 @@ * under the License. * * file_system_wrapper.cpp - * Storage facade dispatching to the registered backend. + * Storage facade implemented once over Arrow filesystems. * * IDENTIFICATION * contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -26,48 +26,322 @@ *------------------------------------------------------------------------- */ -#include "common/dl_pg_api.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "common/storage_arrow.h" +#include "format/arrow_memory_pool.h" +#include "common/dl_pg_api.h" #include "common/backend_registry.h" #include "common/dl_wrappers.h" #include "common/file_system_wrapper.h" /* - * Dispatch only: each call finds the backend registered for the location's - * scheme and hands the work over. Nothing here is reachable from SQL in this - * skeleton, so what the regression suite asserts is the behaviour of the - * layers above. + * Credentials reach the facade once, as options at mount time, but a status + * that has to be redacted can come out of any later call. So the values are + * copied into the file system handle and every file opened on it shares that + * copy; nothing else in the module has to remember to pass them along. */ +typedef std::shared_ptr> DlSecrets; + +struct DatalakeFileSystemData +{ + std::shared_ptr fs; + std::string root; + const DatalakeStorageBackend *backend; + DlSecrets secrets; +}; + +struct DatalakeFileData +{ + std::shared_ptr fs; + std::string path; + std::shared_ptr input; + std::shared_ptr output; + DlSecrets secrets; +}; + +DlStatusDetail::DlStatusDetail(DlErrCode code, std::string type) + : code_(code), type_(std::move(type)) +{ +} + +const char * +DlStatusDetail::type_id() const +{ + return "datalake::DlStatusDetail"; +} + +std::string +DlStatusDetail::ToString() const +{ + return type_; +} + +DlErrCode +DlStatusDetail::code() const +{ + return code_; +} + +const std::string & +DlStatusDetail::type() const +{ + return type_; +} + +/* The option values worth hiding: the key names credentials arrive under. */ +std::vector +dl_storage_collect_secrets(const DlKeyValue *kv, int nkv) +{ + std::vector secrets; + + for (int i = 0; kv != NULL && i < nkv; i++) + { + if (kv[i].key == NULL || kv[i].value == NULL || + kv[i].value[0] == '\0') + continue; + + const std::string key(kv[i].key); + + if (key.find("secret") == std::string::npos && + key.find("token") == std::string::npos && + key.find("access_key") == std::string::npos) + continue; + + secrets.push_back(kv[i].value); + } + return secrets; +} + +static std::string +redact_secrets(std::string text, const std::vector *secrets) +{ + if (secrets == NULL) + return text; + + for (const std::string &value : *secrets) + { + std::string::size_type pos = 0; + + while ((pos = text.find(value, pos)) != std::string::npos) + { + text.replace(pos, value.size(), "***"); + pos += 3; + } + } + return text; +} + +/* + * Whether the text carries the token on its own rather than inside a longer + * run of digits or letters -- "404" the status, not the 404 in a port number + * or an object named log4040. + */ +static bool +contains_token(const std::string &text, const char *token) +{ + const std::string needle(token); + std::string::size_type pos = 0; + + while ((pos = text.find(needle, pos)) != std::string::npos) + { + std::string::size_type end = pos + needle.size(); + bool left = pos == 0 || + !isalnum(static_cast(text[pos - 1])); + bool right = end == text.size() || + !isalnum(static_cast(text[end])); + + if (left && right) + return true; + pos = end; + } + return false; +} + +DlErrCode +dl_storage_status_to_err(const arrow::Status &status, const char *operation, + const std::vector *secrets) +{ + DlErrCode code; + std::string type; + std::string message; + + if (status.ok()) + return DL_OK; + + if (status.detail() != NULL && + strcmp(status.detail()->type_id(), "datalake::DlStatusDetail") == 0) + { + const DlStatusDetail *detail = + static_cast(status.detail().get()); + + code = detail->code(); + type = detail->type(); + } + else if (status.IsNotImplemented()) + code = DL_ERR_NOT_SUPPORTED; + else if (status.IsAlreadyExists()) + code = DL_ERR_ALREADY_EXISTS; + else if (status.IsOutOfMemory()) + code = DL_ERR_OUT_OF_MEMORY; + else if (status.IsIOError()) + { + message = status.message(); + if (message.find("NoSuchKey") != std::string::npos || + message.find("NoSuchBucket") != std::string::npos || + message.find("does not exist") != std::string::npos || + message.find("No such file or directory") != std::string::npos || + contains_token(message, "404")) + code = DL_ERR_NOT_FOUND; + else + code = DL_ERR_IO; + } + else + code = DL_ERR_IO; + + if (type.empty()) + type = status.CodeAsString(); + if (message.empty()) + message = status.message(); + + /* Both fields reach the user, so both are redacted. */ + message = redact_secrets(std::move(message), secrets); + type = redact_secrets(std::move(type), secrets); + dl_error_set(code, operation, type.c_str(), message.c_str()); + return code; +} + +bool +dl_storage_path_is_safe(const char *relative) +{ + const char *segment = relative; + + if (relative == NULL || relative[0] == '\0') + return true; /* the mount root itself */ + if (relative[0] == '/') + return false; /* absolute: not relative to the mount */ + + while (segment != NULL) + { + const char *slash = strchr(segment, '/'); + size_t length = slash == NULL ? strlen(segment) : + static_cast(slash - segment); + + if ((length == 1 && segment[0] == '.') || + (length == 2 && segment[0] == '.' && segment[1] == '.')) + return false; + segment = slash == NULL ? NULL : slash + 1; + } + return true; +} + +/* Reports the rejection the caller should return for an unsafe path. */ +static DlErrCode +reject_unsafe_path(const char *operation, const char *relative) +{ + std::string message = "storage path \"" + std::string(relative) + + "\" must be relative to the volume and must not contain \".\" or \"..\""; + + dl_error_set(DL_ERR_INVALID_OPTION, operation, NULL, message.c_str()); + return DL_ERR_INVALID_OPTION; +} + +arrow::fs::FileSystem * +dl_storage_arrow_fs(DatalakeFileSystem fs) +{ + return fs == NULL ? NULL : fs->fs.get(); +} + +std::string +dl_storage_native_path(DatalakeFileSystem fs, const char *relative) +{ + if (fs == NULL || relative == NULL || relative[0] == '\0') + return fs == NULL ? std::string() : fs->root; + if (fs->root.empty()) + return relative; + if (fs->root.back() == '/') + return fs->root + relative; + return fs->root + "/" + relative; +} extern "C" DlErrCode -datalake_fs_open(const DatalakeLocation *location, - const DlKeyValue *credentials, int ncredentials, - DatalakeFileSystem *fs_out) +datalake_fs_open(const DatalakeLocation *location, const DlKeyValue *kv, + int nkv, DatalakeFileSystem *fs_out) { DlErrCode rc = DL_ERR_INTERNAL; DL_ABI_GUARD_BEGIN { - const struct DatalakeStorageOps *ops; - if (fs_out == NULL || location == NULL || location->scheme == NULL || - ncredentials < 0) - rc = DL_ERR_INVALID_OPTION; + nkv < 0 || (nkv > 0 && kv == NULL)) + rc = DL_ARG_ERROR("fs_open"); else { + const DatalakeStorageBackend *backend; + *fs_out = NULL; - ops = datalake_lookup_storage_backend(location->scheme); + dl_error_reset(); + backend = datalake_lookup_storage_backend(location->scheme); + if (backend == NULL) + { + std::string message = "no storage backend registered for scheme \"" + + std::string(location->scheme) + "\""; - if (ops == NULL) + dl_error_set(DL_ERR_NOT_SUPPORTED, "mount storage", NULL, + message.c_str()); rc = DL_ERR_NOT_SUPPORTED; + } else { - rc = ops->fs_open(location, credentials, ncredentials, fs_out); + auto secrets = std::make_shared>( + dl_storage_collect_secrets(kv, nkv)); + arrow::Status status = datalake_initialize_storage_backend(backend); + + if (!status.ok()) + rc = dl_storage_status_to_err(status, "initialize storage", + secrets.get()); + else + { + DatalakeStorageHost host; + + host.struct_size = sizeof(host); + host.pool = DlArrowMemoryPool(); + + auto mounted = backend->mount(location, kv, nkv, &host); + + if (!mounted.ok()) + rc = dl_storage_status_to_err(mounted.status(), + "mount storage", secrets.get()); + else if (mounted->fs == NULL) + { + dl_error_set(DL_ERR_INTERNAL, "mount storage", NULL, + "storage backend returned a null filesystem"); + rc = DL_ERR_INTERNAL; + } + else + { + std::unique_ptr handle( + new DatalakeFileSystemData()); - if (rc == DL_OK && *fs_out == NULL) - rc = DL_ERR_INTERNAL; - else if (rc == DL_OK) - (*fs_out)->ops = ops; + handle->fs = std::move(mounted->fs); + handle->root = std::move(mounted->root); + handle->backend = backend; + handle->secrets = secrets; + *fs_out = handle.release(); + rc = DL_OK; + } + } } } } @@ -81,18 +355,12 @@ datalake_fs_close(DatalakeFileSystem *fs) { DL_CLEANUP_GUARD_BEGIN { - /* - * Clear the caller's handle before releasing it, so that a repeated - * close -- the normal shape of resource-owner cleanup after an error - * that already closed things -- finds nothing to do instead of - * reaching a backend that has freed itself. - */ if (fs != NULL && *fs != NULL) { DatalakeFileSystem doomed = *fs; *fs = NULL; - doomed->ops->fs_close(doomed); + delete doomed; } } DL_CLEANUP_GUARD_END; @@ -106,14 +374,82 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, DL_ABI_GUARD_BEGIN { - if (fs == NULL || prefix == NULL || names_out == NULL || - nnames_out == NULL) - rc = DL_ERR_INVALID_OPTION; + if (fs == NULL || prefix == NULL || names_out == NULL || nnames_out == NULL) + rc = DL_ARG_ERROR("fs_list"); + else if (!dl_storage_path_is_safe(prefix)) + rc = reject_unsafe_path("list storage", prefix); else { + arrow::fs::FileSelector selector; + std::vector paths; + *names_out = NULL; *nnames_out = 0; - rc = fs->ops->fs_list(fs, prefix, names_out, nnames_out); + selector.base_dir = dl_storage_native_path(fs, prefix); + selector.recursive = true; + selector.allow_not_found = false; + auto infos = fs->fs->GetFileInfo(selector); + + if (!infos.ok()) + rc = dl_storage_status_to_err(infos.status(), "list storage", + fs->secrets.get()); + else + { + for (const auto &info : *infos) + { + if (info.IsFile()) + paths.push_back(info.path()); + } + std::sort(paths.begin(), paths.end()); + + /* The count leaves through an int, so it has to fit in one. */ + if (paths.size() > static_cast(INT_MAX)) + { + dl_error_set(DL_ERR_IO, "list storage", NULL, + "storage listing has too many entries to return"); + rc = DL_ERR_IO; + } + else + { + char **names = static_cast( + std::calloc(paths.size(), sizeof(char *))); + + if (!paths.empty() && names == NULL) + rc = dl_storage_status_to_err( + arrow::Status::OutOfMemory("allocating storage listing"), + "list storage", fs->secrets.get()); + else + { + size_t i = 0; + + for (; i < paths.size(); i++) + { + names[i] = static_cast( + std::malloc(paths[i].size() + 1)); + if (names[i] == NULL) + break; + std::memcpy(names[i], paths[i].c_str(), + paths[i].size() + 1); + } + + if (i != paths.size()) + { + while (i > 0) + std::free(names[--i]); + std::free(names); + rc = dl_storage_status_to_err( + arrow::Status::OutOfMemory("allocating storage listing"), + "list storage", fs->secrets.get()); + } + else + { + *names_out = names; + *nnames_out = static_cast(paths.size()); + rc = DL_OK; + } + } + } + } } } DL_ABI_GUARD_END(rc, "fs_list"); @@ -129,19 +465,79 @@ datalake_file_open(DatalakeFileSystem fs, const char *path, DL_ABI_GUARD_BEGIN { - if (file_out == NULL || fs == NULL || path == NULL) - rc = DL_ERR_INVALID_OPTION; - else if (mode != DATALAKE_FILE_READ && mode != DATALAKE_FILE_WRITE) - rc = DL_ERR_INVALID_OPTION; + if (file_out == NULL || fs == NULL || path == NULL || + (mode != DATALAKE_FILE_READ && mode != DATALAKE_FILE_WRITE)) + rc = DL_ARG_ERROR("file_open"); + else if (!dl_storage_path_is_safe(path)) + rc = reject_unsafe_path("open storage file", path); else { + std::string native_path = dl_storage_native_path(fs, path); + auto info = fs->fs->GetFileInfo(native_path); + *file_out = NULL; - rc = fs->ops->file_open(fs, path, mode, file_out); + if (!info.ok()) + rc = dl_storage_status_to_err(info.status(), "inspect storage file", + fs->secrets.get()); + else if (mode == DATALAKE_FILE_READ && + info->type() == arrow::fs::FileType::NotFound) + { + std::string message = "storage file \"" + native_path + + "\" does not exist"; - if (rc == DL_OK && *file_out == NULL) - rc = DL_ERR_INTERNAL; - else if (rc == DL_OK) - (*file_out)->ops = fs->ops; + dl_error_set(DL_ERR_NOT_FOUND, "open storage file", NULL, + message.c_str()); + rc = DL_ERR_NOT_FOUND; + } + else if (mode == DATALAKE_FILE_WRITE && + info->type() != arrow::fs::FileType::NotFound) + { + std::string message = "storage file \"" + native_path + + "\" already exists"; + + dl_error_set(DL_ERR_ALREADY_EXISTS, "create storage file", NULL, + message.c_str()); + rc = DL_ERR_ALREADY_EXISTS; + } + else + { + std::unique_ptr handle(new DatalakeFileData()); + + handle->fs = fs->fs; + handle->path = native_path; + handle->secrets = fs->secrets; + if (mode == DATALAKE_FILE_READ) + { + auto input = fs->fs->OpenInputFile(native_path); + + if (!input.ok()) + rc = dl_storage_status_to_err(input.status(), + "open storage file", + fs->secrets.get()); + else + { + handle->input = *input; + rc = DL_OK; + } + } + else + { + auto output = fs->fs->OpenOutputStream(native_path); + + if (!output.ok()) + rc = dl_storage_status_to_err(output.status(), + "create storage file", + fs->secrets.get()); + else + { + handle->output = *output; + rc = DL_OK; + } + } + + if (rc == DL_OK) + *file_out = handle.release(); + } } } DL_ABI_GUARD_END(rc, "file_open"); @@ -157,12 +553,22 @@ datalake_file_read(DatalakeFile file, void *buffer, int64_t length, DL_ABI_GUARD_BEGIN { - if (file == NULL || nread == NULL || length < 0) - rc = DL_ERR_INVALID_OPTION; + if (file == NULL || file->input == NULL || nread == NULL || length < 0 || + (length > 0 && buffer == NULL)) + rc = DL_ARG_ERROR("file_read"); else { + auto read = file->input->Read(length, buffer); + *nread = 0; - rc = file->ops->file_read(file, buffer, length, nread); + if (!read.ok()) + rc = dl_storage_status_to_err(read.status(), "read storage file", + file->secrets.get()); + else + { + *nread = *read; + rc = DL_OK; + } } } DL_ABI_GUARD_END(rc, "file_read"); @@ -177,10 +583,13 @@ datalake_file_write(DatalakeFile file, const void *buffer, int64_t length) DL_ABI_GUARD_BEGIN { - if (file == NULL || length < 0) - rc = DL_ERR_INVALID_OPTION; + if (file == NULL || file->output == NULL || length < 0 || + (length > 0 && buffer == NULL)) + rc = DL_ARG_ERROR("file_write"); else - rc = file->ops->file_write(file, buffer, length); + rc = dl_storage_status_to_err(file->output->Write(buffer, length), + "write storage file", + file->secrets.get()); } DL_ABI_GUARD_END(rc, "file_write"); @@ -195,18 +604,25 @@ datalake_file_close(DatalakeFile *file) DL_ABI_GUARD_BEGIN { if (file == NULL || *file == NULL) - rc = DL_ERR_INVALID_OPTION; + rc = DL_ARG_ERROR("file_close"); else { - DatalakeFile doomed = *file; + std::unique_ptr doomed(*file); + arrow::Status status; + + *file = NULL; /* - * The handle is consumed even when the close reports an error: - * the backend has released it either way, and there is nothing - * left to retry the close against. + * A stream cleans up after itself: the output stream removes what + * it created, and only that. Deleting by path from here would + * reach an object that some other writer created in the meantime. */ - *file = NULL; - rc = doomed->ops->file_close(doomed); + status = doomed->input != NULL ? doomed->input->Close() : + doomed->output->Close(); + if (!status.ok() && doomed->output != NULL) + (void) doomed->output->Abort(); + rc = dl_storage_status_to_err(status, "close storage file", + doomed->secrets.get()); } } DL_ABI_GUARD_END(rc, "file_close"); @@ -219,13 +635,15 @@ datalake_file_abort(DatalakeFile *file) { DL_CLEANUP_GUARD_BEGIN { - /* Cleared first, so a repeated abort finds nothing to do. */ if (file != NULL && *file != NULL) { - DatalakeFile doomed = *file; + std::unique_ptr doomed(*file); *file = NULL; - doomed->ops->file_abort(doomed); + if (doomed->output != NULL) + (void) doomed->output->Abort(); + else if (doomed->input != NULL) + (void) doomed->input->Close(); } } DL_CLEANUP_GUARD_END; diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.h b/contrib/datalake_fdw/src/common/file_system_wrapper.h index 37b6c037d2c..75eeddf1ead 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.h +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.h @@ -31,9 +31,9 @@ #include -#include "common/datalake_location.h" -#include "common/dl_err.h" -#include "common/dl_kv.h" +#include "datalake_location.h" +#include "dl_err.h" +#include "dl_kv.h" /* * A file system reached over one storage protocol, and an open file in it. diff --git a/contrib/datalake_fdw/src/common/local_file_system.cpp b/contrib/datalake_fdw/src/common/local_file_system.cpp new file mode 100644 index 00000000000..4d9c8186a88 --- /dev/null +++ b/contrib/datalake_fdw/src/common/local_file_system.cpp @@ -0,0 +1,215 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * local_file_system.cpp + * The create-only local storage backend. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/local_file_system.cpp + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "common/local_file_system.h" +#include "common/storage_backend.h" + +extern "C" +{ +#include "postgres.h" +#include "common/file_perm.h" +} + +namespace +{ + +/* + * The descriptor between open() and the point where Arrow takes it over. Any + * error in between has to undo the creation, or an empty file is left behind + * and every later attempt at the same path fails with ALREADY_EXISTS. + */ +class CreatedFile +{ +public: + CreatedFile(int fd, std::string path) : fd_(fd), path_(std::move(path)) {} + + ~CreatedFile() + { + if (fd_ >= 0) + { + (void) close(fd_); + (void) unlink(path_.c_str()); + } + } + + CreatedFile(const CreatedFile &) = delete; + CreatedFile &operator=(const CreatedFile &) = delete; + + int fd() const { return fd_; } + void release() { fd_ = -1; } + +private: + int fd_; + std::string path_; +}; + +/* + * Wraps the Arrow stream so that giving up removes what this writer created + * and nothing else. Only the path this stream created with O_EXCL is ever + * unlinked, which is what keeps a failed writer from destroying a file that + * belongs to somebody else. + */ +class CreateOnlyOutputStream : public arrow::io::OutputStream +{ +public: + CreateOnlyOutputStream(std::shared_ptr file, + std::string path) + : file_(std::move(file)), path_(std::move(path)) + { + } + + ~CreateOnlyOutputStream() override = default; + + arrow::Status Close() override + { + if (done_) + return arrow::Status::OK(); + done_ = true; + + arrow::Status status = file_->Close(); + + /* A close that fails leaves a file nobody asked for. */ + if (!status.ok()) + (void) unlink(path_.c_str()); + return status; + } + + /* + * Abort is the caller saying "this object must not exist". Close the + * descriptor, then remove the file this stream created. + */ + arrow::Status Abort() override + { + if (done_) + return arrow::Status::OK(); + done_ = true; + + arrow::Status status = file_->Close(); + + if (unlink(path_.c_str()) != 0 && errno != ENOENT) + return arrow::Status::IOError("could not remove \"", path_, + "\": ", strerror(errno)); + return status; + } + + bool closed() const override { return done_ || file_->closed(); } + + arrow::Result Tell() const override { return file_->Tell(); } + + arrow::Status Write(const void *data, int64_t nbytes) override + { + return file_->Write(data, nbytes); + } + + arrow::Status Write(const std::shared_ptr &data) override + { + return file_->Write(data); + } + + arrow::Status Flush() override { return file_->Flush(); } + +private: + std::shared_ptr file_; + std::string path_; + bool done_ = false; +}; + +} /* namespace */ + +arrow::Result> +LocalCreateOnlyFileSystem::OpenOutputStream( + const std::string &path, + const std::shared_ptr &metadata) +{ + int fd; + + (void) metadata; + fd = open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL, pg_file_create_mode); + if (fd < 0) + { + if (errno == EEXIST) + return arrow::Status::AlreadyExists("\"", path, "\" already exists"); + return arrow::Status::IOError("could not create \"", path, "\": ", + strerror(errno)); + } + + CreatedFile created(fd, path); + auto stream = arrow::io::FileOutputStream::Open(fd); + + if (!stream.ok()) + return stream.status(); /* CreatedFile closes and unlinks */ + + created.release(); /* the stream owns the descriptor now */ + return std::make_shared(*stream, path); +} + +static arrow::Result +mount_local(const DatalakeLocation *location, const DlKeyValue *kv, int nkv, + const DatalakeStorageHost *host) +{ + (void) kv; + (void) nkv; + + if (location == NULL || location->path_prefix == NULL) + return arrow::Status::Invalid("file location has no path"); + + DatalakeMountedFs mounted; + + mounted.fs = std::make_shared( + arrow::io::IOContext(dl_storage_host_pool(host))); + mounted.root = location->path_prefix; + return mounted; +} + +static const DatalakeStorageBackend local_storage_backend = { + DL_STORAGE_ABI_VERSION, + sizeof(DatalakeStorageBackend), + "file", + ARROW_VERSION_STRING, + DL_STORAGE_ABI_FINGERPRINT, + mount_local, + NULL, + NULL +}; + +DlErrCode +datalake_register_local_backend(void) +{ + return datalake_register_storage_backend(&local_storage_backend); +} diff --git a/contrib/datalake_fdw/src/common/local_file_system.h b/contrib/datalake_fdw/src/common/local_file_system.h new file mode 100644 index 00000000000..0ff89a40387 --- /dev/null +++ b/contrib/datalake_fdw/src/common/local_file_system.h @@ -0,0 +1,64 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * local_file_system.h + * The create-only local file system, shared with the test backend. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/local_file_system.h + * + *------------------------------------------------------------------------- + */ + +#ifndef LOCAL_FILE_SYSTEM_H +#define LOCAL_FILE_SYSTEM_H + +#include +#include + +#include +#include + +/* + * A local file system whose OpenOutputStream creates the file and fails if it + * is already there, and whose streams clean up after themselves. + * + * Arrow's own LocalFileSystem opens output with O_TRUNC, which would let a + * failed write destroy an existing file; this keeps the O_EXCL rule the + * Parquet writer has had since #1951. The returned stream owns what it + * created: Abort(), and a Close() that fails, unlink exactly the path this + * stream created, and nothing else ever deletes on its behalf. + * + * The test backend mounts the same implementation through SubTreeFileSystem, + * so the storage conformance cases exercise one implementation, not two. + */ +class LocalCreateOnlyFileSystem : public arrow::fs::LocalFileSystem +{ +public: + explicit LocalCreateOnlyFileSystem(const arrow::io::IOContext &io_context) + : arrow::fs::LocalFileSystem(io_context) + { + } + + arrow::Result> OpenOutputStream( + const std::string &path, + const std::shared_ptr &metadata) override; +}; + +#endif /* LOCAL_FILE_SYSTEM_H */ diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp deleted file mode 100644 index ebb7fadf2ad..00000000000 --- a/contrib/datalake_fdw/src/common/s3_file_system.cpp +++ /dev/null @@ -1,261 +0,0 @@ -/*------------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * s3_file_system.cpp - * The S3 storage backend. - * - * IDENTIFICATION - * contrib/datalake_fdw/src/common/s3_file_system.cpp - * - *------------------------------------------------------------------------- - */ - -#include "common/dl_pg_api.h" - -#include "common/backend_registry.h" -#include "common/dl_wrappers.h" - -#include - -/* - * The S3 backend, without an S3 client yet: the shape a backend takes is what - * this file establishes, so that the change adding a real client replaces - * method bodies rather than the structure around them. Every entry point - * reports that the operation is not supported. - */ -class S3FileSystem -{ -public: - DlErrCode - Initialize(const DatalakeLocation *location, const DlKeyValue *credentials, - int ncredentials) - { - (void) location; - (void) credentials; - (void) ncredentials; - - return DL_ERR_NOT_SUPPORTED; - } - - DlErrCode - OpenFile(const char *path, DatalakeFileMode mode, DatalakeFile *file_out) - { - (void) path; - (void) mode; - - if (file_out != NULL) - *file_out = NULL; - - return DL_ERR_NOT_SUPPORTED; - } - - DlErrCode - List(const char *prefix, char ***names_out, int *nnames_out) - { - (void) prefix; - - if (names_out != NULL) - *names_out = NULL; - if (nnames_out != NULL) - *nnames_out = 0; - - return DL_ERR_NOT_SUPPORTED; - } -}; - -/* - * A handle the facade can hold. - * - * Deriving from the C struct rather than embedding it as a first member is what - * makes recovering the handle defined behaviour: a derived-to-base pointer - * conversion and a static_cast back are guaranteed for any class, while the - * first-member trick is only guaranteed for standard-layout types -- which this - * is not, because of the unique_ptr. The C side still sees a plain - * DatalakeFileSystemData, since that is what the base subobject is. - */ -struct S3FileSystemHandle : public DatalakeFileSystemData -{ - std::unique_ptr impl; -}; - -static DlErrCode -s3_fs_open(const DatalakeLocation *location, const DlKeyValue *credentials, - int ncredentials, DatalakeFileSystem *fs_out) -{ - DlErrCode rc = DL_ERR_INTERNAL; - - DL_ABI_GUARD_BEGIN - { - if (fs_out == NULL) - rc = DL_ERR_INVALID_OPTION; - else - { - /* - * Owned by unique_ptr until the handle is published, so that an - * exception from the second allocation or from Initialize() -- - * which the guard below turns into an error code -- cannot leave - * the first allocation behind. - */ - std::unique_ptr handle(new S3FileSystemHandle()); - - *fs_out = NULL; - handle->impl.reset(new S3FileSystem()); - rc = handle->impl->Initialize(location, credentials, ncredentials); - - if (rc == DL_OK) - *fs_out = handle.release(); - } - } - DL_ABI_GUARD_END(rc, "s3_fs_open"); - - return rc; -} - -static void -s3_fs_close(DatalakeFileSystem fs) -{ - DL_CLEANUP_GUARD_BEGIN - { - /* The facade has already cleared its caller's handle. */ - delete static_cast(fs); - } - DL_CLEANUP_GUARD_END; -} - -static DlErrCode -s3_fs_list(DatalakeFileSystem fs, const char *prefix, char ***names_out, - int *nnames_out) -{ - DlErrCode rc = DL_ERR_INTERNAL; - - DL_ABI_GUARD_BEGIN - { - S3FileSystemHandle *handle = static_cast(fs); - - if (handle == NULL) - rc = DL_ERR_INVALID_OPTION; - else - rc = handle->impl->List(prefix, names_out, nnames_out); - } - DL_ABI_GUARD_END(rc, "s3_fs_list"); - - return rc; -} - -static DlErrCode -s3_file_open(DatalakeFileSystem fs, const char *path, DatalakeFileMode mode, - DatalakeFile *file_out) -{ - DlErrCode rc = DL_ERR_INTERNAL; - - DL_ABI_GUARD_BEGIN - { - S3FileSystemHandle *handle = static_cast(fs); - - if (handle == NULL) - rc = DL_ERR_INVALID_OPTION; - else - rc = handle->impl->OpenFile(path, mode, file_out); - } - DL_ABI_GUARD_END(rc, "s3_file_open"); - - return rc; -} - -static DlErrCode -s3_file_read(DatalakeFile file, void *buffer, int64_t length, int64_t *nread) -{ - DlErrCode rc = DL_ERR_INTERNAL; - - DL_ABI_GUARD_BEGIN - { - (void) file; - (void) buffer; - (void) length; - - if (nread != NULL) - *nread = 0; - - rc = DL_ERR_NOT_SUPPORTED; - } - DL_ABI_GUARD_END(rc, "s3_file_read"); - - return rc; -} - -static DlErrCode -s3_file_write(DatalakeFile file, const void *buffer, int64_t length) -{ - DlErrCode rc = DL_ERR_INTERNAL; - - DL_ABI_GUARD_BEGIN - { - (void) file; - (void) buffer; - (void) length; - - rc = DL_ERR_NOT_SUPPORTED; - } - DL_ABI_GUARD_END(rc, "s3_file_write"); - - return rc; -} - -static DlErrCode -s3_file_close(DatalakeFile file) -{ - DlErrCode rc = DL_ERR_INTERNAL; - - DL_ABI_GUARD_BEGIN - { - (void) file; - - rc = DL_ERR_NOT_SUPPORTED; - } - DL_ABI_GUARD_END(rc, "s3_file_close"); - - return rc; -} - -static void -s3_file_abort(DatalakeFile file) -{ - DL_CLEANUP_GUARD_BEGIN - { - (void) file; - } - DL_CLEANUP_GUARD_END; -} - -static const struct DatalakeStorageOps s3_storage_ops = { - s3_fs_open, - s3_fs_close, - s3_fs_list, - s3_file_open, - s3_file_read, - s3_file_write, - s3_file_close, - s3_file_abort -}; - -DlErrCode -datalake_register_s3_backend(void) -{ - return datalake_register_storage_backend("s3", &s3_storage_ops); -} diff --git a/contrib/datalake_fdw/src/common/storage_arrow.h b/contrib/datalake_fdw/src/common/storage_arrow.h new file mode 100644 index 00000000000..584fad33eb6 --- /dev/null +++ b/contrib/datalake_fdw/src/common/storage_arrow.h @@ -0,0 +1,79 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * storage_arrow.h + * C++ access to the Arrow objects behind the storage facade. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/storage_arrow.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STORAGE_ARROW_H +#define STORAGE_ARROW_H + +#include +#include + +#include +#include +#include + +#include "common/file_system_wrapper.h" + +arrow::fs::FileSystem *dl_storage_arrow_fs(DatalakeFileSystem fs); +std::string dl_storage_native_path(DatalakeFileSystem fs, + const char *relative); + +class DlStatusDetail : public arrow::StatusDetail +{ +public: + DlStatusDetail(DlErrCode code, std::string type); + const char *type_id() const override; + std::string ToString() const override; + DlErrCode code() const; + const std::string &type() const; + +private: + DlErrCode code_; + std::string type_; +}; + +/* The credential values to hide, taken from the mount options. */ +std::vector dl_storage_collect_secrets(const DlKeyValue *kv, + int nkv); + +/* + * Turn an Arrow status into a DlErrCode and record it, hiding every value in + * `secrets` from both the message and the error class. Pass the secrets the + * handle carries; NULL only where no credentials exist yet. + */ +DlErrCode dl_storage_status_to_err(const arrow::Status &status, + const char *operation, + const std::vector *secrets); + +/* + * Whether a path may be joined onto a mount root: relative, and free of "." + * and ".." components. The root names where a volume lives, it does not + * confine what a path can reach, so the check belongs before the join. + */ +bool dl_storage_path_is_safe(const char *relative); + +#endif /* STORAGE_ARROW_H */ diff --git a/contrib/datalake_fdw/src/common/storage_backend.h b/contrib/datalake_fdw/src/common/storage_backend.h new file mode 100644 index 00000000000..a7717a19acf --- /dev/null +++ b/contrib/datalake_fdw/src/common/storage_backend.h @@ -0,0 +1,144 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * storage_backend.h + * Public contract for pluggable storage backends. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/storage_backend.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STORAGE_BACKEND_H +#define STORAGE_BACKEND_H + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include "datalake_location.h" +#include "dl_err.h" +#include "dl_kv.h" + +#define DL_STORAGE_ABI_VERSION 1 + +#define DL_STORAGE_STRINGIFY_DETAIL(value) #value +#define DL_STORAGE_STRINGIFY(value) DL_STORAGE_STRINGIFY_DETAIL(value) + +#if defined(__clang__) +#define DL_STORAGE_COMPILER_FINGERPRINT \ + "clang" DL_STORAGE_STRINGIFY(__clang_major__) +#elif defined(__GNUC__) +#define DL_STORAGE_COMPILER_FINGERPRINT \ + "gcc" DL_STORAGE_STRINGIFY(__GNUC__) +#else +#define DL_STORAGE_COMPILER_FINGERPRINT "unknown" +#endif + +#ifdef _GLIBCXX_USE_CXX11_ABI +#define DL_STORAGE_CXX11_ABI_FINGERPRINT \ + DL_STORAGE_STRINGIFY(_GLIBCXX_USE_CXX11_ABI) +#else +#define DL_STORAGE_CXX11_ABI_FINGERPRINT "na" +#endif + +#define DL_STORAGE_ABI_FINGERPRINT \ + DL_STORAGE_COMPILER_FINGERPRINT ";cxx11abi=" \ + DL_STORAGE_CXX11_ABI_FINGERPRINT ";arrow=" ARROW_VERSION_STRING + +struct DatalakeMountedFs +{ + std::shared_ptr fs; + std::string root; +}; + +/* + * Services supplied by the host. Future versions may append fields, so a + * backend reads a field only after struct_size says it is there -- which is + * what dl_storage_host_pool() below does for the one field there is today. + */ +struct DatalakeStorageHost +{ + uint32_t struct_size; + arrow::MemoryPool *pool; /* memory a backend allocates through is + * charged to the query, so IOContext and + * every Buffer must come from here */ +}; + +/* + * The pool to allocate through, or Arrow's default when the host predates the + * field. Allocating outside the host's pool means the memory escapes + * Cloudberry's accounting, so a backend should always route Arrow through it. + */ +static inline arrow::MemoryPool * +dl_storage_host_pool(const DatalakeStorageHost *host) +{ + size_t needed = offsetof(DatalakeStorageHost, pool) + + sizeof(((DatalakeStorageHost *) nullptr)->pool); + + if (host == nullptr || host->struct_size < needed || host->pool == nullptr) + return arrow::default_memory_pool(); + return host->pool; +} + +/* + * A backend only constructs an Arrow filesystem. The facade uses precisely + * GetFileInfo(path), GetFileInfo(FileSelector), OpenInputFile and its + * GetSize/ReadAt/Read methods, OpenOutputStream and its Write/Close/Abort + * methods, and DeleteFile. Other methods may return NotImplemented. + * + * Instances are borrowed by the registry and therefore need static lifetime. + * struct_size is prefix-compatible: future versions may append fields. + */ +struct DatalakeStorageBackend +{ + uint32_t abi_version; + uint32_t struct_size; + const char *uri_scheme; + const char *arrow_version; + const char *abi_fingerprint; + arrow::Result (*mount) (const DatalakeLocation *, + const DlKeyValue *kv, int nkv, + const DatalakeStorageHost *host); + arrow::Status (*initialize) (void); + void (*finalize) (void); +}; + +#ifdef __cplusplus +extern "C" +{ +#endif + +extern DlErrCode datalake_register_storage_backend( + const DatalakeStorageBackend *backend); + +#ifdef __cplusplus +} +#endif + +#endif /* STORAGE_BACKEND_H */ diff --git a/contrib/datalake_fdw/src/common/storage_backend_register.h b/contrib/datalake_fdw/src/common/storage_backend_register.h new file mode 100644 index 00000000000..cbb5c914844 --- /dev/null +++ b/contrib/datalake_fdw/src/common/storage_backend_register.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * storage_backend_register.h + * Order-independent registration helper for storage plugins. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/storage_backend_register.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STORAGE_BACKEND_REGISTER_H +#define STORAGE_BACKEND_REGISTER_H + +/* Arrow headers must precede PostgreSQL headers in a C++ translation unit. */ +#include "storage_backend.h" + +extern "C" +{ +#include "postgres.h" +#include "fmgr.h" +} + +static inline DlErrCode +datalake_storage_register(const DatalakeStorageBackend *backend) +{ + typedef DlErrCode (*dl_register_fn) (const DatalakeStorageBackend *); + dl_register_fn fn = reinterpret_cast( + load_external_function("$libdir/datalake_fdw", + "datalake_register_storage_backend", true, NULL)); + + return fn(backend); +} + +#endif /* STORAGE_BACKEND_REGISTER_H */ diff --git a/contrib/datalake_fdw/src/test/datalake_fdw_test.c b/contrib/datalake_fdw/src/test/datalake_fdw_test.c index e6c96f0a57a..4a17a86293b 100644 --- a/contrib/datalake_fdw/src/test/datalake_fdw_test.c +++ b/contrib/datalake_fdw/src/test/datalake_fdw_test.c @@ -37,6 +37,8 @@ #include "postgres.h" +#include + #include "catalog/pg_type.h" #include "executor/spi.h" #include "funcapi.h" @@ -46,13 +48,127 @@ #include "utils/tuplestore.h" #include "am_iceberg/pg_iceberg_guc.h" +#include "am_iceberg/pg_iceberg_options.h" #include "common/dl_err.h" +#include "common/file_system_wrapper.h" #include "format/arrow_builder.h" #include "format/arrow_decode.h" #include "format/format.h" PG_FUNCTION_INFO_V1(datalake_parquet_write); PG_FUNCTION_INFO_V1(datalake_parquet_read); +PG_FUNCTION_INFO_V1(datalake_storage_write_text); +PG_FUNCTION_INFO_V1(datalake_storage_read_text); +PG_FUNCTION_INFO_V1(datalake_storage_list); +PG_FUNCTION_INFO_V1(datalake_storage_probe); +PG_FUNCTION_INFO_V1(datalake_storage_register_bad); + +extern DlErrCode datalake_test_register_bad_storage_backend(const char *kind); + +static DlKeyValue * +storage_kv(FunctionCallInfo fcinfo, int argno, int *nkv) +{ + Datum *values; + bool *nulls; + DlKeyValue *kv; + int i; + + *nkv = 0; + if (PG_ARGISNULL(argno)) + return NULL; + + deconstruct_array(PG_GETARG_ARRAYTYPE_P(argno), TEXTOID, -1, false, + TYPALIGN_INT, &values, &nulls, nkv); + kv = palloc0(*nkv * sizeof(*kv)); + for (i = 0; i < *nkv; i++) + { + char *equal; + + if (nulls[i]) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("a storage option cannot be null"))); + kv[i].key = TextDatumGetCString(values[i]); + equal = strchr(kv[i].key, '='); + if (equal == NULL || equal == kv[i].key) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("storage option must have the form key=value"))); + *equal = '\0'; + kv[i].value = equal + 1; + } + return kv; +} + +static void +storage_parse_uri(const char *uri, bool leaf, DatalakeLocation *location, + char **relative) +{ + char *detail = NULL; + DlErrCode rc; + + if (strncmp(uri, "dltest://", 9) == 0) + { + const char *path = uri + 9; + + if (path[0] != '/' || strchr(path, '?') != NULL || strchr(path, '#') != NULL) + { + dl_error_set(DL_ERR_INVALID_OPTION, "parse storage location", NULL, + "dltest location must have an absolute path and no query or fragment"); + dl_error_report(ERROR, DL_ERR_INVALID_OPTION, "parse storage location"); + } + memset(location, 0, sizeof(*location)); + location->abi_version = DATALAKE_LOCATION_ABI_VERSION; + location->scheme = pstrdup("dltest"); + location->authority = pstrdup(""); + location->path_prefix = pstrdup(path); + } + else + { + rc = pg_iceberg_parse_location(uri, NULL, NULL, location, &detail); + if (rc != DL_OK) + { + dl_error_set(rc, "parse storage location", NULL, detail); + dl_error_report(ERROR, rc, "parse storage location"); + } + } + + *relative = pstrdup(""); + if (leaf) + { + char *slash = strrchr(location->path_prefix, '/'); + + if (slash == NULL || slash[1] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("storage URI must name a file"))); + *relative = pstrdup(slash + 1); + if (slash == location->path_prefix) + location->path_prefix = pstrdup( + strcmp(location->scheme, "s3") == 0 ? "" : "/"); + else + *slash = '\0'; + } +} + +static DatalakeFileSystem +storage_open_uri(FunctionCallInfo fcinfo, int uri_arg, int kv_arg, bool leaf, + char **relative) +{ + DatalakeLocation location; + DatalakeFileSystem fs = NULL; + DlKeyValue *kv; + char *uri = text_to_cstring(PG_GETARG_TEXT_PP(uri_arg)); + int nkv; + DlErrCode rc; + + storage_parse_uri(uri, leaf, &location, relative); + kv = storage_kv(fcinfo, kv_arg, &nkv); + rc = datalake_fs_open(&location, kv, nkv, &fs); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "open storage"); + return fs; +} /* * The SQL declaration and the C function have to agree on the argument list, @@ -488,3 +604,223 @@ datalake_parquet_read(PG_FUNCTION_ARGS) return (Datum) 0; } + +/* + * The handles an error has to release live in volatile locals, which is what + * lets PG_CATCH still read them after a longjmp; same shape as the Parquet + * writer above. Each is cleared as soon as something else owns it. + */ +Datum +datalake_storage_write_text(PG_FUNCTION_ARGS) +{ + text *content; + char *relative; + DatalakeFileSystem volatile open_fs = NULL; + DatalakeFile volatile open_file = NULL; + int64 length; + + check_nargs(fcinfo, 3); + if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) + PG_RETURN_NULL(); + content = PG_GETARG_TEXT_PP(1); + length = VARSIZE_ANY_EXHDR(content); + open_fs = storage_open_uri(fcinfo, 0, 2, true, &relative); + + PG_TRY(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + DatalakeFile file = NULL; + DlErrCode rc; + + rc = datalake_file_open(fs, relative, DATALAKE_FILE_WRITE, &file); + open_file = file; + if (rc == DL_OK) + rc = datalake_file_write(file, VARDATA_ANY(content), length); + if (rc == DL_OK) + { + rc = datalake_file_close(&file); + open_file = file; /* close consumes the handle */ + } + if (rc != DL_OK) + dl_error_report(ERROR, rc, "write storage file"); + } + PG_CATCH(); + { + DatalakeFile file = (DatalakeFile) open_file; + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_file_abort(&file); + datalake_fs_close(&fs); + PG_RE_THROW(); + } + PG_END_TRY(); + + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + } + PG_RETURN_INT64(length); +} + +Datum +datalake_storage_read_text(PG_FUNCTION_ARGS) +{ + char *relative; + DatalakeFileSystem volatile open_fs = NULL; + DatalakeFile volatile open_file = NULL; + StringInfoData data; + + check_nargs(fcinfo, 2); + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + open_fs = storage_open_uri(fcinfo, 0, 1, true, &relative); + initStringInfo(&data); + PG_TRY(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + DatalakeFile file = NULL; + char buffer[8192]; + DlErrCode rc; + + rc = datalake_file_open(fs, relative, DATALAKE_FILE_READ, &file); + open_file = file; + while (rc == DL_OK) + { + int64 nread; + + rc = datalake_file_read(file, buffer, sizeof(buffer), &nread); + if (rc != DL_OK || nread == 0) + break; + appendBinaryStringInfo(&data, buffer, nread); + } + if (rc == DL_OK) + { + rc = datalake_file_close(&file); + open_file = file; /* close consumes the handle */ + } + if (rc != DL_OK) + dl_error_report(ERROR, rc, "read storage file"); + } + PG_CATCH(); + { + DatalakeFile file = (DatalakeFile) open_file; + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_file_abort(&file); + datalake_fs_close(&fs); + PG_RE_THROW(); + } + PG_END_TRY(); + + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + } + PG_RETURN_TEXT_P(cstring_to_text_with_len(data.data, data.len)); +} + +Datum +datalake_storage_list(PG_FUNCTION_ARGS) +{ + char *relative; + DatalakeFileSystem volatile open_fs = NULL; + char **volatile open_names = NULL; + int volatile open_nnames = 0; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + check_nargs(fcinfo, 2); + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + open_fs = storage_open_uri(fcinfo, 0, 1, false, &relative); + PG_TRY(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + char **names = NULL; + int nnames = 0; + int i; + DlErrCode rc; + + rc = datalake_fs_list(fs, relative, &names, &nnames); + open_names = names; + open_nnames = nnames; + if (rc != DL_OK) + dl_error_report(ERROR, rc, "list storage"); + InitMaterializedSRF(fcinfo, MAT_SRF_USE_EXPECTED_DESC); + for (i = 0; i < nnames; i++) + { + Datum value = CStringGetTextDatum(names[i]); + bool isnull = false; + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + &value, &isnull); + free(names[i]); + names[i] = NULL; /* so the cleanup path cannot free it twice */ + } + free(names); + open_names = NULL; + open_nnames = 0; + } + PG_CATCH(); + { + char **names = (char **) open_names; + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + int i; + + for (i = 0; names != NULL && i < open_nnames; i++) + free(names[i]); + free(names); + datalake_fs_close(&fs); + PG_RE_THROW(); + } + PG_END_TRY(); + + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + } + return (Datum) 0; +} + +Datum +datalake_storage_probe(PG_FUNCTION_ARGS) +{ + DatalakeLocation location = {0}; + DatalakeFileSystem fs = NULL; + char *scheme; + DlErrCode rc; + + check_nargs(fcinfo, 1); + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + scheme = text_to_cstring(PG_GETARG_TEXT_PP(0)); + location.abi_version = DATALAKE_LOCATION_ABI_VERSION; + location.scheme = scheme; + location.authority = pstrdup(strcmp(scheme, "s3") == 0 ? "probe-bucket" : ""); + location.path_prefix = pstrdup("/tmp"); + rc = datalake_fs_open(&location, NULL, 0, &fs); + if (rc == DL_OK) + { + datalake_fs_close(&fs); + PG_RETURN_TEXT_P(cstring_to_text("supported")); + } + PG_RETURN_TEXT_P(cstring_to_text(dl_error_get()->message)); +} + +Datum +datalake_storage_register_bad(PG_FUNCTION_ARGS) +{ + char *kind; + DlErrCode rc; + + check_nargs(fcinfo, 1); + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + kind = text_to_cstring(PG_GETARG_TEXT_PP(0)); + rc = datalake_test_register_bad_storage_backend(kind); + if (rc == DL_OK) + PG_RETURN_TEXT_P(cstring_to_text("accepted")); + PG_RETURN_TEXT_P(cstring_to_text(dl_error_get()->message)); +} diff --git a/contrib/datalake_fdw/src/test/storage_test_backend.cpp b/contrib/datalake_fdw/src/test/storage_test_backend.cpp new file mode 100644 index 00000000000..ad729dab082 --- /dev/null +++ b/contrib/datalake_fdw/src/test/storage_test_backend.cpp @@ -0,0 +1,72 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file for details. */ + +#include + +#include +#include +#include + +#include "common/local_file_system.h" +#include "common/storage_backend.h" + +/* + * A second backend registered through the public contract, so the storage + * cases prove that a backend which is not the built-in one works the same + * way. It mounts the same create-only local file system under a subtree, + * which keeps one implementation of the write and abort semantics rather + * than a second one that could drift. + */ +static arrow::Result +mount_dltest(const DatalakeLocation *location, const DlKeyValue *, int, + const DatalakeStorageHost *host) +{ + if (location == NULL || location->path_prefix == NULL) + return arrow::Status::Invalid("dltest location has no path"); + + auto local = std::make_shared( + arrow::io::IOContext(dl_storage_host_pool(host))); + DatalakeMountedFs mounted; + + mounted.fs = std::make_shared( + location->path_prefix, local); + mounted.root = ""; + return mounted; +} + +static const DatalakeStorageBackend dltest_backend = { + DL_STORAGE_ABI_VERSION, sizeof(DatalakeStorageBackend), "dltest", + ARROW_VERSION_STRING, DL_STORAGE_ABI_FINGERPRINT, mount_dltest, NULL, NULL +}; + +extern "C" DlErrCode +datalake_register_test_storage_backend(void) +{ + return datalake_register_storage_backend(&dltest_backend); +} + +/* + * Each kind breaks exactly one of the registration checks, so a case can + * assert the message that check produces. The scheme stays "dltest", which + * is already registered: a check that stopped working would fall through to + * the duplicate-scheme rejection, and the cases tell those apart by naming + * the field and the value they expect to see reported. + */ +extern "C" DlErrCode +datalake_test_register_bad_storage_backend(const char *kind) +{ + DatalakeStorageBackend bad = dltest_backend; + + if (strcmp(kind, "abi_version") == 0) + bad.abi_version++; + else if (strcmp(kind, "struct_size") == 0) + bad.struct_size = 0; + else if (strcmp(kind, "arrow_version") == 0) + bad.arrow_version = "0.0.0-test"; + else if (strcmp(kind, "abi_fingerprint") == 0) + bad.abi_fingerprint = "gcc0;cxx11abi=9;arrow=0.0.0-test"; + else if (strcmp(kind, "duplicate") != 0) + return DL_ARG_ERROR("register bad storage backend"); + + return datalake_register_storage_backend(&bad); +} diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out index d6974b509f7..f12ee9f4c48 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out @@ -417,7 +417,7 @@ CREATE SERVER dlskel_bad_scheme FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 'ftp://x/y'); ERROR: invalid iceberg volume base_path "ftp://x/y" -DETAIL: location URI "ftp://x/y" has unsupported scheme; expected s3 or hdfs +DETAIL: location URI "ftp://x/y" has unsupported scheme; expected s3 or file CREATE SERVER dlskel_bad_authority FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://'); @@ -427,12 +427,12 @@ CREATE SERVER dlskel_bad_query FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://b/p?versionId=3'); ERROR: invalid iceberg volume base_path "s3://b/p?versionId=3" -DETAIL: location URI "s3://b/p?versionId=3" must not contain a query +DETAIL: location URI must not contain a query CREATE SERVER dlskel_bad_userinfo FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://user@b/p'); ERROR: invalid iceberg volume base_path "s3://user@b/p" -DETAIL: location URI "s3://user@b/p" must not contain userinfo +DETAIL: location URI must not contain userinfo CREATE SERVER dlskel_bad_bucket FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://UPPER_case/p'); diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out new file mode 100644 index 00000000000..690dc26712f --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out @@ -0,0 +1,128 @@ +-- Storage facade behavior shared by the built-in file backend and a backend +-- registered through the public plugin contract. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_storage_local && mkdir -p /tmp/datalake_fdw_storage_local/file /tmp/datalake_fdw_storage_local/dltest'; +SELECT datalake_storage_write_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt', 'file-data'); + datalake_storage_write_text +----------------------------- + 9 +(1 row) + +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt'); + datalake_storage_read_text +---------------------------- + file-data +(1 row) + +SELECT * FROM datalake_storage_list( + 'file:///tmp/datalake_fdw_storage_local/file'); + datalake_storage_list +-------------------------------------------- + /tmp/datalake_fdw_storage_local/file/a.txt +(1 row) + +\set VERBOSITY sqlstate +SELECT datalake_storage_write_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt', 'replacement'); +ERROR: 42P07 +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/missing.txt'); +ERROR: 42704 +SELECT datalake_storage_read_text('file://host/tmp/a.txt'); +ERROR: 22023 +SELECT datalake_storage_read_text('/tmp/a.txt'); +ERROR: 22023 +-- A path may not climb out of the volume it was resolved against. +SELECT datalake_storage_read_text('file:///tmp/datalake_fdw_storage_local/..'); +ERROR: 22023 +\set VERBOSITY default +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt'); + datalake_storage_read_text +---------------------------- + file-data +(1 row) + +SELECT datalake_storage_write_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt', 'dltest-data'); + datalake_storage_write_text +----------------------------- + 11 +(1 row) + +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt'); + datalake_storage_read_text +---------------------------- + dltest-data +(1 row) + +SELECT * FROM datalake_storage_list( + 'dltest:///tmp/datalake_fdw_storage_local/dltest'); + datalake_storage_list +----------------------- + a.txt +(1 row) + +\set VERBOSITY sqlstate +SELECT datalake_storage_write_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt', 'replacement'); +ERROR: 42P07 +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/missing.txt'); +ERROR: 42704 +\set VERBOSITY default +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt'); + datalake_storage_read_text +---------------------------- + dltest-data +(1 row) + +-- A write that is refused leaves the file it refused to replace untouched, +-- and leaves nothing else behind either. +SELECT * FROM datalake_storage_list( + 'file:///tmp/datalake_fdw_storage_local/file'); + datalake_storage_list +-------------------------------------------- + /tmp/datalake_fdw_storage_local/file/a.txt +(1 row) + +SELECT datalake_storage_probe('s3'); + datalake_storage_probe +----------------------------------------------- + no storage backend registered for scheme "s3" +(1 row) + +-- Each malformed registration must be rejected by the check it breaks, not by +-- some later one: every pattern names the field and the value it reports, so +-- a check that stopped working could not fall through to another and still +-- match. +SELECT kind, + datalake_storage_register_bad(kind) LIKE pattern AS rejected_by_its_check +FROM (VALUES + ('abi_version', '%ABI version mismatch: expected 1, got 2%'), + ('struct_size', '%struct size mismatch: expected at least %, got 0%'), + ('arrow_version', '%Arrow version mismatch: expected "%", got "0.0.0-test"%'), + ('abi_fingerprint', + '%ABI fingerprint mismatch: expected "%", got "gcc0;cxx11abi=9;arrow=0.0.0-test"%'), + ('duplicate', '%scheme expected to be unique, got duplicate "dltest"%') +) AS t(kind, pattern) +ORDER BY kind; + kind | rejected_by_its_check +-----------------+----------------------- + abi_fingerprint | t + abi_version | t + arrow_version | t + duplicate | t + struct_size | t +(5 rows) + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_storage_local'; +-- End of storage_local. diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/sql/storage_local.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/sql/storage_local.sql new file mode 100644 index 00000000000..6ab78953e4b --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/sql/storage_local.sql @@ -0,0 +1,71 @@ +-- Storage facade behavior shared by the built-in file backend and a backend +-- registered through the public plugin contract. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_storage_local && mkdir -p /tmp/datalake_fdw_storage_local/file /tmp/datalake_fdw_storage_local/dltest'; + +SELECT datalake_storage_write_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt', 'file-data'); +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt'); +SELECT * FROM datalake_storage_list( + 'file:///tmp/datalake_fdw_storage_local/file'); + +\set VERBOSITY sqlstate +SELECT datalake_storage_write_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt', 'replacement'); +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/missing.txt'); +SELECT datalake_storage_read_text('file://host/tmp/a.txt'); +SELECT datalake_storage_read_text('/tmp/a.txt'); +-- A path may not climb out of the volume it was resolved against. +SELECT datalake_storage_read_text('file:///tmp/datalake_fdw_storage_local/..'); +\set VERBOSITY default +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt'); + +SELECT datalake_storage_write_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt', 'dltest-data'); +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt'); +SELECT * FROM datalake_storage_list( + 'dltest:///tmp/datalake_fdw_storage_local/dltest'); +\set VERBOSITY sqlstate +SELECT datalake_storage_write_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt', 'replacement'); +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/missing.txt'); +\set VERBOSITY default +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt'); + +-- A write that is refused leaves the file it refused to replace untouched, +-- and leaves nothing else behind either. +SELECT * FROM datalake_storage_list( + 'file:///tmp/datalake_fdw_storage_local/file'); + +SELECT datalake_storage_probe('s3'); + +-- Each malformed registration must be rejected by the check it breaks, not by +-- some later one: every pattern names the field and the value it reports, so +-- a check that stopped working could not fall through to another and still +-- match. +SELECT kind, + datalake_storage_register_bad(kind) LIKE pattern AS rejected_by_its_check +FROM (VALUES + ('abi_version', '%ABI version mismatch: expected 1, got 2%'), + ('struct_size', '%struct size mismatch: expected at least %, got 0%'), + ('arrow_version', '%Arrow version mismatch: expected "%", got "0.0.0-test"%'), + ('abi_fingerprint', + '%ABI fingerprint mismatch: expected "%", got "gcc0;cxx11abi=9;arrow=0.0.0-test"%'), + ('duplicate', '%scheme expected to be unique, got duplicate "dltest"%') +) AS t(kind, pattern) +ORDER BY kind; + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_storage_local'; + +-- End of storage_local. From 4a3ff1053aeb0f82c4a6a6d64931622449f494fb Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:25 +0800 Subject: [PATCH 4/9] datalake_fdw: read and write s3 through the AWS SDK The s3 backend was a stub. This makes it an arrow::fs::FileSystem of our own over the AWS SDK for C++, rather than Arrow's S3FileSystem: no RPM of Arrow is built with S3 support -- EPEL's and the Arrow project's own both set use_s3 0 -- so depending on it would mean asking every user to build Arrow. A synchronous S3Client, so a cancelled query cannot leave a callback holding a backend's memory. Reads go through PreallocatedStreamBuf straight into a tracked buffer rather than the SDK's default stringstream. Writes buffer 8 MiB and begin a multipart upload only when they exceed it, so a small file is one PutObject. Cleanup depends on whether an upload is still live rather than on whether the stream was closed, so a Close() that fails at the last part still aborts instead of leaving parts to be billed for. Connect timeout 5 s, request timeout 300 s, three retries, set here rather than inherited, so a black-holed endpoint answers in about 24 seconds on every Arrow version. The SDK is found by prefix -- the one AWS_SDK_PREFIX names, or the usual places a hand-built one lands. Without it the module still builds and says so, and opening an s3:// location reports that it was left out; a prefix with no SDK in it is an error rather than a silent fallback. It links statically and adds libcurl, OpenSSL and zlib to NEEDED and nothing else. --- contrib/datalake_fdw/Makefile | 59 +- .../datalake_fdw/datalake_fdw_test--1.0.sql | 3 + contrib/datalake_fdw/exports.txt | 2 + .../src/common/backend_registry.cpp | 10 + .../src/common/file_system_wrapper.cpp | 31 +- .../src/common/file_system_wrapper.h | 7 + .../src/common/s3_file_system.cpp | 993 ++++++++++++++++++ .../datalake_fdw/src/test/datalake_fdw_test.c | 37 + .../storage_local/expected/storage_local.out | 6 +- .../expected/storage_local_1.out | 128 +++ .../smoke/storage_s3/expected/storage_s3.out | 162 +++ .../smoke/storage_s3/sql/storage_s3.sql | 147 +++ 12 files changed, 1577 insertions(+), 8 deletions(-) create mode 100644 contrib/datalake_fdw/src/common/s3_file_system.cpp create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local_1.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile index 1540cb14b23..025396a8811 100644 --- a/contrib/datalake_fdw/Makefile +++ b/contrib/datalake_fdw/Makefile @@ -52,6 +52,7 @@ OBJS = \ src/common/parser_option.o \ src/common/file_system_wrapper.o \ src/common/local_file_system.o \ + src/common/s3_file_system.o \ src/common/backend_registry.o \ src/test/storage_test_backend.o \ src/test/datalake_fdw_test.o @@ -89,13 +90,52 @@ Debian and Ubuntu) endif endif +# The AWS SDK for C++, which the s3 backend is built on. No distribution +# packages it, so it is found by prefix rather than by pkg-config: either the +# one named on the command line, or the usual places a hand-built one lands. +# Without it the module still builds, and opening an s3:// location is what +# reports that it was left out. +AWS_SDK_SEARCH = /usr/local /opt/datalake +ifeq ($(origin AWS_SDK_PREFIX),undefined) +AWS_SDK_PREFIX := $(firstword $(foreach d,$(AWS_SDK_SEARCH),\ + $(if $(wildcard $(d)/include/aws/s3/S3Client.h),$(d)))) +else +# Named explicitly and wrong is a mistake to report, not to work around. +ifeq ($(wildcard $(AWS_SDK_PREFIX)/include/aws/s3/S3Client.h),) +ifeq ($(filter clean distclean maintainer-clean,$(MAKECMDGOALS)),) +$(error AWS_SDK_PREFIX=$(AWS_SDK_PREFIX) has no include/aws/s3/S3Client.h) +endif +endif +endif + +ifneq ($(AWS_SDK_PREFIX),) +AWS_SDK_CPPFLAGS = -DDL_HAVE_AWS_SDK -I$(AWS_SDK_PREFIX)/include +# One group, because these static archives refer to each other both ways. +AWS_SDK_LIBS = -L$(AWS_SDK_PREFIX)/lib64 -L$(AWS_SDK_PREFIX)/lib \ + -Wl,--start-group \ + -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ + -laws-crt-cpp -laws-c-s3 -laws-c-auth -laws-c-http -laws-c-io \ + -laws-c-cal -laws-c-compression -laws-c-mqtt -laws-c-event-stream \ + -laws-c-sdkutils -laws-c-common -laws-checksums -ls2n \ + -Wl,--end-group \ + -lssl -lcrypto -lz -lcurl +else +AWS_SDK_CPPFLAGS = +AWS_SDK_LIBS = +ifeq ($(filter clean distclean maintainer-clean,$(MAKECMDGOALS)),) +$(info NOTICE: the AWS SDK for C++ was not found in $(AWS_SDK_SEARCH), so \ +datalake_fdw is being built without the s3 storage backend. Point \ +AWS_SDK_PREFIX at an installed SDK to build with it.) +endif +endif + # Use the documented PGXS knobs: pgxs.mk appends these AFTER the flags configure # chose, so optimization/warning settings survive. A pre-include # "override CFLAGS +=" would give CFLAGS override origin and silently discard # Makefile.global's own "CFLAGS = @CFLAGS@" assignment. PG_CFLAGS = -fvisibility=hidden PG_CXXFLAGS = -fvisibility=hidden -fvisibility-inlines-hidden -std=c++17 -PG_CPPFLAGS = -I$(srcdir)/src $(ARROW_CPPFLAGS) +PG_CPPFLAGS = -I$(srcdir)/src $(ARROW_CPPFLAGS) $(AWS_SDK_CPPFLAGS) # The regression cases live with the rest of the test material rather than in a # second place of their own; pg_regress is pointed at them. REGRESS_OPTS is @@ -116,6 +156,9 @@ FORMAT_PARQUET_REGRESS = parquet_roundtrip FORMAT_PARQUET_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/format_parquet STORAGE_LOCAL_REGRESS = storage_local STORAGE_LOCAL_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_local +# The s3 category needs a service to talk to, so it runs where one is named. +STORAGE_S3_REGRESS = storage_s3 +STORAGE_S3_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_s3 EXTRA_CLEAN = exports_darwin.list exports.map @@ -139,7 +182,7 @@ endif # Shared libraries are linked with $(CC) (see src/Makefile.shlib COMPILER), so a # module containing C++ translation units must pull in the C++ runtime itself. -SHLIB_LINK += -lstdc++ $(ARROW_LIBS) +SHLIB_LINK += -lstdc++ $(ARROW_LIBS) $(AWS_SDK_LIBS) # The export list is the single place that decides what stays visible -- which # now also means none of Arrow's symbols become symbols this module offers. @@ -172,7 +215,7 @@ $(shlib): $(EXPORT_LIST) # two runs creating the same extension in the same database race. The # categories are chained through their prerequisites rather than listed side # by side, which is what keeps the order under a parallel make. -installcheck: installcheck-storage-local +installcheck: installcheck-storage-s3 installcheck-format-parquet: submake $(REGRESS_PREP) $(pg_regress_installcheck) $(REGRESS_OPTS) \ @@ -186,6 +229,16 @@ installcheck-storage-local: submake $(REGRESS_PREP) installcheck-format-parquet .PHONY: installcheck-storage-local +installcheck-storage-s3: submake $(REGRESS_PREP) installcheck-storage-local +ifeq ($(DATALAKE_TEST_S3_ENDPOINT),) + @echo "NOTICE: DATALAKE_TEST_S3_ENDPOINT is not set, skipping the s3 storage tests" +else + $(pg_regress_installcheck) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_S3_INPUTDIR) $(STORAGE_S3_REGRESS) +endif + +.PHONY: installcheck-storage-s3 + # "make check" is in-tree only -- under PGXS pgxs.mk refuses the target -- and # it is the only run that supplies the temp-config that preloads this module. ifndef USE_PGXS diff --git a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql index ee512382d48..29d0af9a5f9 100644 --- a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql +++ b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql @@ -63,6 +63,8 @@ RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; CREATE FUNCTION datalake_storage_list(uri text, kv text[] DEFAULT NULL) RETURNS SETOF text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE EXECUTE ON COORDINATOR; +CREATE FUNCTION datalake_storage_delete(uri text, kv text[] DEFAULT NULL) +RETURNS boolean AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; CREATE FUNCTION datalake_storage_probe(scheme text) RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; CREATE FUNCTION datalake_storage_register_bad(kind text) @@ -71,5 +73,6 @@ RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; REVOKE EXECUTE ON FUNCTION datalake_storage_write_text(text, text, text[]) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION datalake_storage_read_text(text, text[]) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION datalake_storage_list(text, text[]) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_storage_delete(text, text[]) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION datalake_storage_probe(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION datalake_storage_register_bad(text) FROM PUBLIC; diff --git a/contrib/datalake_fdw/exports.txt b/contrib/datalake_fdw/exports.txt index 9f6932ce60f..08ac1ed74cb 100644 --- a/contrib/datalake_fdw/exports.txt +++ b/contrib/datalake_fdw/exports.txt @@ -44,6 +44,8 @@ pg_finfo_datalake_storage_read_text datalake_storage_read_text pg_finfo_datalake_storage_list datalake_storage_list +pg_finfo_datalake_storage_delete +datalake_storage_delete pg_finfo_datalake_storage_probe datalake_storage_probe pg_finfo_datalake_storage_register_bad diff --git a/contrib/datalake_fdw/src/common/backend_registry.cpp b/contrib/datalake_fdw/src/common/backend_registry.cpp index 9d74c0bb1fc..9b654f9cbc4 100644 --- a/contrib/datalake_fdw/src/common/backend_registry.cpp +++ b/contrib/datalake_fdw/src/common/backend_registry.cpp @@ -67,6 +67,7 @@ typedef struct DatalakeStorageRegistry } DatalakeStorageRegistry; extern DlErrCode datalake_register_local_backend(void); +extern DlErrCode datalake_register_s3_backend(void); static DatalakeStorageRegistry * storage_registry(bool create) @@ -308,4 +309,13 @@ datalake_register_storage_backends(void) if (rc != DL_OK && rc != DL_ERR_ALREADY_EXISTS) dl_error_report(ERROR, rc, "register file storage backend"); + + DL_TRY + { + rc = datalake_register_s3_backend(); + } + DL_CATCH_END(); + + if (rc != DL_OK && rc != DL_ERR_ALREADY_EXISTS) + dl_error_report(ERROR, rc, "register s3 storage backend"); } diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp index d1c3bd51728..e119f2f8c79 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -100,7 +100,13 @@ DlStatusDetail::type() const return type_; } -/* The option values worth hiding: the key names credentials arrive under. */ +/* + * The option values worth hiding. A secret access key and a session token + * are credentials; an access key id is an identifier that appears in request + * headers and audit records anyway, and hiding it only costs the reader the + * one fact that says which credential was used -- worse, its value tends to + * occur inside bucket and prefix names, which would then be masked too. + */ std::vector dl_storage_collect_secrets(const DlKeyValue *kv, int nkv) { @@ -116,7 +122,7 @@ dl_storage_collect_secrets(const DlKeyValue *kv, int nkv) if (key.find("secret") == std::string::npos && key.find("token") == std::string::npos && - key.find("access_key") == std::string::npos) + key.find("password") == std::string::npos) continue; secrets.push_back(kv[i].value); @@ -457,6 +463,27 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, return rc; } +extern "C" DlErrCode +datalake_file_delete(DatalakeFileSystem fs, const char *path) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (fs == NULL || path == NULL) + rc = DL_ARG_ERROR("file_delete"); + else if (!dl_storage_path_is_safe(path)) + rc = reject_unsafe_path("delete storage file", path); + else + rc = dl_storage_status_to_err( + fs->fs->DeleteFile(dl_storage_native_path(fs, path)), + "delete storage file", fs->secrets.get()); + } + DL_ABI_GUARD_END(rc, "file_delete"); + + return rc; +} + extern "C" DlErrCode datalake_file_open(DatalakeFileSystem fs, const char *path, DatalakeFileMode mode, DatalakeFile *file_out) diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.h b/contrib/datalake_fdw/src/common/file_system_wrapper.h index 75eeddf1ead..98ed4bb0aaa 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.h +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.h @@ -82,6 +82,13 @@ extern void datalake_fs_close(DatalakeFileSystem *fs); extern DlErrCode datalake_fs_list(DatalakeFileSystem fs, const char *prefix, char ***names_out, int *nnames_out); +/* + * Remove one file. Nothing in the write path calls this -- a stream that + * gives up removes its own work -- but a caller that knows an object is + * finished with can say so. + */ +extern DlErrCode datalake_file_delete(DatalakeFileSystem fs, const char *path); + extern DlErrCode datalake_file_open(DatalakeFileSystem fs, const char *path, DatalakeFileMode mode, DatalakeFile *file_out); diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp new file mode 100644 index 00000000000..3f9a39e2ccd --- /dev/null +++ b/contrib/datalake_fdw/src/common/s3_file_system.cpp @@ -0,0 +1,993 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * s3_file_system.cpp + * The S3 storage backend, over the AWS SDK for C++. + * + * Arrow can be built with an S3 file system of its own, but no RPM of it is: + * Apache's own spec turns the option off, and so does EPEL, which is every + * Arrow a Rocky or RHEL user can install. So the file system is ours, built + * on the SDK directly, and the rest of the module neither knows nor cares -- + * it sees an arrow::fs::FileSystem like any other backend produces. + * + * Only the synchronous S3Client is used. A PostgreSQL backend is a single + * thread that must stay interruptible and must account for its own memory, so + * the CRT client and the transfer manager, which bring their own thread pools + * and buffers, would both be working against us. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/s3_file_system.cpp + * + *------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/storage_backend.h" + +#ifdef DL_HAVE_AWS_SDK + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/storage_arrow.h" + +extern "C" +{ +#include "postgres.h" +#include "miscadmin.h" +} + +namespace +{ + +const char *const DL_S3_ALLOC_TAG = "datalake_fdw"; + +/* Big enough to clear S3's 5 MiB minimum for every part but the last. */ +constexpr int64_t DL_S3_PART_SIZE = 8 * 1024 * 1024; + +Aws::SDKOptions sdk_options; + +/* ---------------------------------------------------------------------- + * Errors + * ---------------------------------------------------------------------- */ + +template +bool +is_not_found(const AwsError &error) +{ + return error.GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND || + error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY || + error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET || + error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND; +} + +/* + * An SDK failure as an Arrow status the facade can classify. The exception + * name travels in the detail, so the user sees "AccessDenied" rather than a + * bare I/O error, and DlErrCode is decided here rather than by matching on + * the text of the message further up. + * + * What goes in the message is the operation, the bucket and the key -- never + * a header or a credential, which is why the SDK's own message is the last + * thing appended and nothing else about the request is. + */ +template +arrow::Status +status_from_aws(const char *operation, const std::string &bucket, + const std::string &key, const AwsError &error) +{ + DlErrCode code = is_not_found(error) ? DL_ERR_NOT_FOUND : DL_ERR_IO; + std::string name(error.GetExceptionName().c_str()); + std::string message = std::string(operation) + " s3://" + bucket + "/" + + key + " failed"; + + if (name.empty()) + name = "S3Error"; + if (error.GetMessage().size() > 0) + message += ": " + std::string(error.GetMessage().c_str()); + + return arrow::Status::IOError(message) + .WithDetail(std::make_shared(code, name)); +} + +/* ---------------------------------------------------------------------- + * Paths + * ---------------------------------------------------------------------- */ + +/* The facade hands down "bucket/key"; the SDK wants the two apart. */ +void +split_path(const std::string &path, std::string *bucket, std::string *key) +{ + std::string::size_type slash = path.find('/'); + + if (slash == std::string::npos) + { + *bucket = path; + key->clear(); + return; + } + *bucket = path.substr(0, slash); + *key = path.substr(slash + 1); + while (!key->empty() && key->back() == '/') + key->pop_back(); +} + +/* ---------------------------------------------------------------------- + * Reading + * ---------------------------------------------------------------------- */ + +class S3InputFile : public arrow::io::RandomAccessFile +{ +public: + S3InputFile(std::shared_ptr client, std::string bucket, + std::string key, int64_t size, arrow::MemoryPool *pool) + : client_(std::move(client)), bucket_(std::move(bucket)), + key_(std::move(key)), size_(size), pool_(pool) + { + } + + arrow::Status Close() override + { + closed_ = true; + return arrow::Status::OK(); + } + + bool closed() const override { return closed_; } + + arrow::Result Tell() const override + { + if (closed_) + return arrow::Status::Invalid("the file is closed"); + return position_; + } + + arrow::Status Seek(int64_t position) override + { + if (closed_) + return arrow::Status::Invalid("the file is closed"); + if (position < 0) + return arrow::Status::Invalid("cannot seek to a negative position"); + position_ = position; + return arrow::Status::OK(); + } + + arrow::Result GetSize() override + { + if (closed_) + return arrow::Status::Invalid("the file is closed"); + return size_; + } + + /* The one call that actually fetches: a ranged GET into the caller's + * memory, so nothing is copied through a buffer of the SDK's own. */ + arrow::Result ReadAt(int64_t position, int64_t nbytes, + void *out) override + { + if (closed_) + return arrow::Status::Invalid("the file is closed"); + if (position < 0 || nbytes < 0) + return arrow::Status::Invalid("read position and length must not " + "be negative"); + + nbytes = std::min(nbytes, std::max(0, size_ - position)); + if (nbytes == 0) + return 0; + + Aws::S3::Model::GetObjectRequest request; + char range[64]; + + snprintf(range, sizeof(range), "bytes=%lld-%lld", + (long long) position, (long long) (position + nbytes - 1)); + request.SetBucket(bucket_.c_str()); + request.SetKey(key_.c_str()); + request.SetRange(range); + + /* + * Hand the SDK the destination instead of taking its stringstream: + * the response body is written straight into memory the caller (and + * so the query's memory accounting) already owns. The stream buffer + * outlives the outcome because it is declared before it. + */ + Aws::Utils::Stream::PreallocatedStreamBuf stream_buf( + reinterpret_cast(out), (uint64_t) nbytes); + + request.SetResponseStreamFactory([&stream_buf]() { + return Aws::New(DL_S3_ALLOC_TAG, &stream_buf); + }); + + auto outcome = client_->GetObject(request); + + if (!outcome.IsSuccess()) + return status_from_aws("read", bucket_, key_, outcome.GetError()); + return outcome.GetResult().GetContentLength(); + } + + arrow::Result> ReadAt(int64_t position, + int64_t nbytes) override + { + ARROW_ASSIGN_OR_RAISE(auto buffer, + arrow::AllocateResizableBuffer(nbytes, pool_)); + + ARROW_ASSIGN_OR_RAISE(int64_t read, + ReadAt(position, nbytes, buffer->mutable_data())); + ARROW_RETURN_NOT_OK(buffer->Resize(read, /* shrink_to_fit = */ false)); + return std::shared_ptr(std::move(buffer)); + } + + arrow::Result Read(int64_t nbytes, void *out) override + { + ARROW_ASSIGN_OR_RAISE(int64_t read, ReadAt(position_, nbytes, out)); + position_ += read; + return read; + } + + arrow::Result> Read(int64_t nbytes) override + { + ARROW_ASSIGN_OR_RAISE(auto buffer, ReadAt(position_, nbytes)); + position_ += buffer->size(); + return buffer; + } + +private: + std::shared_ptr client_; + std::string bucket_; + std::string key_; + int64_t size_; + arrow::MemoryPool *pool_; + int64_t position_ = 0; + bool closed_ = false; +}; + +/* ---------------------------------------------------------------------- + * Writing + * ---------------------------------------------------------------------- */ + +/* + * Buffers up to a part at a time. A write that stays under the part size + * never starts a multipart upload and goes out as a single PutObject on + * close, so the common case of a small file costs one request; a larger one + * starts the upload only when it has a full part to send. + * + * Nothing exists under the key until Close(), and Abort() removes the one + * thing this stream may have created, the multipart upload. That is what + * lets the facade stop deleting by path: an abandoned write leaves nothing + * to delete, and an object at that key belongs to somebody else. + */ +class S3OutputStream : public arrow::io::OutputStream +{ +public: + S3OutputStream(std::shared_ptr client, std::string bucket, + std::string key, arrow::MemoryPool *pool) + : client_(std::move(client)), bucket_(std::move(bucket)), + key_(std::move(key)), pool_(pool) + { + } + + ~S3OutputStream() override + { + if (!closed_ && !upload_id_.empty()) + (void) AbortUpload(); + } + + arrow::Status Init() + { + ARROW_ASSIGN_OR_RAISE(buffer_, + arrow::AllocateResizableBuffer(0, pool_)); + ARROW_RETURN_NOT_OK(buffer_->Reserve(DL_S3_PART_SIZE)); + return arrow::Status::OK(); + } + + arrow::Status Write(const void *data, int64_t nbytes) override + { + if (closed_) + return arrow::Status::Invalid("the stream is closed"); + if (nbytes < 0) + return arrow::Status::Invalid("cannot write a negative length"); + + const uint8_t *from = reinterpret_cast(data); + + while (nbytes > 0) + { + int64_t room = DL_S3_PART_SIZE - buffer_->size(); + int64_t take = std::min(room, nbytes); + int64_t filled = buffer_->size(); + + ARROW_RETURN_NOT_OK(buffer_->Resize(filled + take, false)); + memcpy(buffer_->mutable_data() + filled, from, (size_t) take); + from += take; + nbytes -= take; + position_ += take; + + if (buffer_->size() == DL_S3_PART_SIZE) + ARROW_RETURN_NOT_OK(UploadPart()); + } + return arrow::Status::OK(); + } + + arrow::Status Flush() override { return arrow::Status::OK(); } + + arrow::Result Tell() const override { return position_; } + + bool closed() const override { return closed_; } + + arrow::Status Close() override + { + if (closed_) + return arrow::Status::OK(); + closed_ = true; + + if (upload_id_.empty()) + return PutWholeObject(); + + if (buffer_->size() > 0) + ARROW_RETURN_NOT_OK(UploadPart()); + return CompleteUpload(); + } + + arrow::Status Abort() override + { + if (closed_) + return arrow::Status::OK(); + closed_ = true; + + if (upload_id_.empty()) + return arrow::Status::OK(); /* nothing was ever created */ + return AbortUpload(); + } + +private: + arrow::Status StartUpload() + { + Aws::S3::Model::CreateMultipartUploadRequest request; + + request.SetBucket(bucket_.c_str()); + request.SetKey(key_.c_str()); + + auto outcome = client_->CreateMultipartUpload(request); + + if (!outcome.IsSuccess()) + return status_from_aws("start upload to", bucket_, key_, + outcome.GetError()); + upload_id_ = outcome.GetResult().GetUploadId().c_str(); + return arrow::Status::OK(); + } + + arrow::Status UploadPart() + { + if (upload_id_.empty()) + ARROW_RETURN_NOT_OK(StartUpload()); + + /* Declared before the request, so it outlives the body stream. */ + Aws::Utils::Stream::PreallocatedStreamBuf stream_buf( + buffer_->mutable_data(), (uint64_t) buffer_->size()); + Aws::S3::Model::UploadPartRequest request; + int part = (int) etags_.size() + 1; + + request.SetBucket(bucket_.c_str()); + request.SetKey(key_.c_str()); + request.SetUploadId(upload_id_.c_str()); + request.SetPartNumber(part); + request.SetContentLength(buffer_->size()); + request.SetBody(Aws::MakeShared(DL_S3_ALLOC_TAG, + &stream_buf)); + + auto outcome = client_->UploadPart(request); + + if (!outcome.IsSuccess()) + return status_from_aws("upload part to", bucket_, key_, + outcome.GetError()); + etags_.push_back(outcome.GetResult().GetETag().c_str()); + return buffer_->Resize(0, false); + } + + arrow::Status PutWholeObject() + { + /* Declared before the request, so it outlives the body stream. */ + Aws::Utils::Stream::PreallocatedStreamBuf stream_buf( + buffer_->mutable_data(), (uint64_t) buffer_->size()); + Aws::S3::Model::PutObjectRequest request; + + request.SetBucket(bucket_.c_str()); + request.SetKey(key_.c_str()); + request.SetContentLength(buffer_->size()); + request.SetBody(Aws::MakeShared(DL_S3_ALLOC_TAG, + &stream_buf)); + + auto outcome = client_->PutObject(request); + + if (!outcome.IsSuccess()) + return status_from_aws("write", bucket_, key_, outcome.GetError()); + return arrow::Status::OK(); + } + + arrow::Status CompleteUpload() + { + Aws::S3::Model::CompletedMultipartUpload completed; + Aws::S3::Model::CompleteMultipartUploadRequest request; + + for (size_t i = 0; i < etags_.size(); i++) + { + Aws::S3::Model::CompletedPart part; + + part.SetPartNumber((int) i + 1); + part.SetETag(etags_[i].c_str()); + completed.AddParts(part); + } + + request.SetBucket(bucket_.c_str()); + request.SetKey(key_.c_str()); + request.SetUploadId(upload_id_.c_str()); + request.SetMultipartUpload(completed); + + auto outcome = client_->CompleteMultipartUpload(request); + + if (!outcome.IsSuccess()) + { + arrow::Status status = status_from_aws("finish writing", bucket_, + key_, outcome.GetError()); + + /* The parts are billable until they are abandoned explicitly. */ + (void) AbortUpload(); + return status; + } + upload_id_.clear(); + return arrow::Status::OK(); + } + + arrow::Status AbortUpload() + { + Aws::S3::Model::AbortMultipartUploadRequest request; + + request.SetBucket(bucket_.c_str()); + request.SetKey(key_.c_str()); + request.SetUploadId(upload_id_.c_str()); + + auto outcome = client_->AbortMultipartUpload(request); + + upload_id_.clear(); + if (!outcome.IsSuccess()) + return status_from_aws("abandon the upload to", bucket_, key_, + outcome.GetError()); + return arrow::Status::OK(); + } + + std::shared_ptr client_; + std::string bucket_; + std::string key_; + arrow::MemoryPool *pool_; + std::shared_ptr buffer_; + std::string upload_id_; + std::vector etags_; + int64_t position_ = 0; + bool closed_ = false; +}; + +/* ---------------------------------------------------------------------- + * The file system + * ---------------------------------------------------------------------- */ + +class S3FileSystem : public arrow::fs::FileSystem +{ +public: + S3FileSystem(std::shared_ptr client, + const arrow::io::IOContext &io_context) + : arrow::fs::FileSystem(io_context), client_(std::move(client)) + { + } + + std::string type_name() const override { return "dl_s3"; } + + bool Equals(const arrow::fs::FileSystem &other) const override + { + return this == &other; + } + + arrow::Result GetFileInfo(const std::string &path) override + { + std::string bucket, + key; + + split_path(path, &bucket, &key); + if (key.empty()) + { + /* The bucket itself: a directory as far as the caller knows. */ + arrow::fs::FileInfo info(path); + + info.set_type(arrow::fs::FileType::Directory); + return info; + } + + Aws::S3::Model::HeadObjectRequest request; + + request.SetBucket(bucket.c_str()); + request.SetKey(key.c_str()); + + auto outcome = client_->HeadObject(request); + + if (outcome.IsSuccess()) + { + arrow::fs::FileInfo info(path); + + info.set_type(arrow::fs::FileType::File); + info.set_size(outcome.GetResult().GetContentLength()); + info.set_mtime(std::chrono::system_clock::time_point( + std::chrono::milliseconds( + outcome.GetResult().GetLastModified().Millis()))); + return info; + } + if (!is_not_found(outcome.GetError())) + return status_from_aws("inspect", bucket, key, outcome.GetError()); + + /* No object by that name; it may still be a prefix with objects. */ + ARROW_ASSIGN_OR_RAISE(bool is_prefix, PrefixExists(bucket, key)); + + arrow::fs::FileInfo info(path); + + info.set_type(is_prefix ? arrow::fs::FileType::Directory : + arrow::fs::FileType::NotFound); + return info; + } + + arrow::Result GetFileInfo( + const arrow::fs::FileSelector &select) override + { + std::string bucket, + key; + arrow::fs::FileInfoVector infos; + Aws::String token; + bool more = true; + + split_path(select.base_dir, &bucket, &key); + + const std::string prefix = key.empty() ? std::string() : key + "/"; + + while (more) + { + Aws::S3::Model::ListObjectsV2Request request; + + request.SetBucket(bucket.c_str()); + if (!prefix.empty()) + request.SetPrefix(prefix.c_str()); + if (!select.recursive) + request.SetDelimiter("/"); + if (!token.empty()) + request.SetContinuationToken(token); + + auto outcome = client_->ListObjectsV2(request); + + if (!outcome.IsSuccess()) + return status_from_aws("list", bucket, key, outcome.GetError()); + + const auto &result = outcome.GetResult(); + + for (const auto &object : result.GetContents()) + { + std::string object_key(object.GetKey().c_str()); + + /* The prefix marker some tools write for an empty folder. */ + if (!object_key.empty() && object_key.back() == '/') + continue; + + arrow::fs::FileInfo info(bucket + "/" + object_key); + + info.set_type(arrow::fs::FileType::File); + info.set_size(object.GetSize()); + info.set_mtime(std::chrono::system_clock::time_point( + std::chrono::milliseconds(object.GetLastModified().Millis()))); + infos.push_back(std::move(info)); + } + + for (const auto &common : result.GetCommonPrefixes()) + { + std::string dir(common.GetPrefix().c_str()); + + while (!dir.empty() && dir.back() == '/') + dir.pop_back(); + + arrow::fs::FileInfo info(bucket + "/" + dir); + + info.set_type(arrow::fs::FileType::Directory); + infos.push_back(std::move(info)); + } + + more = result.GetIsTruncated(); + token = result.GetNextContinuationToken(); + } + + /* + * Object storage has no empty directory, so "nothing under this + * prefix" and "this prefix does not exist" are the same answer. The + * caller asked to be told about the second one. + */ + if (infos.empty() && !select.allow_not_found) + return arrow::Status::IOError("s3://", bucket, "/", key, + " does not exist") + .WithDetail(std::make_shared(DL_ERR_NOT_FOUND, + "NoSuchKey")); + return infos; + } + + arrow::Result> OpenInputStream( + const std::string &path) override + { + ARROW_ASSIGN_OR_RAISE(auto file, OpenInputFile(path)); + return file; + } + + arrow::Result> OpenInputFile( + const std::string &path) override + { + std::string bucket, + key; + + split_path(path, &bucket, &key); + if (key.empty()) + return arrow::Status::IOError("s3://", bucket, + " names a bucket, not an object"); + + Aws::S3::Model::HeadObjectRequest request; + + request.SetBucket(bucket.c_str()); + request.SetKey(key.c_str()); + + auto outcome = client_->HeadObject(request); + + if (!outcome.IsSuccess()) + return status_from_aws("open", bucket, key, outcome.GetError()); + + return std::make_shared(client_, bucket, key, + outcome.GetResult().GetContentLength(), + io_context().pool()); + } + + arrow::Result> OpenOutputStream( + const std::string &path, + const std::shared_ptr &metadata) override + { + std::string bucket, + key; + + (void) metadata; + split_path(path, &bucket, &key); + if (key.empty()) + return arrow::Status::IOError("s3://", bucket, + " names a bucket, not an object"); + + auto stream = std::make_shared( + client_, bucket, key, io_context().pool()); + + ARROW_RETURN_NOT_OK(stream->Init()); + return stream; + } + + arrow::Status DeleteFile(const std::string &path) override + { + std::string bucket, + key; + + split_path(path, &bucket, &key); + if (key.empty()) + return arrow::Status::IOError("s3://", bucket, + " names a bucket, not an object"); + + Aws::S3::Model::HeadObjectRequest head; + + head.SetBucket(bucket.c_str()); + head.SetKey(key.c_str()); + + auto found = client_->HeadObject(head); + + if (!found.IsSuccess()) + return status_from_aws("delete", bucket, key, found.GetError()); + + Aws::S3::Model::DeleteObjectRequest request; + + request.SetBucket(bucket.c_str()); + request.SetKey(key.c_str()); + + auto outcome = client_->DeleteObject(request); + + if (!outcome.IsSuccess()) + return status_from_aws("delete", bucket, key, outcome.GetError()); + return arrow::Status::OK(); + } + + /* + * The rest of the interface is not part of what this module asks of a + * backend (see storage_backend.h), and object storage has no directories + * to create or rename anyway. + */ + arrow::Status CreateDir(const std::string &path, bool recursive) override + { + (void) path; + (void) recursive; + return arrow::Status::NotImplemented("s3: creating a directory"); + } + + arrow::Status DeleteDir(const std::string &path) override + { + (void) path; + return arrow::Status::NotImplemented("s3: deleting a directory"); + } + + arrow::Status DeleteDirContents(const std::string &path, + bool missing_dir_ok) override + { + (void) path; + (void) missing_dir_ok; + return arrow::Status::NotImplemented("s3: deleting a directory"); + } + + arrow::Status DeleteRootDirContents() override + { + return arrow::Status::NotImplemented("s3: deleting a directory"); + } + + arrow::Status Move(const std::string &src, const std::string &dest) override + { + (void) src; + (void) dest; + return arrow::Status::NotImplemented("s3: moving an object"); + } + + arrow::Status CopyFile(const std::string &src, const std::string &dest) override + { + (void) src; + (void) dest; + return arrow::Status::NotImplemented("s3: copying an object"); + } + + arrow::Result> OpenAppendStream( + const std::string &path, + const std::shared_ptr &metadata) override + { + (void) path; + (void) metadata; + return arrow::Status::NotImplemented("s3: appending to an object"); + } + +private: + arrow::Result PrefixExists(const std::string &bucket, + const std::string &key) + { + Aws::S3::Model::ListObjectsV2Request request; + + request.SetBucket(bucket.c_str()); + request.SetPrefix((key + "/").c_str()); + request.SetMaxKeys(1); + + auto outcome = client_->ListObjectsV2(request); + + if (!outcome.IsSuccess()) + return status_from_aws("inspect", bucket, key, outcome.GetError()); + return outcome.GetResult().GetKeyCount() > 0; + } + + std::shared_ptr client_; +}; + +/* ---------------------------------------------------------------------- + * Mounting + * ---------------------------------------------------------------------- */ + +const char * +option_value(const DlKeyValue *kv, int nkv, const char *key) +{ + for (int i = 0; i < nkv; i++) + { + if (kv[i].key != NULL && strcmp(kv[i].key, key) == 0 && + kv[i].value != NULL && kv[i].value[0] != '\0') + return kv[i].value; + } + return NULL; +} + +bool +option_is_true(const char *value) +{ + return value != NULL && + (strcasecmp(value, "true") == 0 || strcasecmp(value, "on") == 0 || + strcasecmp(value, "yes") == 0 || strcasecmp(value, "t") == 0 || + strcasecmp(value, "y") == 0 || strcmp(value, "1") == 0); +} + +arrow::Status +initialize_s3(void) +{ + /* + * The SDK starts threads and opens handles, neither of which survives a + * fork, so this may only ever run in a backend. The facade calls it + * before the first mount in each process and registers the matching + * shutdown there. + */ + Assert(MyProcPid != PostmasterPid); + Aws::InitAPI(sdk_options); + return arrow::Status::OK(); +} + +void +finalize_s3(void) +{ + Aws::ShutdownAPI(sdk_options); +} + +arrow::Result +mount_s3(const DatalakeLocation *location, const DlKeyValue *kv, int nkv, + const DatalakeStorageHost *host) +{ + if (location == NULL || location->authority == NULL || + location->authority[0] == '\0') + return arrow::Status::Invalid("s3 location has no bucket"); + + const char *endpoint = option_value(kv, nkv, "endpoint"); + const char *region = option_value(kv, nkv, "region"); + const char *path_style = option_value(kv, nkv, "path_style_access"); + const char *access_key = option_value(kv, nkv, "aws_access_key_id"); + const char *secret_key = option_value(kv, nkv, "aws_secret_access_key"); + const char *session_token = option_value(kv, nkv, "aws_session_token"); + + if (endpoint == NULL) + endpoint = location->endpoint; + if (region == NULL) + region = location->region; + + Aws::S3::S3ClientConfiguration config; + + config.region = region != NULL ? region : "us-east-1"; + + /* + * Bounded rather than left to the SDK's defaults: a backend blocked on a + * socket is a session that cannot be cancelled, and an unreachable + * endpoint has to become an error while someone is still waiting for it. + */ + config.connectTimeoutMs = 5000; + config.requestTimeoutMs = 300000; + config.retryStrategy = std::make_shared(3); + + /* + * Virtual-host addressing asks DNS for bucket.host, which is right for + * AWS and wrong for most things you can run yourself. So the server's + * setting decides, and where there is no setting, an explicit endpoint + * means path style and its absence means AWS. + */ + if (path_style != NULL) + config.useVirtualAddressing = !option_is_true(path_style); + else + config.useVirtualAddressing = endpoint == NULL || endpoint[0] == '\0'; + + if (endpoint != NULL && endpoint[0] != '\0') + { + std::string url(endpoint); + + if (url.compare(0, 7, "http://") == 0) + { + config.scheme = Aws::Http::Scheme::HTTP; + url = url.substr(7); + } + else if (url.compare(0, 8, "https://") == 0) + { + config.scheme = Aws::Http::Scheme::HTTPS; + url = url.substr(8); + } + while (!url.empty() && url.back() == '/') + url.pop_back(); + config.endpointOverride = url.c_str(); + } + + std::shared_ptr client; + + if (access_key != NULL && secret_key != NULL) + { + /* Credentials the user gave us, through a user mapping. */ + auto provider = + Aws::MakeShared( + DL_S3_ALLOC_TAG, access_key, secret_key, + session_token != NULL ? session_token : ""); + + client = std::make_shared(provider, nullptr, config); + } + else + { + /* None given: whatever the environment already grants this host. */ + auto provider = + Aws::MakeShared( + DL_S3_ALLOC_TAG); + + client = std::make_shared(provider, nullptr, config); + } + + DatalakeMountedFs mounted; + + mounted.fs = std::make_shared( + client, arrow::io::IOContext(dl_storage_host_pool(host))); + mounted.root = std::string(location->authority) + + (location->path_prefix != NULL ? location->path_prefix : ""); + return mounted; +} + +} /* namespace */ + +static const DatalakeStorageBackend s3_storage_backend = { + DL_STORAGE_ABI_VERSION, + sizeof(DatalakeStorageBackend), + "s3", + ARROW_VERSION_STRING, + DL_STORAGE_ABI_FINGERPRINT, + mount_s3, + initialize_s3, + finalize_s3 +}; + +#else /* !DL_HAVE_AWS_SDK */ + +static arrow::Result +mount_s3(const DatalakeLocation *, const DlKeyValue *, int, + const DatalakeStorageHost *) +{ + return arrow::Status::NotImplemented( + "datalake_fdw was built without the AWS SDK for C++, so s3:// " + "locations cannot be opened; rebuild the extension with " + "AWS_SDK_PREFIX= pointing at an installed SDK"); +} + +static const DatalakeStorageBackend s3_storage_backend = { + DL_STORAGE_ABI_VERSION, + sizeof(DatalakeStorageBackend), + "s3", + ARROW_VERSION_STRING, + DL_STORAGE_ABI_FINGERPRINT, + mount_s3, + NULL, + NULL +}; + +#endif /* DL_HAVE_AWS_SDK */ + +DlErrCode +datalake_register_s3_backend(void) +{ + return datalake_register_storage_backend(&s3_storage_backend); +} diff --git a/contrib/datalake_fdw/src/test/datalake_fdw_test.c b/contrib/datalake_fdw/src/test/datalake_fdw_test.c index 4a17a86293b..3945f4fe894 100644 --- a/contrib/datalake_fdw/src/test/datalake_fdw_test.c +++ b/contrib/datalake_fdw/src/test/datalake_fdw_test.c @@ -60,6 +60,7 @@ PG_FUNCTION_INFO_V1(datalake_parquet_read); PG_FUNCTION_INFO_V1(datalake_storage_write_text); PG_FUNCTION_INFO_V1(datalake_storage_read_text); PG_FUNCTION_INFO_V1(datalake_storage_list); +PG_FUNCTION_INFO_V1(datalake_storage_delete); PG_FUNCTION_INFO_V1(datalake_storage_probe); PG_FUNCTION_INFO_V1(datalake_storage_register_bad); @@ -784,6 +785,42 @@ datalake_storage_list(PG_FUNCTION_ARGS) return (Datum) 0; } +Datum +datalake_storage_delete(PG_FUNCTION_ARGS) +{ + char *relative; + DatalakeFileSystem volatile open_fs = NULL; + + check_nargs(fcinfo, 2); + if (PG_ARGISNULL(0)) + PG_RETURN_NULL(); + open_fs = storage_open_uri(fcinfo, 0, 1, true, &relative); + + PG_TRY(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + DlErrCode rc = datalake_file_delete(fs, relative); + + if (rc != DL_OK) + dl_error_report(ERROR, rc, "delete storage file"); + } + PG_CATCH(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + PG_RE_THROW(); + } + PG_END_TRY(); + + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + } + PG_RETURN_BOOL(true); +} + Datum datalake_storage_probe(PG_FUNCTION_ARGS) { diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out index 690dc26712f..f45c9ae82ee 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local.out @@ -95,9 +95,9 @@ SELECT * FROM datalake_storage_list( (1 row) SELECT datalake_storage_probe('s3'); - datalake_storage_probe ------------------------------------------------ - no storage backend registered for scheme "s3" + datalake_storage_probe +------------------------ + supported (1 row) -- Each malformed registration must be rejected by the check it breaks, not by diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local_1.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local_1.out new file mode 100644 index 00000000000..398a4ef06f4 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_local/expected/storage_local_1.out @@ -0,0 +1,128 @@ +-- Storage facade behavior shared by the built-in file backend and a backend +-- registered through the public plugin contract. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_storage_local && mkdir -p /tmp/datalake_fdw_storage_local/file /tmp/datalake_fdw_storage_local/dltest'; +SELECT datalake_storage_write_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt', 'file-data'); + datalake_storage_write_text +----------------------------- + 9 +(1 row) + +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt'); + datalake_storage_read_text +---------------------------- + file-data +(1 row) + +SELECT * FROM datalake_storage_list( + 'file:///tmp/datalake_fdw_storage_local/file'); + datalake_storage_list +-------------------------------------------- + /tmp/datalake_fdw_storage_local/file/a.txt +(1 row) + +\set VERBOSITY sqlstate +SELECT datalake_storage_write_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt', 'replacement'); +ERROR: 42P07 +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/missing.txt'); +ERROR: 42704 +SELECT datalake_storage_read_text('file://host/tmp/a.txt'); +ERROR: 22023 +SELECT datalake_storage_read_text('/tmp/a.txt'); +ERROR: 22023 +-- A path may not climb out of the volume it was resolved against. +SELECT datalake_storage_read_text('file:///tmp/datalake_fdw_storage_local/..'); +ERROR: 22023 +\set VERBOSITY default +SELECT datalake_storage_read_text( + 'file:///tmp/datalake_fdw_storage_local/file/a.txt'); + datalake_storage_read_text +---------------------------- + file-data +(1 row) + +SELECT datalake_storage_write_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt', 'dltest-data'); + datalake_storage_write_text +----------------------------- + 11 +(1 row) + +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt'); + datalake_storage_read_text +---------------------------- + dltest-data +(1 row) + +SELECT * FROM datalake_storage_list( + 'dltest:///tmp/datalake_fdw_storage_local/dltest'); + datalake_storage_list +----------------------- + a.txt +(1 row) + +\set VERBOSITY sqlstate +SELECT datalake_storage_write_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt', 'replacement'); +ERROR: 42P07 +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/missing.txt'); +ERROR: 42704 +\set VERBOSITY default +SELECT datalake_storage_read_text( + 'dltest:///tmp/datalake_fdw_storage_local/dltest/a.txt'); + datalake_storage_read_text +---------------------------- + dltest-data +(1 row) + +-- A write that is refused leaves the file it refused to replace untouched, +-- and leaves nothing else behind either. +SELECT * FROM datalake_storage_list( + 'file:///tmp/datalake_fdw_storage_local/file'); + datalake_storage_list +-------------------------------------------- + /tmp/datalake_fdw_storage_local/file/a.txt +(1 row) + +SELECT datalake_storage_probe('s3'); + datalake_storage_probe +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + datalake_fdw was built without the AWS SDK for C++, so s3:// locations cannot be opened; rebuild the extension with AWS_SDK_PREFIX= pointing at an installed SDK +(1 row) + +-- Each malformed registration must be rejected by the check it breaks, not by +-- some later one: every pattern names the field and the value it reports, so +-- a check that stopped working could not fall through to another and still +-- match. +SELECT kind, + datalake_storage_register_bad(kind) LIKE pattern AS rejected_by_its_check +FROM (VALUES + ('abi_version', '%ABI version mismatch: expected 1, got 2%'), + ('struct_size', '%struct size mismatch: expected at least %, got 0%'), + ('arrow_version', '%Arrow version mismatch: expected "%", got "0.0.0-test"%'), + ('abi_fingerprint', + '%ABI fingerprint mismatch: expected "%", got "gcc0;cxx11abi=9;arrow=0.0.0-test"%'), + ('duplicate', '%scheme expected to be unique, got duplicate "dltest"%') +) AS t(kind, pattern) +ORDER BY kind; + kind | rejected_by_its_check +-----------------+----------------------- + abi_fingerprint | t + abi_version | t + arrow_version | t + duplicate | t + struct_size | t +(5 rows) + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_storage_local'; +-- End of storage_local. diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out new file mode 100644 index 00000000000..7f0dcd06681 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out @@ -0,0 +1,162 @@ +-- The s3 backend against a real S3-compatible service. Endpoint and +-- credentials come from the environment, so this category only runs where +-- DATALAKE_TEST_S3_ENDPOINT is set; the Makefile skips it otherwise. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +-- This file hands the server credentials, and psql puts them in the statement +-- it sends, so with statement logging on they would land in the server log +-- and no check could tell that apart from the extension leaking them. With +-- it off, a credential in the log came from the code under test. +SET log_statement = 'none'; +SET log_min_duration_statement = -1; +SET log_min_error_statement = 'panic'; +-- Which service, which bucket and whose credentials differ from one machine +-- to the next, so they are read into settings with the echo off: what this +-- file asserts must not depend on where it ran. Objects also go under a +-- prefix of this run's own, so two runs against one bucket cannot collide. +\set ECHO none +SELECT current_setting('datalake.s3_endpoint') <> '' AS have_endpoint, + current_setting('datalake.s3_bucket') <> '' AS have_bucket, + current_setting('datalake.s3_run') <> '' AS have_run; + have_endpoint | have_bucket | have_run +---------------+-------------+---------- + t | t | t +(1 row) + +CREATE FUNCTION s3_kv(secret text DEFAULT NULL) RETURNS text[] LANGUAGE sql AS $$ + SELECT ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'aws_access_key_id=' || current_setting('datalake.s3_access_key'), + 'aws_secret_access_key=' || + coalesce(secret, current_setting('datalake.s3_secret'))] +$$; +CREATE FUNCTION s3_prefix() RETURNS text LANGUAGE sql AS $$ + SELECT format('s3://%s/datalake_regress/%s', + current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run')) +$$; +CREATE FUNCTION s3_uri(name text) RETURNS text LANGUAGE sql AS $$ + SELECT s3_prefix() || '/' || name +$$; +-- Round trip. +SELECT datalake_storage_write_text(s3_uri('a.txt'), 'first-object', s3_kv()); + datalake_storage_write_text +----------------------------- + 12 +(1 row) + +SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); + datalake_storage_read_text +---------------------------- + first-object +(1 row) + +-- Larger than the 8 MiB part size, so this goes out as a multipart upload and +-- has to come back byte for byte. +SELECT datalake_storage_write_text(s3_uri('big.bin'), + repeat('0123456789', 900000), s3_kv()) AS bytes_written; + bytes_written +--------------- + 9000000 +(1 row) + +SELECT length(datalake_storage_read_text(s3_uri('big.bin'), s3_kv())) AS bytes_read, + md5(datalake_storage_read_text(s3_uri('big.bin'), s3_kv())) + = md5(repeat('0123456789', 900000)) AS same_bytes; + bytes_read | same_bytes +------------+------------ + 9000000 | t +(1 row) + +-- Listing reports this run's objects by their native path. +SELECT replace(path, + format('%s/datalake_regress/%s/', current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run')), '') AS object +FROM datalake_storage_list(s3_prefix(), s3_kv()) AS path +ORDER BY 1; + object +--------- + a.txt + big.bin +(2 rows) + +\set VERBOSITY sqlstate +-- An object that exists is never replaced. +SELECT datalake_storage_write_text(s3_uri('a.txt'), 'replacement', s3_kv()); +ERROR: 42P07 +-- A key that is not there, and a bucket that is not there. +SELECT datalake_storage_read_text(s3_uri('missing.txt'), s3_kv()); +ERROR: 42704 +SELECT datalake_storage_read_text('s3://datalake-no-such-bucket-9f2b/x.txt', s3_kv()); +ERROR: 42704 +-- A wrong secret is refused. +SELECT datalake_storage_read_text(s3_uri('a.txt'), + s3_kv(current_setting('datalake.s3_bad_secret'))); +ERROR: 58030 +\set VERBOSITY default +-- The refused write left the object as it was. +SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); + datalake_storage_read_text +---------------------------- + first-object +(1 row) + +-- What the user is told names the object but not the secret. The text of an +-- SDK message varies between services, so this asks what must and must not be +-- in it rather than pinning the whole string. +CREATE FUNCTION s3_error(uri text, kv text[]) RETURNS text LANGUAGE plpgsql AS $fn$ +DECLARE + message text; + detail text; +BEGIN + PERFORM datalake_storage_read_text(uri, kv); + RETURN 'no error'; +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS message = MESSAGE_TEXT, detail = PG_EXCEPTION_DETAIL; + RETURN message || ' | ' || detail; +END +$fn$; +SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret'))) + LIKE '%' || current_setting('datalake.s3_bad_secret') || '%' + AS leaks_the_secret, + s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret'))) + LIKE '%a.txt%' AS names_the_object, + s3_error(s3_uri('a.txt'), s3_kv()) + = 'no error' AS good_credentials_still_work; + leaks_the_secret | names_the_object | good_credentials_still_work +------------------+------------------+----------------------------- + f | t | t +(1 row) + +-- An endpoint that answers nothing has to become an error while someone is +-- still waiting for it, rather than a session that cannot be cancelled. +\set VERBOSITY sqlstate +SELECT datalake_storage_read_text(s3_uri('a.txt'), + ARRAY['endpoint=http://10.255.255.1:9000', + 'region=us-east-1', 'path_style_access=true', + 'aws_access_key_id=x', 'aws_secret_access_key=y']); +ERROR: 58030 +\set VERBOSITY default +-- The session still works afterwards. +SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); + datalake_storage_read_text +---------------------------- + first-object +(1 row) + +-- Take this run's objects back out again. +SELECT datalake_storage_delete(s3_uri('a.txt'), s3_kv()) AS deleted_a, + datalake_storage_delete(s3_uri('big.bin'), s3_kv()) AS deleted_big; + deleted_a | deleted_big +-----------+------------- + t | t +(1 row) + +\set VERBOSITY sqlstate +SELECT datalake_storage_delete(s3_uri('a.txt'), s3_kv()); +ERROR: 42704 +\set VERBOSITY default +DROP FUNCTION s3_error(text, text[]), s3_uri(text), s3_prefix(), s3_kv(text); diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql new file mode 100644 index 00000000000..3a325e3d90f --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql @@ -0,0 +1,147 @@ +-- The s3 backend against a real S3-compatible service. Endpoint and +-- credentials come from the environment, so this category only runs where +-- DATALAKE_TEST_S3_ENDPOINT is set; the Makefile skips it otherwise. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +-- This file hands the server credentials, and psql puts them in the statement +-- it sends, so with statement logging on they would land in the server log +-- and no check could tell that apart from the extension leaking them. With +-- it off, a credential in the log came from the code under test. +SET log_statement = 'none'; +SET log_min_duration_statement = -1; +SET log_min_error_statement = 'panic'; + +-- Which service, which bucket and whose credentials differ from one machine +-- to the next, so they are read into settings with the echo off: what this +-- file asserts must not depend on where it ran. Objects also go under a +-- prefix of this run's own, so two runs against one bucket cannot collide. +\set ECHO none +\getenv endpoint DATALAKE_TEST_S3_ENDPOINT +\getenv bucket DATALAKE_TEST_S3_BUCKET +\getenv access_key DATALAKE_TEST_S3_ACCESS_KEY +\getenv secret_key DATALAKE_TEST_S3_SECRET_KEY +\getenv region DATALAKE_TEST_S3_REGION +\getenv path_style DATALAKE_TEST_S3_PATH_STYLE +\getenv bad_secret DATALAKE_TEST_S3_BAD_SECRET + +SELECT set_config('datalake.s3_endpoint', :'endpoint', false) AS endpoint, + set_config('datalake.s3_bucket', :'bucket', false) AS bucket, + set_config('datalake.s3_access_key', :'access_key', false) AS access_key, + set_config('datalake.s3_secret', :'secret_key', false) AS secret, + set_config('datalake.s3_region', + coalesce(nullif(:'region', ''), 'us-east-1'), false) AS region, + set_config('datalake.s3_path_style', + coalesce(nullif(:'path_style', ''), 'true'), false) AS path_style, + set_config('datalake.s3_run', + to_char(clock_timestamp(), 'YYYYMMDDHH24MISSMS') || '_' || + pg_backend_pid(), false) AS run, + -- A wrong secret, taken from the environment rather than written here: + -- whatever this file says ends up in the server log as statement text, + -- and a secret in a log is the very thing the harness watches for. + set_config('datalake.s3_bad_secret', + coalesce(nullif(:'bad_secret', ''), 'not-the-secret'), + false) AS bad_secret +\gset +\set ECHO all + +SELECT current_setting('datalake.s3_endpoint') <> '' AS have_endpoint, + current_setting('datalake.s3_bucket') <> '' AS have_bucket, + current_setting('datalake.s3_run') <> '' AS have_run; + +CREATE FUNCTION s3_kv(secret text DEFAULT NULL) RETURNS text[] LANGUAGE sql AS $$ + SELECT ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'aws_access_key_id=' || current_setting('datalake.s3_access_key'), + 'aws_secret_access_key=' || + coalesce(secret, current_setting('datalake.s3_secret'))] +$$; +CREATE FUNCTION s3_prefix() RETURNS text LANGUAGE sql AS $$ + SELECT format('s3://%s/datalake_regress/%s', + current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run')) +$$; +CREATE FUNCTION s3_uri(name text) RETURNS text LANGUAGE sql AS $$ + SELECT s3_prefix() || '/' || name +$$; + +-- Round trip. +SELECT datalake_storage_write_text(s3_uri('a.txt'), 'first-object', s3_kv()); +SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); + +-- Larger than the 8 MiB part size, so this goes out as a multipart upload and +-- has to come back byte for byte. +SELECT datalake_storage_write_text(s3_uri('big.bin'), + repeat('0123456789', 900000), s3_kv()) AS bytes_written; +SELECT length(datalake_storage_read_text(s3_uri('big.bin'), s3_kv())) AS bytes_read, + md5(datalake_storage_read_text(s3_uri('big.bin'), s3_kv())) + = md5(repeat('0123456789', 900000)) AS same_bytes; + +-- Listing reports this run's objects by their native path. +SELECT replace(path, + format('%s/datalake_regress/%s/', current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run')), '') AS object +FROM datalake_storage_list(s3_prefix(), s3_kv()) AS path +ORDER BY 1; + +\set VERBOSITY sqlstate +-- An object that exists is never replaced. +SELECT datalake_storage_write_text(s3_uri('a.txt'), 'replacement', s3_kv()); +-- A key that is not there, and a bucket that is not there. +SELECT datalake_storage_read_text(s3_uri('missing.txt'), s3_kv()); +SELECT datalake_storage_read_text('s3://datalake-no-such-bucket-9f2b/x.txt', s3_kv()); +-- A wrong secret is refused. +SELECT datalake_storage_read_text(s3_uri('a.txt'), + s3_kv(current_setting('datalake.s3_bad_secret'))); +\set VERBOSITY default + +-- The refused write left the object as it was. +SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); + +-- What the user is told names the object but not the secret. The text of an +-- SDK message varies between services, so this asks what must and must not be +-- in it rather than pinning the whole string. +CREATE FUNCTION s3_error(uri text, kv text[]) RETURNS text LANGUAGE plpgsql AS $fn$ +DECLARE + message text; + detail text; +BEGIN + PERFORM datalake_storage_read_text(uri, kv); + RETURN 'no error'; +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS message = MESSAGE_TEXT, detail = PG_EXCEPTION_DETAIL; + RETURN message || ' | ' || detail; +END +$fn$; + +SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret'))) + LIKE '%' || current_setting('datalake.s3_bad_secret') || '%' + AS leaks_the_secret, + s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret'))) + LIKE '%a.txt%' AS names_the_object, + s3_error(s3_uri('a.txt'), s3_kv()) + = 'no error' AS good_credentials_still_work; + +-- An endpoint that answers nothing has to become an error while someone is +-- still waiting for it, rather than a session that cannot be cancelled. +\set VERBOSITY sqlstate +SELECT datalake_storage_read_text(s3_uri('a.txt'), + ARRAY['endpoint=http://10.255.255.1:9000', + 'region=us-east-1', 'path_style_access=true', + 'aws_access_key_id=x', 'aws_secret_access_key=y']); +\set VERBOSITY default + +-- The session still works afterwards. +SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); + +-- Take this run's objects back out again. +SELECT datalake_storage_delete(s3_uri('a.txt'), s3_kv()) AS deleted_a, + datalake_storage_delete(s3_uri('big.bin'), s3_kv()) AS deleted_big; +\set VERBOSITY sqlstate +SELECT datalake_storage_delete(s3_uri('a.txt'), s3_kv()); +\set VERBOSITY default + +DROP FUNCTION s3_error(text, text[]), s3_uri(text), s3_prefix(), s3_kv(text); From 0748365736a6b403ac92493448233056829654c4 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:25 +0800 Subject: [PATCH 5/9] datalake_fdw: read and write Parquet wherever the volume is The Parquet reader and writer opened paths through the local file system directly. They now open through the storage facade, so a fragment names a filesystem and a path relative to its mount, and the same reader works against a volume on s3 as on a shared mount. "Create only" moves with them: the writer asks the facade, which is the one place that checks a name is free, and giving up is the stream's Abort rather than a delete by path -- after a failed write the name may already belong to somebody else. A volume resolves to a location plus credentials in one place, checking USAGE on the server and reading the user mapping, the PUBLIC mapping, or neither. --- .../datalake_fdw/datalake_fdw_test--1.0.sql | 18 ++-- .../src/common/file_system_wrapper.cpp | 45 ++++++++++ .../datalake_fdw/src/common/storage_arrow.h | 13 +++ .../datalake_fdw/src/format/arrow_support.cpp | 34 ++++++- contrib/datalake_fdw/src/format/format.h | 12 ++- .../src/format/parquet/parquet_internal.h | 3 +- .../src/format/parquet/parquet_read.cpp | 7 +- .../src/format/parquet/parquet_write.cpp | 72 +++++---------- .../iceberg_volume_fdw/iceberg_volume_fdw.c | 89 +++++++++++++++++++ .../iceberg_volume_option.h | 12 +++ .../datalake_fdw/src/test/datalake_fdw_test.c | 89 ++++++++++++++++++- 11 files changed, 325 insertions(+), 69 deletions(-) diff --git a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql index 29d0af9a5f9..efa82bbe749 100644 --- a/contrib/datalake_fdw/datalake_fdw_test--1.0.sql +++ b/contrib/datalake_fdw/datalake_fdw_test--1.0.sql @@ -39,20 +39,26 @@ CREATE FUNCTION datalake_parquet_write(path text, query text, row_group_size int DEFAULT 0, - compression text DEFAULT '') -RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C STRICT VOLATILE; + compression text DEFAULT '', + volume text DEFAULT NULL) +-- Not STRICT: volume defaults to NULL, and a strict function would answer NULL +-- rather than run. The required arguments are checked in C instead. +RETURNS bigint AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE; -REVOKE EXECUTE ON FUNCTION datalake_parquet_write(text, text, int, text) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_parquet_write(text, text, int, text, text) + FROM PUBLIC; -- field_ids names, for each column of the definition list, the Iceberg field -- id it is read from; empty reads the file as it is, column for column. CREATE FUNCTION datalake_parquet_read(path text, first_row_group int DEFAULT 0, n_row_groups int DEFAULT 0, - field_ids int[] DEFAULT '{}') -RETURNS SETOF record AS 'MODULE_PATHNAME' LANGUAGE C STRICT EXECUTE ON COORDINATOR; + field_ids int[] DEFAULT '{}', + volume text DEFAULT NULL) +RETURNS SETOF record AS 'MODULE_PATHNAME' LANGUAGE C EXECUTE ON COORDINATOR; -REVOKE EXECUTE ON FUNCTION datalake_parquet_read(text, int, int, int[]) FROM PUBLIC; +REVOKE EXECUTE ON FUNCTION datalake_parquet_read(text, int, int, int[], text) + FROM PUBLIC; -- Test-only storage contract functions. CREATE FUNCTION datalake_storage_write_text(uri text, content text, diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp index e119f2f8c79..d30dbf1255e 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -463,6 +463,51 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, return rc; } +/* + * The existence rule both openers share. Checking before creating is not what + * makes a write safe -- two writers can still pass the check together, and on + * object storage there is nothing to hold -- it is what turns a name collision + * into an error a user can read instead of a file somebody silently lost. The + * backends enforce the rule where they can: the local one creates with O_EXCL. + */ +static arrow::Status +require_absent(DatalakeFileSystem fs, const std::string &native) +{ + ARROW_ASSIGN_OR_RAISE(auto info, fs->fs->GetFileInfo(native)); + + if (info.type() != arrow::fs::FileType::NotFound) + return arrow::Status::AlreadyExists("\"", native, "\" already exists"); + return arrow::Status::OK(); +} + +arrow::Result> +dl_storage_open_input(DatalakeFileSystem fs, const char *relative) +{ + if (fs == NULL || relative == NULL) + return arrow::Status::Invalid("no file system or path to open"); + if (!dl_storage_path_is_safe(relative)) + return arrow::Status::Invalid("storage path \"", relative, + "\" must be relative to the volume and " + "must not contain \".\" or \"..\""); + return fs->fs->OpenInputFile(dl_storage_native_path(fs, relative)); +} + +arrow::Result> +dl_storage_open_output(DatalakeFileSystem fs, const char *relative) +{ + if (fs == NULL || relative == NULL) + return arrow::Status::Invalid("no file system or path to create"); + if (!dl_storage_path_is_safe(relative)) + return arrow::Status::Invalid("storage path \"", relative, + "\" must be relative to the volume and " + "must not contain \".\" or \"..\""); + + const std::string native = dl_storage_native_path(fs, relative); + + ARROW_RETURN_NOT_OK(require_absent(fs, native)); + return fs->fs->OpenOutputStream(native); +} + extern "C" DlErrCode datalake_file_delete(DatalakeFileSystem fs, const char *path) { diff --git a/contrib/datalake_fdw/src/common/storage_arrow.h b/contrib/datalake_fdw/src/common/storage_arrow.h index 584fad33eb6..2c4b16e4408 100644 --- a/contrib/datalake_fdw/src/common/storage_arrow.h +++ b/contrib/datalake_fdw/src/common/storage_arrow.h @@ -34,11 +34,24 @@ #include #include +#include #include #include "common/file_system_wrapper.h" arrow::fs::FileSystem *dl_storage_arrow_fs(DatalakeFileSystem fs); + +/* + * Open a file below a mount, for the format layer, under the same rules the C + * facade applies: a read must find the file, and a write must not. Both take + * a path relative to the mount root, and both refuse one that could climb out + * of it. They exist so that "creating a file never replaces one" is decided + * once, rather than once per format. + */ +arrow::Result> + dl_storage_open_input(DatalakeFileSystem fs, const char *relative); +arrow::Result> + dl_storage_open_output(DatalakeFileSystem fs, const char *relative); std::string dl_storage_native_path(DatalakeFileSystem fs, const char *relative); diff --git a/contrib/datalake_fdw/src/format/arrow_support.cpp b/contrib/datalake_fdw/src/format/arrow_support.cpp index a4e951eeb9f..8ed1622cd6c 100644 --- a/contrib/datalake_fdw/src/format/arrow_support.cpp +++ b/contrib/datalake_fdw/src/format/arrow_support.cpp @@ -39,6 +39,7 @@ #include +#include "common/storage_arrow.h" #include "format/arrow_support.h" extern "C" @@ -56,6 +57,23 @@ DlArrowStatus(const arrow::Status &status, const char *operation) if (status.ok()) return DL_OK; + /* + * A status that came from the storage layer already carries the answer: + * only the backend knows that a 404 from one service and a missing file + * on another are both "not found". Reclassifying it here by status code + * would throw that away. + */ + if (status.detail() != nullptr && + strcmp(status.detail()->type_id(), "datalake::DlStatusDetail") == 0) + { + const DlStatusDetail *detail = + static_cast(status.detail().get()); + + dl_error_set(detail->code(), operation, detail->type().c_str(), + status.message().c_str()); + return detail->code(); + } + switch (status.code()) { case arrow::StatusCode::IOError: @@ -72,13 +90,25 @@ DlArrowStatus(const arrow::Status &status, const char *operation) case arrow::StatusCode::OutOfMemory: code = DL_ERR_OUT_OF_MEMORY; break; + case arrow::StatusCode::AlreadyExists: + code = DL_ERR_ALREADY_EXISTS; + break; default: code = DL_ERR_INTERNAL; break; } - dl_error_set(code, operation, arrow::Status::CodeAsString(status.code()).c_str(), - status.message().c_str()); + /* + * Arrow's own name for a code it does not print is "Unknown", which says + * less than nothing next to a message that already explains itself. + */ + { + std::string type = arrow::Status::CodeAsString(status.code()); + + dl_error_set(code, operation, + type == "Unknown" ? NULL : type.c_str(), + status.message().c_str()); + } return code; } diff --git a/contrib/datalake_fdw/src/format/format.h b/contrib/datalake_fdw/src/format/format.h index 436240aa5f5..d9d052e9259 100644 --- a/contrib/datalake_fdw/src/format/format.h +++ b/contrib/datalake_fdw/src/format/format.h @@ -32,6 +32,8 @@ #include #include +#include "common/file_system_wrapper.h" + #include "common/dl_err.h" /* Arrow C data interface: stable public ABI. */ @@ -72,7 +74,8 @@ struct ArrowArray { */ typedef struct Fragment { - const char *path; + DatalakeFileSystem fs; /* where the file lives; never NULL */ + const char *path; /* relative to that file system's root */ int first_row_group; /* 0-based */ int n_row_groups; /* 0 == to the end of the file */ } Fragment; @@ -170,15 +173,18 @@ struct FormatWriter { const FormatWriterOps *ops; void *impl; }; /* * Bumped when an existing field changes meaning; appending does not need it. * 2: ProjectionSet names field ids rather than positions in the file. + * 3: a fragment and a writer name a file system, and their paths are relative + * to its root, so a format reads and writes wherever the volume is. */ -#define DL_FORMAT_ABI_VERSION 2 +#define DL_FORMAT_ABI_VERSION 3 typedef struct FormatRoutine { uint32_t abi_version, struct_size; /* same prefix-compat semantics as meta engine */ const char *name; /* "parquet" */ DlErrCode (*open_reader)(const Fragment *, const ProjectionSet *, const RowGroupFilterSet *, FormatReader **out); - DlErrCode (*open_writer)(const char *path, /* TupleDesc */ void *tupdesc, + DlErrCode (*open_writer)(DatalakeFileSystem fs, const char *path, + /* TupleDesc */ void *tupdesc, const WriterOptions *, FormatWriter **out); } FormatRoutine; diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_internal.h b/contrib/datalake_fdw/src/format/parquet/parquet_internal.h index f7acbdd2f94..5b7b6e1129b 100644 --- a/contrib/datalake_fdw/src/format/parquet/parquet_internal.h +++ b/contrib/datalake_fdw/src/format/parquet/parquet_internal.h @@ -40,7 +40,8 @@ extern DlErrCode parquet_open_reader(const Fragment *fragment, const RowGroupFilterSet *filters, FormatReader **out); -extern DlErrCode parquet_open_writer(const char *path, void *tupdesc, +extern DlErrCode parquet_open_writer(DatalakeFileSystem fs, const char *path, + void *tupdesc, const WriterOptions *options, FormatWriter **out); diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp index c5432fd2292..72e06bb253f 100644 --- a/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp +++ b/contrib/datalake_fdw/src/format/parquet/parquet_read.cpp @@ -39,6 +39,7 @@ #include #include +#include "common/storage_arrow.h" #include "format/arrow_support.h" #include "am_iceberg/pg_iceberg_guc.h" @@ -297,7 +298,7 @@ parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, return DL_ARG_ERROR("open_reader"); *out = NULL; - if (fragment == NULL || fragment->path == NULL) + if (fragment == NULL || fragment->path == NULL || fragment->fs == NULL) return DL_ARG_ERROR("open_reader"); /* @@ -325,8 +326,8 @@ parquet_open_reader(const Fragment *fragment, const ProjectionSet *projection, parquet::arrow::FileReaderBuilder builder; parquet::ArrowReaderProperties properties; - arrow::Result> file = - arrow::io::ReadableFile::Open(fragment->path, pool); + arrow::Result> file = + dl_storage_open_input(fragment->fs, fragment->path); if (!file.ok()) return DlArrowStatus(file.status(), "open a Parquet file"); diff --git a/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp b/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp index ea60301df97..d9268f93909 100644 --- a/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp +++ b/contrib/datalake_fdw/src/format/parquet/parquet_write.cpp @@ -45,6 +45,7 @@ #include #include +#include "common/storage_arrow.h" #include "format/arrow_support.h" extern "C" @@ -60,9 +61,9 @@ extern "C" struct ParquetWriter { FormatWriter base; - std::string path; + std::string path; /* relative to the mount, for messages */ std::shared_ptr schema; - std::shared_ptr sink; + std::shared_ptr sink; std::unique_ptr writer; /* @@ -98,23 +99,22 @@ class ParquetReleaseBatch }; /* - * Gives up on the file being written. The sink is closed first and the writer - * left to its destructor: a Parquet writer writes the footer when it closes, - * and against a sink that is already closed it cannot -- which is what stops a - * complete, valid, truncated file appearing at the path if the unlink does not - * take. What it leaves then has no footer, so nothing can read it, and that is - * why the unlink's result is not worth reporting. + * Gives up on the file being written. Aborting the sink is what removes it: + * a stream cleans up whatever it created and nothing else, which on a local + * file means unlinking the one it created and on object storage means + * abandoning the upload, leaving no object at all. Deleting by path from here + * would instead reach whatever is at that name by now, which after a failed + * write may belong to another writer entirely. * - * The file is ours to delete: parquet_open_writer() created it with O_EXCL, so - * nothing was at the path before, and nothing this removes was anyone else's. + * The writer itself is left to its destructor. A Parquet writer writes the + * footer when it closes, and against an aborted sink it cannot, so what a + * half-written file leaves behind is unreadable rather than plausible. */ static void parquet_discard(ParquetWriter *impl) { if (impl->sink != nullptr) - (void) impl->sink->Close(); - - (void) unlink(impl->path.c_str()); + (void) impl->sink->Abort(); } /* @@ -387,7 +387,7 @@ parquet_compression(const char *name, arrow::Compression::type *out) } DlErrCode -parquet_open_writer(const char *path, void *tupdesc_arg, +parquet_open_writer(DatalakeFileSystem fs, const char *path, void *tupdesc_arg, const WriterOptions *options, FormatWriter **out) { DlErrCode result = DL_OK; @@ -396,7 +396,7 @@ parquet_open_writer(const char *path, void *tupdesc_arg, return DL_ARG_ERROR("open_writer"); *out = NULL; - if (path == NULL || tupdesc_arg == NULL) + if (fs == NULL || path == NULL || tupdesc_arg == NULL) return DL_ARG_ERROR("open_writer"); DL_ABI_GUARD_BEGIN @@ -406,7 +406,6 @@ parquet_open_writer(const char *path, void *tupdesc_arg, parquet::WriterProperties::Builder properties; arrow::Compression::type compression = arrow::Compression::SNAPPY; DlErrCode rc; - int fd; impl->path = path; impl->schema = DlArrowSchemaFromTupleDesc((TupleDesc) tupdesc_arg, @@ -435,44 +434,17 @@ parquet_open_writer(const char *path, void *tupdesc_arg, impl->pending_rows = 0; /* - * Created here rather than by Arrow. Arrow's path form opens with - * O_TRUNC, which would empty a file that was already there -- and this - * writer deletes the file it holds whenever it cannot finish it, so a - * truncated file would then be a deleted one. With O_EXCL the kernel - * answers "did I create this", and the writer only ever deletes what - * it created. A lake's data file names are unique by construction, so - * a path that exists is a mistake, and refusing it is right anyway. + * The storage layer creates it, and refuses if something is already + * there: a lake's data file names are unique by construction, so a + * path that exists is a mistake rather than something to overwrite. + * The stream that comes back owns what it created, which is what lets + * parquet_discard() give the file up without deleting by path. */ - fd = open(path, O_WRONLY | O_CREAT | O_EXCL, pg_file_create_mode); - if (fd < 0) - { - int saved_errno = errno; - std::string message; - - if (saved_errno == EEXIST) - { - message = std::string("\"") + path + "\" already exists"; - dl_error_set(DL_ERR_ALREADY_EXISTS, "create a Parquet file", NULL, - message.c_str()); - return DL_ERR_ALREADY_EXISTS; - } - - message = std::string("could not create \"") + path + "\": " + - strerror(saved_errno); - dl_error_set(DL_ERR_IO, "create a Parquet file", NULL, message.c_str()); - return DL_ERR_IO; - } - - /* From here the file exists and is ours, so every failure discards it. */ - arrow::Result> sink = - arrow::io::FileOutputStream::Open(fd); + arrow::Result> sink = + dl_storage_open_output(fs, path); if (!sink.ok()) - { - (void) close(fd); /* Arrow took nothing */ - parquet_discard(impl.get()); return DlArrowStatus(sink.status(), "create a Parquet file"); - } impl->sink = *sink; /* diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c index ab5f5c735d0..c21921790ae 100644 --- a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c @@ -33,6 +33,9 @@ #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_user_mapping.h" +#include "commands/dbcommands.h" +#include "miscadmin.h" +#include "utils/acl.h" #include "commands/defrem.h" #include "common/dl_option_util.h" #include "fmgr.h" @@ -153,3 +156,89 @@ iceberg_volume_fdw_validator(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } + +/* + * Everything the storage layer needs to reach one volume: where it is, and + * under what credentials. + * + * The options are read the way the validator reads them, so a volume that was + * accepted at CREATE SERVER resolves here too. A user mapping is optional -- + * without one the backend falls back to whatever credentials the host already + * has, which is how an instance profile or a ticket cache is meant to be used + * -- so this returns an empty credential set rather than refusing. + */ +void +iceberg_volume_resolve(const char *server_name, Oid userid, + DatalakeLocation *location_out, + DlKeyValue **kv_out, int *nkv_out) +{ + ForeignServer *server; + IcebergVolumeOptions *options; + MetaKv *credentials; + DlKeyValue *kv; + char *parse_detail = NULL; + AclResult aclresult; + DlErrCode rc; + int ncredentials = 0; + int nkv = 0; + + Assert(location_out != NULL && kv_out != NULL && nkv_out != NULL); + + server = GetForeignServerByName(server_name, false); + + aclresult = object_aclcheck(ForeignServerRelationId, server->serverid, + userid, ACL_USAGE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, server->servername); + + options = get_iceberg_volume_options(server); + if (options->foreign_volume.base_path == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg volume server option \"%s\" is required", + DATALAKE_ICEBERG_VOLUME_BASE_PATH))); + + rc = pg_iceberg_parse_location(options->foreign_volume.base_path, + options->volume_server.endpoint, + options->volume_server.region, + location_out, &parse_detail); + if (rc != DL_OK) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid iceberg volume %s", + DATALAKE_ICEBERG_VOLUME_BASE_PATH), + errdetail("%s", parse_detail))); + + /* + * The settings the server carries, then the current user's credentials. + * Backends read these by the names they were written under in DDL, so + * nothing in between has to know what any particular protocol wants. + */ + credentials = pg_iceberg_resolve_credentials(server->serverid, userid, + &ncredentials); + kv = (DlKeyValue *) palloc0((3 + ncredentials) * sizeof(DlKeyValue)); + + if (options->volume_server.endpoint != NULL) + { + kv[nkv].key = DATALAKE_ICEBERG_VOLUME_ENDPOINT; + kv[nkv].value = options->volume_server.endpoint; + nkv++; + } + if (options->volume_server.region != NULL) + { + kv[nkv].key = DATALAKE_ICEBERG_VOLUME_REGION; + kv[nkv].value = options->volume_server.region; + nkv++; + } + if (options->volume_server.path_style_access_set) + { + kv[nkv].key = DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS; + kv[nkv].value = options->volume_server.path_style_access ? "true" : "false"; + nkv++; + } + for (int i = 0; i < ncredentials; i++) + kv[nkv++] = credentials[i]; + + *kv_out = kv; + *nkv_out = nkv; +} diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h index 2b4ab16b46c..d1554df2c60 100644 --- a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h @@ -37,6 +37,8 @@ #include "postgres.h" #include "common/dl_option_util.h" +#include "common/datalake_location.h" +#include "common/dl_kv.h" #include "foreign/foreign.h" #include "nodes/pg_list.h" @@ -143,4 +145,14 @@ extern void parse_iceberg_foreign_volume_options(IcebergForeignVolumeOptions *op */ extern IcebergVolumeOptions *get_iceberg_volume_options(ForeignServer *server); +/* + * Resolve a volume by name into the location its data lives at and the + * options a storage backend needs to reach it. Reports through ereport: the + * server has to exist and the user has to be allowed to use it. A user + * mapping is optional, so an empty credential set is a valid answer. + */ +extern void iceberg_volume_resolve(const char *server_name, Oid userid, + DatalakeLocation *location_out, + DlKeyValue **kv_out, int *nkv_out); + #endif /* ICEBERG_VOLUME_OPTION_H */ diff --git a/contrib/datalake_fdw/src/test/datalake_fdw_test.c b/contrib/datalake_fdw/src/test/datalake_fdw_test.c index 3945f4fe894..f6d7504e0a4 100644 --- a/contrib/datalake_fdw/src/test/datalake_fdw_test.c +++ b/contrib/datalake_fdw/src/test/datalake_fdw_test.c @@ -49,6 +49,7 @@ #include "am_iceberg/pg_iceberg_guc.h" #include "am_iceberg/pg_iceberg_options.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" #include "common/dl_err.h" #include "common/file_system_wrapper.h" #include "format/arrow_builder.h" @@ -152,6 +153,48 @@ storage_parse_uri(const char *uri, bool leaf, DatalakeLocation *location, } } +/* + * Mount whatever the caller named. A bare path is a local absolute path -- + * which is what the Parquet cases have always passed -- and a URI names a + * volume's scheme; when a volume is given, its server options and the calling + * user's credentials are what the backend gets. + */ +static DatalakeFileSystem +storage_open_volume(const char *path_or_uri, const char *volume, bool leaf, + char **relative) +{ + DatalakeLocation location; + DatalakeFileSystem fs = NULL; + DlKeyValue *kv = NULL; + char *uri; + int nkv = 0; + DlErrCode rc; + + uri = strstr(path_or_uri, "://") != NULL ? pstrdup(path_or_uri) : + psprintf("file://%s", path_or_uri); + + storage_parse_uri(uri, leaf, &location, relative); + + if (volume != NULL) + { + DatalakeLocation volume_location; + + iceberg_volume_resolve(volume, GetUserId(), &volume_location, &kv, &nkv); + + /* + * The volume says where and how; the URI says which object. Taking + * the endpoint and region from the volume is the point of naming one. + */ + location.endpoint = volume_location.endpoint; + location.region = volume_location.region; + } + + rc = datalake_fs_open(&location, kv, nkv, &fs); + if (rc != DL_OK) + dl_error_report(ERROR, rc, "open storage"); + return fs; +} + static DatalakeFileSystem storage_open_uri(FunctionCallInfo fcinfo, int uri_arg, int kv_arg, bool leaf, char **relative) @@ -247,16 +290,25 @@ datalake_parquet_write(PG_FUNCTION_ARGS) WriterOptions options = {0}; FormatWriter *volatile open_writer = NULL; DlArrowBuilder volatile open_builder = NULL; + DatalakeFileSystem volatile open_fs = NULL; + char *relative; long batch_rows = iceberg_batch_rows; int64 written = 0; MemoryContext row_context; - check_nargs(fcinfo, 4); + check_nargs(fcinfo, 5); + /* Not STRICT, because volume is optional; the rest are not. */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) + PG_RETURN_NULL(); path = text_to_cstring(PG_GETARG_TEXT_PP(0)); query = text_to_cstring(PG_GETARG_TEXT_PP(1)); row_group_size = PG_GETARG_INT32(2); compression = text_to_cstring(PG_GETARG_TEXT_PP(3)); routine = parquet_routine(); + open_fs = storage_open_volume(path, + PG_ARGISNULL(4) ? NULL : + text_to_cstring(PG_GETARG_TEXT_PP(4)), + true, &relative); /* * Bounded above as well as below, and by the same number as @@ -326,7 +378,8 @@ datalake_parquet_write(PG_FUNCTION_ARGS) { tupdesc = CreateTupleDescCopy(SPI_tuptable->tupdesc); - rc = routine->open_writer(path, tupdesc, &options, &writer); + rc = routine->open_writer((DatalakeFileSystem) open_fs, relative, + tupdesc, &options, &writer); if (rc != DL_OK) dl_error_report(ERROR, rc, "open_writer"); open_writer = writer; @@ -406,11 +459,13 @@ datalake_parquet_write(PG_FUNCTION_ARGS) { DlArrowBuilder builder = open_builder; FormatWriter *writer = open_writer; + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; if (builder != NULL) dl_arrow_builder_close(&builder); if (writer != NULL) writer->ops->abort(&writer); + datalake_fs_close(&fs); PG_RE_THROW(); } @@ -418,6 +473,12 @@ datalake_parquet_write(PG_FUNCTION_ARGS) SPI_finish(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + } + PG_RETURN_INT64(written); } @@ -461,15 +522,27 @@ datalake_parquet_read(PG_FUNCTION_ARGS) Datum *values; bool *nulls; FormatReader *reader = NULL; + DatalakeFileSystem volatile open_fs = NULL; + char *relative; MemoryContext row_context; DlErrCode rc; - check_nargs(fcinfo, 4); + check_nargs(fcinfo, 5); + /* Not STRICT, because volume is optional; the rest are not. */ + if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2) || PG_ARGISNULL(3)) + ereport(ERROR, + (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("path, first_row_group, n_row_groups and field_ids are required"))); path = text_to_cstring(PG_GETARG_TEXT_PP(0)); field_id_array = PG_GETARG_ARRAYTYPE_P(3); routine = parquet_routine(); + open_fs = storage_open_volume(path, + PG_ARGISNULL(4) ? NULL : + text_to_cstring(PG_GETARG_TEXT_PP(4)), + true, &relative); - fragment.path = path; + fragment.fs = (DatalakeFileSystem) open_fs; + fragment.path = relative; fragment.first_row_group = PG_GETARG_INT32(1); fragment.n_row_groups = PG_GETARG_INT32(2); @@ -591,6 +664,7 @@ datalake_parquet_read(PG_FUNCTION_ARGS) PG_CATCH(); { FormatReader *failed = open_reader; + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; if (batch->release != NULL) batch->release(batch); @@ -598,11 +672,18 @@ datalake_parquet_read(PG_FUNCTION_ARGS) schema->release(schema); if (failed != NULL) failed->ops->close(&failed); + datalake_fs_close(&fs); PG_RE_THROW(); } PG_END_TRY(); + { + DatalakeFileSystem fs = (DatalakeFileSystem) open_fs; + + datalake_fs_close(&fs); + } + return (Datum) 0; } From 0293d0b3e364ecd781c323bb9b2f02def0738e08 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:26 +0800 Subject: [PATCH 6/9] datalake_fdw: run one body of storage behaviour against every backend Three backends that must behave identically are worth one body of tests, not three that drift. A shared SQL file is parameterised over a URI prefix and a volume and run for file://, for s3:// and for the test backend: the type round trip, field-id projection, a row-group range, listing, a rejected overwrite, and a write that fails halfway leaving nothing behind. Volume resolution gets its own case, on a file volume so it runs everywhere: no USAGE, a PUBLIC mapping, a URI outside its volume, and a volume that does not exist. --- contrib/datalake_fdw/Makefile | 16 ++- .../src/common/file_system_wrapper.cpp | 119 +++++++++++----- .../src/common/s3_file_system.cpp | 11 +- .../datalake_fdw/src/common/storage_arrow.h | 3 + .../expected/conformance_dltest.out | 124 ++++++++++++++++ .../expected/conformance_file.out | 123 ++++++++++++++++ .../expected/conformance_s3.out | 134 ++++++++++++++++++ .../sql/conformance_body.sql | 68 +++++++++ .../sql/conformance_dltest.sql | 22 +++ .../sql/conformance_file.sql | 21 +++ .../sql/conformance_s3.sql | 61 ++++++++ .../smoke/storage_s3/expected/storage_s3.out | 6 +- .../smoke/storage_s3/sql/storage_s3.sql | 6 +- 13 files changed, 669 insertions(+), 45 deletions(-) create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_dltest.sql create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_file.sql create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile index 025396a8811..f8a75bf3290 100644 --- a/contrib/datalake_fdw/Makefile +++ b/contrib/datalake_fdw/Makefile @@ -159,6 +159,10 @@ STORAGE_LOCAL_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_local # The s3 category needs a service to talk to, so it runs where one is named. STORAGE_S3_REGRESS = storage_s3 STORAGE_S3_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_s3 +# One body of storage behaviour, run against each backend that can be reached. +STORAGE_CONFORMANCE_REGRESS = conformance_file conformance_dltest +STORAGE_CONFORMANCE_S3_REGRESS = conformance_s3 +STORAGE_CONFORMANCE_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_conformance EXTRA_CLEAN = exports_darwin.list exports.map @@ -215,7 +219,7 @@ $(shlib): $(EXPORT_LIST) # two runs creating the same extension in the same database race. The # categories are chained through their prerequisites rather than listed side # by side, which is what keeps the order under a parallel make. -installcheck: installcheck-storage-s3 +installcheck: installcheck-storage-conformance installcheck-format-parquet: submake $(REGRESS_PREP) $(pg_regress_installcheck) $(REGRESS_OPTS) \ @@ -239,6 +243,16 @@ endif .PHONY: installcheck-storage-s3 +installcheck-storage-conformance: submake $(REGRESS_PREP) installcheck-storage-s3 + $(pg_regress_installcheck) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_CONFORMANCE_INPUTDIR) $(STORAGE_CONFORMANCE_REGRESS) +ifneq ($(DATALAKE_TEST_S3_ENDPOINT),) + $(pg_regress_installcheck) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_CONFORMANCE_INPUTDIR) $(STORAGE_CONFORMANCE_S3_REGRESS) +endif + +.PHONY: installcheck-storage-conformance + # "make check" is in-tree only -- under PGXS pgxs.mk refuses the target -- and # it is the only run that supplies the temp-config that preloads this module. ifndef USE_PGXS diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp index d30dbf1255e..1fd65460c86 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -175,49 +175,89 @@ contains_token(const std::string &text, const char *token) return false; } -DlErrCode -dl_storage_status_to_err(const arrow::Status &status, const char *operation, - const std::vector *secrets) +/* + * Which DlErrCode a status means, and what to call its class. + * + * A backend that classified the failure itself is believed: only it can know + * that one service's 404 and another's ENOENT are the same answer. For any + * other status this is the fallback, and it reads the whole status rather than + * the message alone -- Arrow puts a failed open's errno in a detail, so the + * words "No such file or directory" are not in the message at all. + */ +static DlErrCode +classify_status(const arrow::Status &status, std::string *type_out) { - DlErrCode code; - std::string type; - std::string message; - - if (status.ok()) - return DL_OK; - if (status.detail() != NULL && strcmp(status.detail()->type_id(), "datalake::DlStatusDetail") == 0) { const DlStatusDetail *detail = static_cast(status.detail().get()); - code = detail->code(); - type = detail->type(); + *type_out = detail->type(); + return detail->code(); } - else if (status.IsNotImplemented()) - code = DL_ERR_NOT_SUPPORTED; - else if (status.IsAlreadyExists()) - code = DL_ERR_ALREADY_EXISTS; - else if (status.IsOutOfMemory()) - code = DL_ERR_OUT_OF_MEMORY; - else if (status.IsIOError()) + + /* + * Arrow prints "Unknown" for the codes it has no name for, AlreadyExists + * among them, and a class of "Unknown" beside a message that explains + * itself is worse than no class at all. + */ + *type_out = status.CodeAsString(); + if (*type_out == "Unknown") + type_out->clear(); + + if (status.IsNotImplemented()) + return DL_ERR_NOT_SUPPORTED; + if (status.IsAlreadyExists()) + return DL_ERR_ALREADY_EXISTS; + if (status.IsOutOfMemory()) + return DL_ERR_OUT_OF_MEMORY; + if (status.IsIOError()) { - message = status.message(); - if (message.find("NoSuchKey") != std::string::npos || - message.find("NoSuchBucket") != std::string::npos || - message.find("does not exist") != std::string::npos || - message.find("No such file or directory") != std::string::npos || - contains_token(message, "404")) - code = DL_ERR_NOT_FOUND; - else - code = DL_ERR_IO; + const std::string whole = status.ToString(); + + if (whole.find("NoSuchKey") != std::string::npos || + whole.find("NoSuchBucket") != std::string::npos || + whole.find("does not exist") != std::string::npos || + whole.find("No such file or directory") != std::string::npos || + contains_token(whole, "404")) + return DL_ERR_NOT_FOUND; } - else - code = DL_ERR_IO; + return DL_ERR_IO; +} - if (type.empty()) - type = status.CodeAsString(); +/* + * The same status, carrying its classification, for a caller that hands it on + * rather than reporting it here -- the format layer, which opens files through + * the storage layer but reports errors in its own terms. + */ +arrow::Status +dl_storage_classify(arrow::Status status) +{ + std::string type; + + if (status.ok() || (status.detail() != NULL && + strcmp(status.detail()->type_id(), + "datalake::DlStatusDetail") == 0)) + return status; + + DlErrCode code = classify_status(status, &type); + + return status.WithDetail(std::make_shared(code, type)); +} + +DlErrCode +dl_storage_status_to_err(const arrow::Status &status, const char *operation, + const std::vector *secrets) +{ + DlErrCode code; + std::string type; + std::string message; + + if (status.ok()) + return DL_OK; + + code = classify_status(status, &type); if (message.empty()) message = status.message(); @@ -489,7 +529,11 @@ dl_storage_open_input(DatalakeFileSystem fs, const char *relative) return arrow::Status::Invalid("storage path \"", relative, "\" must be relative to the volume and " "must not contain \".\" or \"..\""); - return fs->fs->OpenInputFile(dl_storage_native_path(fs, relative)); + auto file = fs->fs->OpenInputFile(dl_storage_native_path(fs, relative)); + + if (!file.ok()) + return dl_storage_classify(file.status()); + return file; } arrow::Result> @@ -504,8 +548,13 @@ dl_storage_open_output(DatalakeFileSystem fs, const char *relative) const std::string native = dl_storage_native_path(fs, relative); - ARROW_RETURN_NOT_OK(require_absent(fs, native)); - return fs->fs->OpenOutputStream(native); + ARROW_RETURN_NOT_OK(dl_storage_classify(require_absent(fs, native))); + + auto stream = fs->fs->OpenOutputStream(native); + + if (!stream.ok()) + return dl_storage_classify(stream.status()); + return stream; } extern "C" DlErrCode diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp index 3f9a39e2ccd..c43bc2f34fa 100644 --- a/contrib/datalake_fdw/src/common/s3_file_system.cpp +++ b/contrib/datalake_fdw/src/common/s3_file_system.cpp @@ -865,9 +865,14 @@ mount_s3(const DatalakeLocation *location, const DlKeyValue *kv, int nkv, const char *endpoint = option_value(kv, nkv, "endpoint"); const char *region = option_value(kv, nkv, "region"); const char *path_style = option_value(kv, nkv, "path_style_access"); - const char *access_key = option_value(kv, nkv, "aws_access_key_id"); - const char *secret_key = option_value(kv, nkv, "aws_secret_access_key"); - const char *session_token = option_value(kv, nkv, "aws_session_token"); + /* + * Named as the DDL names them: these arrive as the options of a user + * mapping, and a backend reading them under some other spelling would be + * a second vocabulary for one thing. + */ + const char *access_key = option_value(kv, nkv, "access_key_id"); + const char *secret_key = option_value(kv, nkv, "secret_access_key"); + const char *session_token = option_value(kv, nkv, "session_token"); if (endpoint == NULL) endpoint = location->endpoint; diff --git a/contrib/datalake_fdw/src/common/storage_arrow.h b/contrib/datalake_fdw/src/common/storage_arrow.h index 2c4b16e4408..3fe8998495b 100644 --- a/contrib/datalake_fdw/src/common/storage_arrow.h +++ b/contrib/datalake_fdw/src/common/storage_arrow.h @@ -89,4 +89,7 @@ DlErrCode dl_storage_status_to_err(const arrow::Status &status, */ bool dl_storage_path_is_safe(const char *relative); +/* The same status with its DlErrCode attached, for a caller that hands it on. */ +arrow::Status dl_storage_classify(arrow::Status status); + #endif /* STORAGE_ARROW_H */ diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out new file mode 100644 index 00000000000..4a283463e9e --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out @@ -0,0 +1,124 @@ +-- A backend registered from outside the module through the public contract, +-- mounted under a subtree. Its paths are relative rather than absolute, which +-- is exactly the difference a backend is allowed to have. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_conformance_dltest && mkdir -p /tmp/datalake_fdw_conformance_dltest'; +\set prefix 'dltest:///tmp/datalake_fdw_conformance_dltest' +\set root '' +\set volume NULL +\set kv NULL +-- pg_regress feeds the script to psql on standard input, so there is no +-- script directory for \ir to resolve against; the path is relative to where +-- make runs, which is the module's own directory. +\i test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql +-- The storage behaviour every backend owes the format layer, written once and +-- run against each of them. Included by conformance_file, conformance_dltest +-- and conformance_s3, which differ only in where :prefix points, whether a +-- :volume supplies credentials, and the :kv the text helpers need. +-- +-- What is asserted here is what changes with the storage underneath: that a +-- Parquet file written through a volume reads back, that a name in use is +-- never overwritten, and that a write which fails partway leaves nothing. +-- What a Parquet file holds is the business of the format_parquet cases. +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, + (i % 2 = 0) AS flag + FROM generate_series(1, 2000) i$q$, + 500, 'snappy', :volume) AS rows_written; + rows_written +-------------- + 2000 +(1 row) + +SELECT count(*) AS rows_read, + sum(id) AS id_sum, + min(label) AS first_label, + count(*) FILTER (WHERE flag) AS flagged +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_read | id_sum | first_label | flagged +-----------+---------+-------------+--------- + 2000 | 2001000 | row 1 | 1000 +(1 row) + +-- Written with 500-row groups, so a range of them is a range of the file. +SELECT count(*) AS rows_in_two_groups +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_in_two_groups +-------------------- + 1000 +(1 row) + +-- The file is there, and it is the only thing there. +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + object +------------------- + roundtrip.parquet +(1 row) + +-- A name in use is refused, and what was there is untouched. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + 'SELECT 1 AS id', 0, '', :volume); +ERROR: 42P07 +\set VERBOSITY default +SELECT count(*) AS rows_still_there +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_still_there +------------------ + 2000 +(1 row) + +-- A write that fails partway leaves nothing behind: not a truncated file, not +-- an empty one, and nothing for the next attempt at that name to trip over. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, + 500, '', :volume); +ERROR: 22012 +\set VERBOSITY default +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + object +------------------- + roundtrip.parquet +(1 row) + +-- And the name is free, so the next writer gets it. +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + 'SELECT 7 AS id', 0, '', :volume) AS rows_written; + rows_written +-------------- + 1 +(1 row) + +SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}', + :volume) AS t(id int); + id +---- + 7 +(1 row) + +-- Reading something that is not there says so, whatever the storage is. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, + '{}', :volume) AS t(id int); +ERROR: 42704 +\set VERBOSITY default +SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, + datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; + cleaned_roundtrip | cleaned_aborted +-------------------+----------------- + t | t +(1 row) + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_conformance_dltest'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out new file mode 100644 index 00000000000..de68c2716aa --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out @@ -0,0 +1,123 @@ +-- The built-in file backend: a shared mount, which is what a volume on a +-- cluster filesystem looks like to this layer. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_conformance_file && mkdir -p /tmp/datalake_fdw_conformance_file'; +\set prefix 'file:///tmp/datalake_fdw_conformance_file' +\set root '/tmp/datalake_fdw_conformance_file/' +\set volume NULL +\set kv NULL +-- pg_regress feeds the script to psql on standard input, so there is no +-- script directory for \ir to resolve against; the path is relative to where +-- make runs, which is the module's own directory. +\i test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql +-- The storage behaviour every backend owes the format layer, written once and +-- run against each of them. Included by conformance_file, conformance_dltest +-- and conformance_s3, which differ only in where :prefix points, whether a +-- :volume supplies credentials, and the :kv the text helpers need. +-- +-- What is asserted here is what changes with the storage underneath: that a +-- Parquet file written through a volume reads back, that a name in use is +-- never overwritten, and that a write which fails partway leaves nothing. +-- What a Parquet file holds is the business of the format_parquet cases. +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, + (i % 2 = 0) AS flag + FROM generate_series(1, 2000) i$q$, + 500, 'snappy', :volume) AS rows_written; + rows_written +-------------- + 2000 +(1 row) + +SELECT count(*) AS rows_read, + sum(id) AS id_sum, + min(label) AS first_label, + count(*) FILTER (WHERE flag) AS flagged +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_read | id_sum | first_label | flagged +-----------+---------+-------------+--------- + 2000 | 2001000 | row 1 | 1000 +(1 row) + +-- Written with 500-row groups, so a range of them is a range of the file. +SELECT count(*) AS rows_in_two_groups +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_in_two_groups +-------------------- + 1000 +(1 row) + +-- The file is there, and it is the only thing there. +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + object +------------------- + roundtrip.parquet +(1 row) + +-- A name in use is refused, and what was there is untouched. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + 'SELECT 1 AS id', 0, '', :volume); +ERROR: 42P07 +\set VERBOSITY default +SELECT count(*) AS rows_still_there +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_still_there +------------------ + 2000 +(1 row) + +-- A write that fails partway leaves nothing behind: not a truncated file, not +-- an empty one, and nothing for the next attempt at that name to trip over. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, + 500, '', :volume); +ERROR: 22012 +\set VERBOSITY default +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + object +------------------- + roundtrip.parquet +(1 row) + +-- And the name is free, so the next writer gets it. +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + 'SELECT 7 AS id', 0, '', :volume) AS rows_written; + rows_written +-------------- + 1 +(1 row) + +SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}', + :volume) AS t(id int); + id +---- + 7 +(1 row) + +-- Reading something that is not there says so, whatever the storage is. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, + '{}', :volume) AS t(id int); +ERROR: 42704 +\set VERBOSITY default +SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, + datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; + cleaned_roundtrip | cleaned_aborted +-------------------+----------------- + t | t +(1 row) + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_conformance_file'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out new file mode 100644 index 00000000000..6f9ff84b106 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out @@ -0,0 +1,134 @@ +-- The s3 backend, reached the way a user reaches it: through a volume server +-- and a user mapping, rather than by handing credentials to a function. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +SET log_statement = 'none'; +SET log_min_duration_statement = -1; +SET log_min_error_statement = 'panic'; +-- Endpoint, bucket and credentials differ from one machine to the next, so +-- they are read with the echo off: what this file asserts must not depend on +-- where it ran. +\set ECHO none +NOTICE: server "dlconf_volume" does not exist, skipping +-- The volume exists and names this run's prefix. +SELECT count(*) = 1 AS volume_created +FROM pg_foreign_server WHERE srvname = 'dlconf_volume'; + volume_created +---------------- + t +(1 row) + +-- pg_regress feeds the script to psql on standard input, so there is no +-- script directory for \ir to resolve against; the path is relative to where +-- make runs, which is the module's own directory. +\i test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql +-- The storage behaviour every backend owes the format layer, written once and +-- run against each of them. Included by conformance_file, conformance_dltest +-- and conformance_s3, which differ only in where :prefix points, whether a +-- :volume supplies credentials, and the :kv the text helpers need. +-- +-- What is asserted here is what changes with the storage underneath: that a +-- Parquet file written through a volume reads back, that a name in use is +-- never overwritten, and that a write which fails partway leaves nothing. +-- What a Parquet file holds is the business of the format_parquet cases. +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, + (i % 2 = 0) AS flag + FROM generate_series(1, 2000) i$q$, + 500, 'snappy', :volume) AS rows_written; + rows_written +-------------- + 2000 +(1 row) + +SELECT count(*) AS rows_read, + sum(id) AS id_sum, + min(label) AS first_label, + count(*) FILTER (WHERE flag) AS flagged +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_read | id_sum | first_label | flagged +-----------+---------+-------------+--------- + 2000 | 2001000 | row 1 | 1000 +(1 row) + +-- Written with 500-row groups, so a range of them is a range of the file. +SELECT count(*) AS rows_in_two_groups +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_in_two_groups +-------------------- + 1000 +(1 row) + +-- The file is there, and it is the only thing there. +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + object +------------------- + roundtrip.parquet +(1 row) + +-- A name in use is refused, and what was there is untouched. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + 'SELECT 1 AS id', 0, '', :volume); +ERROR: 42P07 +\set VERBOSITY default +SELECT count(*) AS rows_still_there +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + rows_still_there +------------------ + 2000 +(1 row) + +-- A write that fails partway leaves nothing behind: not a truncated file, not +-- an empty one, and nothing for the next attempt at that name to trip over. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, + 500, '', :volume); +ERROR: 22012 +\set VERBOSITY default +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + object +------------------- + roundtrip.parquet +(1 row) + +-- And the name is free, so the next writer gets it. +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + 'SELECT 7 AS id', 0, '', :volume) AS rows_written; + rows_written +-------------- + 1 +(1 row) + +SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}', + :volume) AS t(id int); + id +---- + 7 +(1 row) + +-- Reading something that is not there says so, whatever the storage is. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, + '{}', :volume) AS t(id int); +ERROR: 42704 +\set VERBOSITY default +SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, + datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; + cleaned_roundtrip | cleaned_aborted +-------------------+----------------- + t | t +(1 row) + +\set ECHO none +NOTICE: drop cascades to user mapping for gpadmin on server dlconf_volume diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql new file mode 100644 index 00000000000..e6c3fded417 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql @@ -0,0 +1,68 @@ +-- The storage behaviour every backend owes the format layer, written once and +-- run against each of them. Included by conformance_file, conformance_dltest +-- and conformance_s3, which differ only in where :prefix points, whether a +-- :volume supplies credentials, and the :kv the text helpers need. +-- +-- What is asserted here is what changes with the storage underneath: that a +-- Parquet file written through a volume reads back, that a name in use is +-- never overwritten, and that a write which fails partway leaves nothing. +-- What a Parquet file holds is the business of the format_parquet cases. + +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, + (i % 2 = 0) AS flag + FROM generate_series(1, 2000) i$q$, + 500, 'snappy', :volume) AS rows_written; + +SELECT count(*) AS rows_read, + sum(id) AS id_sum, + min(label) AS first_label, + count(*) FILTER (WHERE flag) AS flagged +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + +-- Written with 500-row groups, so a range of them is a range of the file. +SELECT count(*) AS rows_in_two_groups +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + +-- The file is there, and it is the only thing there. +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + +-- A name in use is refused, and what was there is untouched. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', + 'SELECT 1 AS id', 0, '', :volume); +\set VERBOSITY default +SELECT count(*) AS rows_still_there +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) + AS t(id int, amount float8, label text, flag boolean); + +-- A write that fails partway leaves nothing behind: not a truncated file, not +-- an empty one, and nothing for the next attempt at that name to trip over. +\set VERBOSITY sqlstate +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, + 500, '', :volume); +\set VERBOSITY default + +SELECT replace(path, :'root', '') AS object +FROM datalake_storage_list(:'prefix', :kv) AS path +ORDER BY 1; + +-- And the name is free, so the next writer gets it. +SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', + 'SELECT 7 AS id', 0, '', :volume) AS rows_written; +SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}', + :volume) AS t(id int); + +-- Reading something that is not there says so, whatever the storage is. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, + '{}', :volume) AS t(id int); +\set VERBOSITY default + +SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, + datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_dltest.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_dltest.sql new file mode 100644 index 00000000000..eca482cc747 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_dltest.sql @@ -0,0 +1,22 @@ +-- A backend registered from outside the module through the public contract, +-- mounted under a subtree. Its paths are relative rather than absolute, which +-- is exactly the difference a backend is allowed to have. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_conformance_dltest && mkdir -p /tmp/datalake_fdw_conformance_dltest'; + +\set prefix 'dltest:///tmp/datalake_fdw_conformance_dltest' +\set root '' +\set volume NULL +\set kv NULL + +-- pg_regress feeds the script to psql on standard input, so there is no +-- script directory for \ir to resolve against; the path is relative to where +-- make runs, which is the module's own directory. +\i test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_conformance_dltest'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_file.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_file.sql new file mode 100644 index 00000000000..71996426e29 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_file.sql @@ -0,0 +1,21 @@ +-- The built-in file backend: a shared mount, which is what a volume on a +-- cluster filesystem looks like to this layer. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_conformance_file && mkdir -p /tmp/datalake_fdw_conformance_file'; + +\set prefix 'file:///tmp/datalake_fdw_conformance_file' +\set root '/tmp/datalake_fdw_conformance_file/' +\set volume NULL +\set kv NULL + +-- pg_regress feeds the script to psql on standard input, so there is no +-- script directory for \ir to resolve against; the path is relative to where +-- make runs, which is the module's own directory. +\i test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql + +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_conformance_file'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql new file mode 100644 index 00000000000..9cef616001b --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql @@ -0,0 +1,61 @@ +-- The s3 backend, reached the way a user reaches it: through a volume server +-- and a user mapping, rather than by handing credentials to a function. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +SET log_statement = 'none'; +SET log_min_duration_statement = -1; +SET log_min_error_statement = 'panic'; + +-- Endpoint, bucket and credentials differ from one machine to the next, so +-- they are read with the echo off: what this file asserts must not depend on +-- where it ran. +\set ECHO none +\getenv endpoint DATALAKE_TEST_S3_ENDPOINT +\getenv bucket DATALAKE_TEST_S3_BUCKET +\getenv access_key DATALAKE_TEST_S3_ACCESS_KEY +\getenv secret_key DATALAKE_TEST_S3_SECRET_KEY +\getenv region DATALAKE_TEST_S3_REGION +\getenv path_style DATALAKE_TEST_S3_PATH_STYLE + +SELECT to_char(clock_timestamp(), 'YYYYMMDDHH24MISSMS') || '_' || + pg_backend_pid() AS run \gset + +-- Server options take literals, so the values are assembled first and +-- interpolated as literals afterwards. +SELECT format('s3://%s/datalake_conformance/%s', :'bucket', :'run') AS base_path, + format('s3://%s/datalake_conformance/%s', :'bucket', :'run') AS prefix, + format('%s/datalake_conformance/%s/', :'bucket', :'run') AS root, + format('ARRAY[%L,%L,%L,%L,%L]', + 'endpoint=' || :'endpoint', 'region=' || :'region', + 'path_style_access=' || coalesce(nullif(:'path_style', ''), 'true'), + 'access_key_id=' || :'access_key', + 'secret_access_key=' || :'secret_key') AS kv \gset + +DROP SERVER IF EXISTS dlconf_volume CASCADE; +CREATE SERVER dlconf_volume FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS ( + base_path :'base_path', + endpoint :'endpoint', + region :'region', + path_style_access :'path_style'); +CREATE USER MAPPING FOR CURRENT_USER SERVER dlconf_volume OPTIONS ( + access_key_id :'access_key', + secret_access_key :'secret_key'); + +\set volume '''dlconf_volume''' +\set ECHO all + +-- The volume exists and names this run's prefix. +SELECT count(*) = 1 AS volume_created +FROM pg_foreign_server WHERE srvname = 'dlconf_volume'; + +-- pg_regress feeds the script to psql on standard input, so there is no +-- script directory for \ir to resolve against; the path is relative to where +-- make runs, which is the module's own directory. +\i test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql + +\set ECHO none +DROP SERVER dlconf_volume CASCADE; +\set ECHO all diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out index 7f0dcd06681..479d0318d30 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out @@ -29,8 +29,8 @@ CREATE FUNCTION s3_kv(secret text DEFAULT NULL) RETURNS text[] LANGUAGE sql AS $ SELECT ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), 'region=' || current_setting('datalake.s3_region'), 'path_style_access=' || current_setting('datalake.s3_path_style'), - 'aws_access_key_id=' || current_setting('datalake.s3_access_key'), - 'aws_secret_access_key=' || + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=' || coalesce(secret, current_setting('datalake.s3_secret'))] $$; CREATE FUNCTION s3_prefix() RETURNS text LANGUAGE sql AS $$ @@ -137,7 +137,7 @@ SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret') SELECT datalake_storage_read_text(s3_uri('a.txt'), ARRAY['endpoint=http://10.255.255.1:9000', 'region=us-east-1', 'path_style_access=true', - 'aws_access_key_id=x', 'aws_secret_access_key=y']); + 'access_key_id=x', 'secret_access_key=y']); ERROR: 58030 \set VERBOSITY default -- The session still works afterwards. diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql index 3a325e3d90f..d18feaf5844 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql @@ -55,8 +55,8 @@ CREATE FUNCTION s3_kv(secret text DEFAULT NULL) RETURNS text[] LANGUAGE sql AS $ SELECT ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), 'region=' || current_setting('datalake.s3_region'), 'path_style_access=' || current_setting('datalake.s3_path_style'), - 'aws_access_key_id=' || current_setting('datalake.s3_access_key'), - 'aws_secret_access_key=' || + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=' || coalesce(secret, current_setting('datalake.s3_secret'))] $$; CREATE FUNCTION s3_prefix() RETURNS text LANGUAGE sql AS $$ @@ -131,7 +131,7 @@ SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret') SELECT datalake_storage_read_text(s3_uri('a.txt'), ARRAY['endpoint=http://10.255.255.1:9000', 'region=us-east-1', 'path_style_access=true', - 'aws_access_key_id=x', 'aws_secret_access_key=y']); + 'access_key_id=x', 'secret_access_key=y']); \set VERBOSITY default -- The session still works afterwards. From efaa71d28e11bc09640a914528f99b44dc864fed Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:26 +0800 Subject: [PATCH 7/9] datalake_fdw: document the storage layer Volumes and their options, where credentials come from and in what order, what a file:// volume requires of the filesystem, how to build with S3 support, and how to write a backend -- the example is compiled against the installed headers rather than written out by hand. Plus the limits that are accepted rather than fixed. --- contrib/datalake_fdw/README.md | 208 +++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 contrib/datalake_fdw/README.md diff --git a/contrib/datalake_fdw/README.md b/contrib/datalake_fdw/README.md new file mode 100644 index 00000000000..ade1f8462d0 --- /dev/null +++ b/contrib/datalake_fdw/README.md @@ -0,0 +1,208 @@ + + +# datalake_fdw + +Apache Iceberg lake tables as a Cloudberry extension. This document covers +the storage layer: where a lake table's files live, how the extension is told +to reach them, and how to add a kind of storage it does not know about. + +## Volumes + +A volume is a foreign server that says where data lives and how to get in. +Every path the extension reads or writes belongs to one. + +```sql +CREATE SERVER warehouse + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://analytics/warehouse', + endpoint 'https://s3.eu-central-1.amazonaws.com', + region 'eu-central-1', + path_style_access 'false'); + +CREATE USER MAPPING FOR analyst SERVER warehouse + OPTIONS (access_key_id 'AKIA...', secret_access_key '...'); +``` + +Server options: + +| Option | Meaning | +|---|---| +| `base_path` | Required. A URI: `s3://bucket/prefix` or `file:///mnt/warehouse`. Its scheme selects the storage backend, so there is no separate option naming the protocol. | +| `endpoint` | The service to talk to, when it is not AWS. `http://` selects plain HTTP; anything else is HTTPS. | +| `region` | Defaults to `us-east-1`. | +| `path_style_access` | `true` addresses a bucket as a path, `false` as a host name. Unset means path style when an `endpoint` is given and host style when it is not, which is what AWS and everything else respectively want. | + +User mapping options, all optional: `access_key_id`, `secret_access_key`, +`session_token`, `username`. + +### Where credentials come from + +In this order, and the first that applies wins: + +1. the user mapping for the querying user; +2. the `PUBLIC` user mapping for that server; +3. whatever the host already grants -- environment variables, an instance + profile, a shared credentials file, a web identity token. + +A volume with no user mapping is therefore not a misconfiguration: it is how +an instance profile is meant to be used. Where none of the three yields a +credential, the request is made unsigned and the service refuses it. + +On a host that is not on EC2, the default chain will try the instance metadata +service and wait for it to time out. Set `AWS_EC2_METADATA_DISABLED=true` in +the server's environment to skip that. + +### `file://` volumes + +A `file://` volume is a directory, and every segment reads and writes it +directly. It is only correct if that directory is the *same* directory on +every host -- a network filesystem, or a cluster filesystem. A path that +happens to exist on each host separately will produce a table whose files +exist in several places and nowhere in full. Nothing checks this; it is +yours to arrange. + +## Building with S3 support + +S3 needs the AWS SDK for C++. No distribution packages it, so unless you are +using an image that already has it, build it once: + +```sh +git clone --depth 1 --branch 1.11.844 --recurse-submodules --shallow-submodules \ + https://github.com/aws/aws-sdk-cpp.git +cmake -S aws-sdk-cpp -B aws-sdk-cpp/build -GNinja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/datalake \ + -DBUILD_ONLY="s3;sts" -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DENABLE_TESTING=OFF -DUSE_OPENSSL=ON +ninja -C aws-sdk-cpp/build install +``` + +It needs libcurl, OpenSSL and zlib headers, and takes well under a minute to +compile; the clone is the slow part. 1.11.844 is the version this extension +is tested against, and 1.11 is the minimum. + +The extension finds it under `/usr/local` or `/opt/datalake`, or wherever +`AWS_SDK_PREFIX` says: + +```sh +make USE_PGXS=1 AWS_SDK_PREFIX=/opt/datalake install +``` + +Without the SDK the extension still builds -- the build says so -- and opening +an `s3://` location then fails with a message telling you to rebuild. Naming +a prefix that has no SDK in it is an error rather than a silent fallback. + +Arrow itself needs no S3 support: this extension does not use Arrow's S3 +filesystem, which is just as well, because no RPM of Arrow is built with it. + +## Writing a storage backend + +A backend is a shared library that answers one question: given a location and +its options, which `arrow::fs::FileSystem` reads and writes it. Everything +else -- opening files, listing, error classification, memory accounting -- is +the extension's side of the boundary. + +`make install` puts five headers under +`$(pg_config --includedir-server)/extension/datalake_fdw/`, and they are all a +backend includes: + +```cpp +#include + +#include "storage_backend.h" +#include "storage_backend_register.h" + +extern "C" { PG_MODULE_MAGIC; void _PG_init(void); } + +static arrow::Result +mount_mine(const DatalakeLocation *location, const DlKeyValue *kv, int nkv, + const DatalakeStorageHost *host) +{ + DatalakeMountedFs mounted; + + mounted.fs = std::make_shared( + arrow::io::IOContext(dl_storage_host_pool(host))); + mounted.root = location->path_prefix; + return mounted; +} + +static const DatalakeStorageBackend mine = { + DL_STORAGE_ABI_VERSION, sizeof(DatalakeStorageBackend), "mine", + ARROW_VERSION_STRING, DL_STORAGE_ABI_FINGERPRINT, mount_mine, NULL, NULL +}; + +void +_PG_init(void) +{ + DlErrCode rc = datalake_storage_register(&mine); + + if (rc != DL_OK) + elog(ERROR, "could not register the \"mine\" storage backend"); +} +``` + +Put the library in `shared_preload_libraries`. The order does not matter: +registering pulls in `datalake_fdw` if it is not loaded yet. Registration +only happens during preload, so that every backend process agrees on which +schemes exist. + +What the extension calls on the filesystem it gets back, and nothing else: + +* `GetFileInfo(path)` and `GetFileInfo(FileSelector)` +* `OpenInputFile`, then `GetSize`, `ReadAt` and `Read` +* `OpenOutputStream`, then `Write`, `Close` and `Abort` +* `DeleteFile` + +Anything else may return `NotImplemented`. + +Three obligations are worth stating, because a backend that gets them wrong +is wrong in ways tests elsewhere will not catch: + +* **`Abort()` removes what this stream created, and only that.** Nothing else + deletes on a stream's behalf, because after a failed write the name may + already belong to another writer. If the stream never created anything -- + an upload that was never started -- abort does nothing. +* **Allocate through `host->pool`.** Arrow's default pool is invisible to + Cloudberry's memory accounting, and a query that allocates outside it is a + query whose memory limit does not apply. +* **Classify failures.** Return `arrow::Status::AlreadyExists` for a name in + use, and an `IOError` whose text says so for something missing (the + extension recognises `NoSuchKey`, `NoSuchBucket`, `does not exist`, + `No such file or directory` and a bare `404`); anything else is reported as + an I/O error. + +`abi_fingerprint` is checked at registration: the compiler's major version, +libstdc++'s dual-ABI setting and the Arrow version have to match the ones the +extension was built with, because `std::shared_ptr` and `arrow::Result` cross +the boundary by value. Build a backend with the same toolchain and Arrow +package as the extension. + +## Known limits + +* The fingerprint catches the mismatches that occur in practice, not every + possible one. A backend built against a different C++ runtime can still + register and then misbehave. +* Registering from inside a backend process instead of during preload affects + only that process, and is not supported. +* Between checking that a name is free and creating it, another writer can + take it. Iceberg's file names are unique by construction, so this does not + arise there. +* An upload abandoned by a crashed backend leaves its parts behind. A bucket + lifecycle rule that expires incomplete multipart uploads is the usual answer. From b76f2fba66ad9b47c7366ab01cc0c25f17e5596a Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:52 +0800 Subject: [PATCH 8/9] datalake_fdw: close what two review rounds and a manual audit found Two independent adversarial review rounds, then a read of the whole diff against this change's own invariants. The defects worth naming: - A multipart upload was abandoned rather than aborted. Close() marked the stream closed before uploading the last part, so the Abort() that followed a failure did nothing and the parts stayed, billable. Cleanup now depends on the upload being live, and the destructor does the same. - A local file could outlive the writer that created it: the guard was released one statement before the wrapper took ownership, and the wrapper had no destructor, so an exception rather than an error code left the file behind. - Two paths skipped credential scrubbing -- an unexpected exception's what(), and the format layer's own status mapping. Scrubbing moved to dl_error_set, which every error goes through. - A bare "404" was matched inside the whole status text, so an error about a directory named 404 read as not-found. Only phrases a filesystem writes about itself are matched now, and the README said three others that never were. - ReadAt trusted the length the service returned; a paginated listing whose continuation token stopped advancing looped forever; half a credential pair silently fell back to the host's own identity; a bucket that does not exist looked like a directory. - The validator quoted a rejected base_path back verbatim in its message while the detail beside it had just stopped doing so, so s3://user:password@host/path wrote the password to the server log. And in the tests, which were also under review: - A loop of the shape "SELECT count(*) FROM (SELECT f(i) FROM generate_series(...)) w" never called f at all -- the target list is unreferenced, so it is optimised away while the count still looks right. The pagination case wrote nothing for 1100 rows and passed. Every such loop now reads the value the function returned. - The redaction case asserted that a message did not contain a secret the service never echoes, so it passed with redaction removed entirely. It uses the bucket name as the secret now. - The paginated listing case had no expected output at all, so the one way CI is meant to run it could only fail. - Nothing checked that an empty prefix reads as not found, which is a rule this change introduced. Measured while auditing: reading and writing through s3:// costs a backend about 15 MiB of resident memory more than the same file locally, and that stays flat as the object grows -- 7.9 MiB for a 2 MB file, 14.8 MiB for a 149 MB one -- so neither the response body nor the part buffer accumulates. It is outside gp_vmem_protect_limit, which the README now says. --- contrib/datalake_fdw/Makefile | 34 +++- contrib/datalake_fdw/README.md | 17 +- .../src/am_iceberg/pg_iceberg_options.c | 36 ++-- .../src/am_iceberg/pg_iceberg_options.h | 7 + .../src/common/backend_registry.cpp | 17 ++ .../src/common/backend_registry.h | 7 + contrib/datalake_fdw/src/common/dl_err.c | 67 +++++++ contrib/datalake_fdw/src/common/dl_err.h | 8 + .../src/common/file_system_wrapper.cpp | 166 +++++++----------- .../src/common/local_file_system.cpp | 21 ++- .../src/common/s3_file_system.cpp | 106 +++++++++-- .../datalake_fdw/src/common/storage_arrow.h | 16 +- .../src/common/storage_backend_register.h | 37 +++- .../iceberg_volume_fdw/iceberg_volume_fdw.c | 2 +- .../datalake_fdw/src/test/datalake_fdw_test.c | 51 +++--- .../src/test/storage_test_backend.cpp | 30 +++- .../iceberg_am/expected/iceberg_am_reject.out | 6 +- .../expected/conformance_dltest.out | 67 +++++-- .../expected/conformance_file.out | 67 +++++-- .../expected/conformance_s3.out | 67 +++++-- .../expected/volume_resolve.out | 118 +++++++++++++ .../sql/conformance_body.sql | 51 ++++-- .../sql/conformance_s3.sql | 12 +- .../sql/volume_resolve.sql | 89 ++++++++++ .../smoke/storage_s3/expected/storage_s3.out | 86 +++++++-- .../expected/storage_s3_pagination.out | 58 ++++++ .../smoke/storage_s3/sql/storage_s3.sql | 79 +++++++-- .../storage_s3/sql/storage_s3_pagination.sql | 70 ++++++++ 28 files changed, 1112 insertions(+), 280 deletions(-) create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/volume_resolve.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/volume_resolve.sql create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3_pagination.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3_pagination.sql diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile index f8a75bf3290..567e4b1a602 100644 --- a/contrib/datalake_fdw/Makefile +++ b/contrib/datalake_fdw/Makefile @@ -111,13 +111,18 @@ endif ifneq ($(AWS_SDK_PREFIX),) AWS_SDK_CPPFLAGS = -DDL_HAVE_AWS_SDK -I$(AWS_SDK_PREFIX)/include # One group, because these static archives refer to each other both ways. +# -Bstatic, because a prefix that also holds shared SDK libraries would +# otherwise bind those, and this module promises to add no runtime dependency +# beyond the system libraries the SDK itself needs. AWS_SDK_LIBS = -L$(AWS_SDK_PREFIX)/lib64 -L$(AWS_SDK_PREFIX)/lib \ + -Wl,-Bstatic \ -Wl,--start-group \ -laws-cpp-sdk-s3 -laws-cpp-sdk-core \ -laws-crt-cpp -laws-c-s3 -laws-c-auth -laws-c-http -laws-c-io \ -laws-c-cal -laws-c-compression -laws-c-mqtt -laws-c-event-stream \ -laws-c-sdkutils -laws-c-common -laws-checksums -ls2n \ -Wl,--end-group \ + -Wl,-Bdynamic \ -lssl -lcrypto -lz -lcurl else AWS_SDK_CPPFLAGS = @@ -158,9 +163,14 @@ STORAGE_LOCAL_REGRESS = storage_local STORAGE_LOCAL_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_local # The s3 category needs a service to talk to, so it runs where one is named. STORAGE_S3_REGRESS = storage_s3 +# Thousands of requests, so it runs where someone asked for it rather than on +# every build; CI does, a local gate does not. +ifneq ($(DATALAKE_TEST_S3_PAGINATION),) +STORAGE_S3_REGRESS += storage_s3_pagination +endif STORAGE_S3_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_s3 # One body of storage behaviour, run against each backend that can be reached. -STORAGE_CONFORMANCE_REGRESS = conformance_file conformance_dltest +STORAGE_CONFORMANCE_REGRESS = conformance_file conformance_dltest volume_resolve STORAGE_CONFORMANCE_S3_REGRESS = conformance_s3 STORAGE_CONFORMANCE_INPUTDIR = $(srcdir)/test/automation/sqlrepo/smoke/storage_conformance @@ -258,7 +268,7 @@ endif ifndef USE_PGXS # Chained for the same reason as installcheck above, and here it matters more: # every category would otherwise start its own temp instance in ./tmp_check. -check: check-storage-local +check: check-storage-conformance check-format-parquet: submake $(REGRESS_PREP) $(pg_regress_check) $(REGRESS_OPTS) \ @@ -271,4 +281,24 @@ check-storage-local: submake $(REGRESS_PREP) check-format-parquet --inputdir=$(STORAGE_LOCAL_INPUTDIR) $(STORAGE_LOCAL_REGRESS) .PHONY: check-storage-local + +check-storage-s3: submake $(REGRESS_PREP) check-storage-local +ifeq ($(DATALAKE_TEST_S3_ENDPOINT),) + @echo "NOTICE: DATALAKE_TEST_S3_ENDPOINT is not set, skipping the s3 storage tests" +else + $(pg_regress_check) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_S3_INPUTDIR) $(STORAGE_S3_REGRESS) +endif + +.PHONY: check-storage-s3 + +check-storage-conformance: submake $(REGRESS_PREP) check-storage-s3 + $(pg_regress_check) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_CONFORMANCE_INPUTDIR) $(STORAGE_CONFORMANCE_REGRESS) +ifneq ($(DATALAKE_TEST_S3_ENDPOINT),) + $(pg_regress_check) $(REGRESS_OPTS) \ + --inputdir=$(STORAGE_CONFORMANCE_INPUTDIR) $(STORAGE_CONFORMANCE_S3_REGRESS) +endif + +.PHONY: check-storage-conformance endif diff --git a/contrib/datalake_fdw/README.md b/contrib/datalake_fdw/README.md index ade1f8462d0..b9a7bec71f4 100644 --- a/contrib/datalake_fdw/README.md +++ b/contrib/datalake_fdw/README.md @@ -183,10 +183,12 @@ is wrong in ways tests elsewhere will not catch: Cloudberry's memory accounting, and a query that allocates outside it is a query whose memory limit does not apply. * **Classify failures.** Return `arrow::Status::AlreadyExists` for a name in - use, and an `IOError` whose text says so for something missing (the - extension recognises `NoSuchKey`, `NoSuchBucket`, `does not exist`, - `No such file or directory` and a bare `404`); anything else is reported as - an I/O error. + use, and for something missing an `IOError` whose text contains + `does not exist` or `No such file or directory` -- those two phrases and + nothing else, because the text also contains the caller's path and a status + number or a service's error code matched inside it would turn an error + *about* a path into an error about the path not existing. Anything else is + reported as an I/O error. `abi_fingerprint` is checked at registration: the compiler's major version, libstdc++'s dual-ABI setting and the Arrow version have to match the ones the @@ -206,3 +208,10 @@ package as the extension. arise there. * An upload abandoned by a crashed backend leaves its parts behind. A bucket lifecycle rule that expires incomplete multipart uploads is the usual answer. +* Reading and writing through `s3://` costs a backend process about 15 MiB of + resident memory that `gp_vmem_protect_limit` does not see: the SDK client, + its connection and one part buffer are allocated by the SDK itself rather + than through the tracked pool. Measured against the same file written + locally, that overhead stays flat as the object grows -- 7.9 MiB for a 2 MB + file, 14.8 MiB for a 149 MB one -- so it is a per-process constant, not a + cost per byte. diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c index 4d1aa2ffc95..16973344c3c 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c @@ -31,6 +31,7 @@ #include "access/relation.h" #include "access/reloptions.h" #include "am_iceberg/pg_iceberg_options.h" +#include "common/backend_registry.h" #include "catalog/dependency.h" #include "catalog/objectaddress.h" #include "catalog/pg_class.h" @@ -61,7 +62,6 @@ static relopt_kind iceberg_relopt_kind; static char *iceberg_relopt_string(IcebergRelOptions *opts, int off); static MetaKv *defelems_to_kvs(List *options, int *n_props); static DlErrCode invalid_location(char **errdetail, char *detail); -static char *redacted_location_uri(const char *uri); static bool s3_bucket_alnum(char ch); static bool s3_bucket_char(char ch); @@ -525,8 +525,8 @@ s3_bucket_char(char ch) * a query string holds a presigned signature, userinfo holds a password. So * those are reported as present rather than reproduced. */ -static char * -redacted_location_uri(const char *uri) +char * +pg_iceberg_redacted_location_uri(const char *uri) { const char *authority; const char *cut; @@ -600,7 +600,7 @@ pg_iceberg_parse_location(const char *uri, const char *endpoint, return invalid_location(errdetail, pstrdup("location URI is null")); - safe_uri = redacted_location_uri(uri); + safe_uri = pg_iceberg_redacted_location_uri(uri); scheme_end = strstr(uri, "://"); if (scheme_end == NULL) @@ -613,10 +613,23 @@ pg_iceberg_parse_location(const char *uri, const char *endpoint, strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, scheme_len) == 0; is_file = scheme_len == strlen("file") && strncmp(uri, "file", scheme_len) == 0; + + /* + * Two schemes have rules of their own below; any other is acceptable + * exactly when something can read it. A fixed list here would mean a + * third party could register a backend and still have no way to name a + * volume that uses it, which would make the extension contract a promise + * the parser breaks. + */ if (!is_s3 && !is_file) - return invalid_location(errdetail, - psprintf("location URI \"%s\" has unsupported scheme; expected s3 or file", - safe_uri)); + { + char *scheme = pnstrdup(uri, scheme_len); + + if (!datalake_storage_scheme_registered(scheme)) + return invalid_location(errdetail, + psprintf("location URI \"%s\" names storage \"%s\", which no backend is registered for", + safe_uri, scheme)); + } /* * These three say what is wrong without repeating the URI. A query string @@ -641,13 +654,14 @@ pg_iceberg_parse_location(const char *uri, const char *endpoint, return invalid_location(errdetail, psprintf("location URI \"%s\" has an empty authority", safe_uri)); - if (is_file && authority_len != 0) + /* A scheme a backend brought is addressed like file: a path, no host. */ + if (!is_s3 && authority_len != 0) return invalid_location(errdetail, - psprintf("file location URI \"%s\" must have an empty authority", + psprintf("location URI \"%s\" must have an empty authority", safe_uri)); - if (is_file && (path_start == NULL || path_start[0] != '/')) + if (!is_s3 && (path_start == NULL || path_start[0] != '/')) return invalid_location(errdetail, - psprintf("file location URI \"%s\" must have an absolute path", + psprintf("location URI \"%s\" must have an absolute path", safe_uri)); if (memchr(authority_start, '@', authority_len) != NULL) return invalid_location(errdetail, diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h index d081ffacdb0..f995f6c13dd 100644 --- a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h @@ -144,6 +144,13 @@ extern Oid pg_iceberg_am_oid(void); extern MetaKv *pg_iceberg_resolve_credentials(Oid serverid, Oid auth_userid, int *n_props); extern void pg_iceberg_check_server_usage(Oid serverid); +/* + * A form of a location URI that is safe to quote back in an error message: + * the parts that carry secrets -- userinfo, a query string -- are reported as + * present rather than reproduced. Anything that echoes a base_path a user + * typed has to go through this, message and detail alike. + */ +extern char *pg_iceberg_redacted_location_uri(const char *uri); extern DlErrCode pg_iceberg_parse_location(const char *uri, const char *endpoint, const char *region, diff --git a/contrib/datalake_fdw/src/common/backend_registry.cpp b/contrib/datalake_fdw/src/common/backend_registry.cpp index 9b654f9cbc4..c557c435939 100644 --- a/contrib/datalake_fdw/src/common/backend_registry.cpp +++ b/contrib/datalake_fdw/src/common/backend_registry.cpp @@ -296,6 +296,23 @@ datalake_initialize_storage_backend(const DatalakeStorageBackend *backend) return arrow::Status::Invalid("storage backend is not registered"); } +extern "C" bool +datalake_storage_scheme_registered(const char *scheme) +{ + bool found = false; + + /* Asked from C, on a path that must not throw. */ + try + { + found = datalake_lookup_storage_backend(scheme) != NULL; + } + catch (...) + { + found = false; + } + return found; +} + extern "C" void datalake_register_storage_backends(void) { diff --git a/contrib/datalake_fdw/src/common/backend_registry.h b/contrib/datalake_fdw/src/common/backend_registry.h index 3de70e6bddf..78fcc326dcc 100644 --- a/contrib/datalake_fdw/src/common/backend_registry.h +++ b/contrib/datalake_fdw/src/common/backend_registry.h @@ -44,6 +44,13 @@ extern "C" extern void datalake_register_storage_backends(void); +/* + * Whether anything can reach this scheme. The location parser asks, so that + * a volume may name any storage a backend has registered rather than only the + * two this module ships. + */ +extern bool datalake_storage_scheme_registered(const char *scheme); + #ifdef __cplusplus } #endif diff --git a/contrib/datalake_fdw/src/common/dl_err.c b/contrib/datalake_fdw/src/common/dl_err.c index ce0d94a6618..0aae78de42c 100644 --- a/contrib/datalake_fdw/src/common/dl_err.c +++ b/contrib/datalake_fdw/src/common/dl_err.c @@ -29,6 +29,8 @@ #include "postgres.h" +#include "utils/memutils.h" + #include "common/dl_err.h" #include "utils/guc.h" @@ -90,6 +92,67 @@ dl_error_copy_field(char *dest, Size dest_size, const char *src) strlcpy(dest, src, dest_size); } +/* + * Values that must never appear in what a user or a log is shown. + * + * Credentials arrive as options and end up inside a backend's client, from + * where any number of things can quote them back: an SDK message, an + * exception's what(), a third-party backend's own wording. Scrubbing at each + * of those places means every one of them has to remember to; scrubbing here, + * where every error is recorded, means none of them has to. + * + * The list only grows. A session that has mounted a volume keeps hiding that + * volume's secrets afterwards, which is the safe direction to be wrong in, + * and it stays small because it holds credentials rather than data. + */ +#define DL_MAX_SECRETS 32 +#define DL_MIN_SECRET_LEN 6 /* shorter than this and masking would eat + * ordinary words out of every message */ + +static char *dl_secrets[DL_MAX_SECRETS]; +static int dl_nsecrets; + +void +dl_error_add_secret(const char *value) +{ + int i; + + if (value == NULL || strlen(value) < DL_MIN_SECRET_LEN) + return; + + for (i = 0; i < dl_nsecrets; i++) + { + if (strcmp(dl_secrets[i], value) == 0) + return; + } + + if (dl_nsecrets == DL_MAX_SECRETS) + return; /* a session with 32 distinct secrets is not + * a session whose 33rd needs hiding */ + dl_secrets[dl_nsecrets] = MemoryContextStrdup(TopMemoryContext, value); + dl_nsecrets++; +} + +/* Replaces each secret in place; "***" is shorter than any of them. */ +static void +dl_error_scrub(char *text) +{ + int i; + + for (i = 0; i < dl_nsecrets; i++) + { + const char *secret = dl_secrets[i]; + Size len = strlen(secret); + char *at; + + while ((at = strstr(text, secret)) != NULL) + { + memcpy(at, "***", 3); + memmove(at + 3, at + len, strlen(at + len) + 1); + } + } +} + void dl_error_reset(void) { @@ -114,6 +177,10 @@ dl_error_set(DlErrCode code, const char *operation, const char *type, dl_error_copy_field(dl_error_detail.message, sizeof(dl_error_detail.message), message); dl_error_detail.stack[0] = '\0'; + + /* Whatever produced these, they are about to be shown to somebody. */ + dl_error_scrub(dl_error_detail.type); + dl_error_scrub(dl_error_detail.message); } void diff --git a/contrib/datalake_fdw/src/common/dl_err.h b/contrib/datalake_fdw/src/common/dl_err.h index 7e27ab28545..116606c343f 100644 --- a/contrib/datalake_fdw/src/common/dl_err.h +++ b/contrib/datalake_fdw/src/common/dl_err.h @@ -97,6 +97,14 @@ extern void dl_error_set(DlErrCode code, const char *operation, /* Record the implementation's own numeric code, when it reports one. */ extern void dl_error_set_remote_code(int remote_code); +/* + * Remember a value that must never be shown. Every error recorded afterwards + * has it replaced with "***", whichever layer produced the text -- a backend, + * an SDK, or an exception nobody expected. Values shorter than six characters + * are ignored, because masking those would eat words out of ordinary messages. + */ +extern void dl_error_add_secret(const char *value); + /* Record a stack from the failing implementation. */ extern void dl_error_set_stack(const char *stack); diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp index 1fd65460c86..1cb307fc0af 100644 --- a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -46,20 +46,11 @@ #include "common/dl_wrappers.h" #include "common/file_system_wrapper.h" -/* - * Credentials reach the facade once, as options at mount time, but a status - * that has to be redacted can come out of any later call. So the values are - * copied into the file system handle and every file opened on it shares that - * copy; nothing else in the module has to remember to pass them along. - */ -typedef std::shared_ptr> DlSecrets; - struct DatalakeFileSystemData { std::shared_ptr fs; std::string root; const DatalakeStorageBackend *backend; - DlSecrets secrets; }; struct DatalakeFileData @@ -68,7 +59,6 @@ struct DatalakeFileData std::string path; std::shared_ptr input; std::shared_ptr output; - DlSecrets secrets; }; DlStatusDetail::DlStatusDetail(DlErrCode code, std::string type) @@ -101,21 +91,22 @@ DlStatusDetail::type() const } /* - * The option values worth hiding. A secret access key and a session token - * are credentials; an access key id is an identifier that appears in request - * headers and audit records anyway, and hiding it only costs the reader the - * one fact that says which credential was used -- worse, its value tends to - * occur inside bucket and prefix names, which would then be masked too. + * The option values worth hiding, handed to the error layer once at mount so + * that everything recorded afterwards has them removed -- including text this + * module never sees, from an SDK or an exception or a third-party backend. + * + * A secret access key and a session token are credentials. An access key id + * is an identifier that appears in request headers and audit records anyway, + * and hiding it costs the reader the one fact that says which credential was + * used -- worse, its value tends to occur inside bucket and prefix names, + * which would then be masked out of every message about them. */ -std::vector -dl_storage_collect_secrets(const DlKeyValue *kv, int nkv) +void +dl_storage_remember_secrets(const DlKeyValue *kv, int nkv) { - std::vector secrets; - for (int i = 0; kv != NULL && i < nkv; i++) { - if (kv[i].key == NULL || kv[i].value == NULL || - kv[i].value[0] == '\0') + if (kv[i].key == NULL || kv[i].value == NULL) continue; const std::string key(kv[i].key); @@ -125,54 +116,8 @@ dl_storage_collect_secrets(const DlKeyValue *kv, int nkv) key.find("password") == std::string::npos) continue; - secrets.push_back(kv[i].value); - } - return secrets; -} - -static std::string -redact_secrets(std::string text, const std::vector *secrets) -{ - if (secrets == NULL) - return text; - - for (const std::string &value : *secrets) - { - std::string::size_type pos = 0; - - while ((pos = text.find(value, pos)) != std::string::npos) - { - text.replace(pos, value.size(), "***"); - pos += 3; - } - } - return text; -} - -/* - * Whether the text carries the token on its own rather than inside a longer - * run of digits or letters -- "404" the status, not the 404 in a port number - * or an object named log4040. - */ -static bool -contains_token(const std::string &text, const char *token) -{ - const std::string needle(token); - std::string::size_type pos = 0; - - while ((pos = text.find(needle, pos)) != std::string::npos) - { - std::string::size_type end = pos + needle.size(); - bool left = pos == 0 || - !isalnum(static_cast(text[pos - 1])); - bool right = end == text.size() || - !isalnum(static_cast(text[end])); - - if (left && right) - return true; - pos = end; + dl_error_add_secret(kv[i].value); } - return false; } /* @@ -212,15 +157,21 @@ classify_status(const arrow::Status &status, std::string *type_out) return DL_ERR_ALREADY_EXISTS; if (status.IsOutOfMemory()) return DL_ERR_OUT_OF_MEMORY; + if (status.IsInvalid() || status.IsTypeError() || status.IsKeyError()) + return DL_ERR_INVALID_OPTION; if (status.IsIOError()) { + /* + * Only phrases a filesystem writes about itself, never a bare status + * number: the text contains the caller's path, and a directory named + * "404" would otherwise turn every error about it into "not found". + * A backend that can tell properly attaches its own classification + * instead of leaving this to guess. + */ const std::string whole = status.ToString(); - if (whole.find("NoSuchKey") != std::string::npos || - whole.find("NoSuchBucket") != std::string::npos || - whole.find("does not exist") != std::string::npos || - whole.find("No such file or directory") != std::string::npos || - contains_token(whole, "404")) + if (whole.find("No such file or directory") != std::string::npos || + whole.find("does not exist") != std::string::npos) return DL_ERR_NOT_FOUND; } return DL_ERR_IO; @@ -247,8 +198,7 @@ dl_storage_classify(arrow::Status status) } DlErrCode -dl_storage_status_to_err(const arrow::Status &status, const char *operation, - const std::vector *secrets) +dl_storage_status_to_err(const arrow::Status &status, const char *operation) { DlErrCode code; std::string type; @@ -260,10 +210,6 @@ dl_storage_status_to_err(const arrow::Status &status, const char *operation, code = classify_status(status, &type); if (message.empty()) message = status.message(); - - /* Both fields reach the user, so both are redacted. */ - message = redact_secrets(std::move(message), secrets); - type = redact_secrets(std::move(type), secrets); dl_error_set(code, operation, type.c_str(), message.c_str()); return code; } @@ -350,13 +296,13 @@ datalake_fs_open(const DatalakeLocation *location, const DlKeyValue *kv, } else { - auto secrets = std::make_shared>( - dl_storage_collect_secrets(kv, nkv)); - arrow::Status status = datalake_initialize_storage_backend(backend); + arrow::Status status; + + dl_storage_remember_secrets(kv, nkv); + status = datalake_initialize_storage_backend(backend); if (!status.ok()) - rc = dl_storage_status_to_err(status, "initialize storage", - secrets.get()); + rc = dl_storage_status_to_err(status, "initialize storage"); else { DatalakeStorageHost host; @@ -368,7 +314,7 @@ datalake_fs_open(const DatalakeLocation *location, const DlKeyValue *kv, if (!mounted.ok()) rc = dl_storage_status_to_err(mounted.status(), - "mount storage", secrets.get()); + "mount storage"); else if (mounted->fs == NULL) { dl_error_set(DL_ERR_INTERNAL, "mount storage", NULL, @@ -383,7 +329,6 @@ datalake_fs_open(const DatalakeLocation *location, const DlKeyValue *kv, handle->fs = std::move(mounted->fs); handle->root = std::move(mounted->root); handle->backend = backend; - handle->secrets = secrets; *fs_out = handle.release(); rc = DL_OK; } @@ -433,12 +378,30 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, *nnames_out = 0; selector.base_dir = dl_storage_native_path(fs, prefix); selector.recursive = true; - selector.allow_not_found = false; + selector.allow_not_found = true; /* the emptiness rule is above */ auto infos = fs->fs->GetFileInfo(selector); if (!infos.ok()) - rc = dl_storage_status_to_err(infos.status(), "list storage", - fs->secrets.get()); + rc = dl_storage_status_to_err(infos.status(), "list storage"); + else if (infos->empty()) + { + /* + * Nothing at all under the prefix. Object storage cannot + * tell an empty prefix from one that was never written, and a + * filesystem would answer differently, so the rule is made + * here rather than by each backend: nothing there is nothing + * to list, and a caller that named the wrong prefix is told + * so instead of reading an empty table. + */ + std::string message = prefix[0] == '\0' ? + std::string("nothing is stored at the root of this " + "location") : + "nothing is stored under \"" + std::string(prefix) + "\""; + + dl_error_set(DL_ERR_NOT_FOUND, "list storage", NULL, + message.c_str()); + rc = DL_ERR_NOT_FOUND; + } else { for (const auto &info : *infos) @@ -463,7 +426,7 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, if (!paths.empty() && names == NULL) rc = dl_storage_status_to_err( arrow::Status::OutOfMemory("allocating storage listing"), - "list storage", fs->secrets.get()); + "list storage"); else { size_t i = 0; @@ -485,7 +448,7 @@ datalake_fs_list(DatalakeFileSystem fs, const char *prefix, std::free(names); rc = dl_storage_status_to_err( arrow::Status::OutOfMemory("allocating storage listing"), - "list storage", fs->secrets.get()); + "list storage"); } else { @@ -571,7 +534,7 @@ datalake_file_delete(DatalakeFileSystem fs, const char *path) else rc = dl_storage_status_to_err( fs->fs->DeleteFile(dl_storage_native_path(fs, path)), - "delete storage file", fs->secrets.get()); + "delete storage file"); } DL_ABI_GUARD_END(rc, "file_delete"); @@ -598,8 +561,7 @@ datalake_file_open(DatalakeFileSystem fs, const char *path, *file_out = NULL; if (!info.ok()) - rc = dl_storage_status_to_err(info.status(), "inspect storage file", - fs->secrets.get()); + rc = dl_storage_status_to_err(info.status(), "inspect storage file"); else if (mode == DATALAKE_FILE_READ && info->type() == arrow::fs::FileType::NotFound) { @@ -626,15 +588,13 @@ datalake_file_open(DatalakeFileSystem fs, const char *path, handle->fs = fs->fs; handle->path = native_path; - handle->secrets = fs->secrets; if (mode == DATALAKE_FILE_READ) { auto input = fs->fs->OpenInputFile(native_path); if (!input.ok()) rc = dl_storage_status_to_err(input.status(), - "open storage file", - fs->secrets.get()); + "open storage file"); else { handle->input = *input; @@ -647,8 +607,7 @@ datalake_file_open(DatalakeFileSystem fs, const char *path, if (!output.ok()) rc = dl_storage_status_to_err(output.status(), - "create storage file", - fs->secrets.get()); + "create storage file"); else { handle->output = *output; @@ -683,8 +642,7 @@ datalake_file_read(DatalakeFile file, void *buffer, int64_t length, *nread = 0; if (!read.ok()) - rc = dl_storage_status_to_err(read.status(), "read storage file", - file->secrets.get()); + rc = dl_storage_status_to_err(read.status(), "read storage file"); else { *nread = *read; @@ -709,8 +667,7 @@ datalake_file_write(DatalakeFile file, const void *buffer, int64_t length) rc = DL_ARG_ERROR("file_write"); else rc = dl_storage_status_to_err(file->output->Write(buffer, length), - "write storage file", - file->secrets.get()); + "write storage file"); } DL_ABI_GUARD_END(rc, "file_write"); @@ -742,8 +699,7 @@ datalake_file_close(DatalakeFile *file) doomed->output->Close(); if (!status.ok() && doomed->output != NULL) (void) doomed->output->Abort(); - rc = dl_storage_status_to_err(status, "close storage file", - doomed->secrets.get()); + rc = dl_storage_status_to_err(status, "close storage file"); } } DL_ABI_GUARD_END(rc, "file_close"); diff --git a/contrib/datalake_fdw/src/common/local_file_system.cpp b/contrib/datalake_fdw/src/common/local_file_system.cpp index 4d9c8186a88..689864d8030 100644 --- a/contrib/datalake_fdw/src/common/local_file_system.cpp +++ b/contrib/datalake_fdw/src/common/local_file_system.cpp @@ -94,7 +94,16 @@ class CreateOnlyOutputStream : public arrow::io::OutputStream { } - ~CreateOnlyOutputStream() override = default; + /* + * Nothing should reach this without a Close or an Abort, but a C++ + * exception on the way out of a caller can, and what it would leave is an + * empty file that makes every later attempt at the name fail. + */ + ~CreateOnlyOutputStream() override + { + if (!done_) + (void) Abort(); + } arrow::Status Close() override { @@ -175,8 +184,14 @@ LocalCreateOnlyFileSystem::OpenOutputStream( if (!stream.ok()) return stream.status(); /* CreatedFile closes and unlinks */ - created.release(); /* the stream owns the descriptor now */ - return std::make_shared(*stream, path); + /* + * Built before the guard is disarmed: allocating the wrapper can throw, + * and the file must not survive that either. + */ + auto owned = std::make_shared(*stream, path); + + created.release(); /* the wrapper owns the file now */ + return owned; } static arrow::Result diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp index c43bc2f34fa..3548d6375d1 100644 --- a/contrib/datalake_fdw/src/common/s3_file_system.cpp +++ b/contrib/datalake_fdw/src/common/s3_file_system.cpp @@ -242,7 +242,23 @@ class S3InputFile : public arrow::io::RandomAccessFile if (!outcome.IsSuccess()) return status_from_aws("read", bucket_, key_, outcome.GetError()); - return outcome.GetResult().GetContentLength(); + + /* + * The buffer handed to the SDK is exactly nbytes long, so a service + * that ignored the range and sent more has already been stopped by + * the stream buffer; what must not happen is reporting those bytes as + * read, which would hand the caller a length its memory does not + * cover. + */ + int64_t got = outcome.GetResult().GetContentLength(); + + if (got < 0 || got > nbytes) + return arrow::Status::IOError( + "read s3://", bucket_, "/", key_, " returned ", got, + " bytes for a ", nbytes, " byte range") + .WithDetail(std::make_shared(DL_ERR_IO, + "ShortRead")); + return got; } arrow::Result> ReadAt(int64_t position, @@ -308,7 +324,8 @@ class S3OutputStream : public arrow::io::OutputStream ~S3OutputStream() override { - if (!closed_ && !upload_id_.empty()) + /* A live upload is billable, so it does not outlive the stream. */ + if (!upload_id_.empty()) (void) AbortUpload(); } @@ -353,6 +370,13 @@ class S3OutputStream : public arrow::io::OutputStream bool closed() const override { return closed_; } + /* + * Closing is not the same as being done with the upload. A tail part or + * a completion can fail, and the upload is still out there afterwards -- + * so what decides whether there is anything to clean up is whether an + * upload id is still live, never whether Close was called. A failure + * here abandons the upload and reports the original error. + */ arrow::Status Close() override { if (closed_) @@ -362,17 +386,20 @@ class S3OutputStream : public arrow::io::OutputStream if (upload_id_.empty()) return PutWholeObject(); + arrow::Status status; + if (buffer_->size() > 0) - ARROW_RETURN_NOT_OK(UploadPart()); - return CompleteUpload(); + status = UploadPart(); + if (status.ok()) + status = CompleteUpload(); + if (!status.ok() && !upload_id_.empty()) + (void) AbortUpload(); + return status; } arrow::Status Abort() override { - if (closed_) - return arrow::Status::OK(); closed_ = true; - if (upload_id_.empty()) return arrow::Status::OK(); /* nothing was ever created */ return AbortUpload(); @@ -533,9 +560,27 @@ class S3FileSystem : public arrow::fs::FileSystem split_path(path, &bucket, &key); if (key.empty()) { - /* The bucket itself: a directory as far as the caller knows. */ + /* + * The bucket itself. Ask before answering: reporting a bucket + * that is not there as a directory would turn a typo into an + * empty scan instead of an error. + */ + Aws::S3::Model::ListObjectsV2Request probe; + + probe.SetBucket(bucket.c_str()); + probe.SetMaxKeys(1); + + auto outcome = client_->ListObjectsV2(probe); arrow::fs::FileInfo info(path); + if (!outcome.IsSuccess()) + { + if (!is_not_found(outcome.GetError())) + return status_from_aws("inspect", bucket, key, + outcome.GetError()); + info.set_type(arrow::fs::FileType::NotFound); + return info; + } info.set_type(arrow::fs::FileType::Directory); return info; } @@ -633,20 +678,27 @@ class S3FileSystem : public arrow::fs::FileSystem infos.push_back(std::move(info)); } + /* + * A service that says "more" without moving the token would spin + * here forever, and this loop is inside a C++ frame where an + * interrupt cannot be checked. + */ + Aws::String next = result.GetNextContinuationToken(); + more = result.GetIsTruncated(); - token = result.GetNextContinuationToken(); + if (more && (next.empty() || next == token)) + return arrow::Status::IOError( + "listing s3://", bucket, "/", key, + " did not advance past a truncated page") + .WithDetail(std::make_shared(DL_ERR_IO, + "ListStalled")); + token = next; } /* - * Object storage has no empty directory, so "nothing under this - * prefix" and "this prefix does not exist" are the same answer. The - * caller asked to be told about the second one. + * What an empty answer means is the same question for every backend, + * so the facade decides it rather than each of us. */ - if (infos.empty() && !select.allow_not_found) - return arrow::Status::IOError("s3://", bucket, "/", key, - " does not exist") - .WithDetail(std::make_shared(DL_ERR_NOT_FOUND, - "NoSuchKey")); return infos; } @@ -795,6 +847,12 @@ class S3FileSystem : public arrow::fs::FileSystem { Aws::S3::Model::ListObjectsV2Request request; + /* + * Without the trailing slash this would also match a sibling whose + * name merely starts with the same characters; with it, an object + * written as the folder marker itself still counts, which is what + * tools that create empty folders leave behind. + */ request.SetBucket(bucket.c_str()); request.SetPrefix((key + "/").c_str()); request.SetMaxKeys(1); @@ -922,6 +980,20 @@ mount_s3(const DatalakeLocation *location, const DlKeyValue *kv, int nkv, config.endpointOverride = url.c_str(); } + /* + * Falling back to the host's own credentials because half a pair was + * given would run the query as whoever the host is, which is not what + * the user who wrote that mapping asked for. + */ + if ((access_key == NULL) != (secret_key == NULL)) + return arrow::Status::Invalid( + "the user mapping has ", access_key != NULL ? + "access_key_id but no secret_access_key" : + "secret_access_key but no access_key_id"); + if (access_key == NULL && session_token != NULL) + return arrow::Status::Invalid( + "the user mapping has session_token but no access_key_id"); + std::shared_ptr client; if (access_key != NULL && secret_key != NULL) diff --git a/contrib/datalake_fdw/src/common/storage_arrow.h b/contrib/datalake_fdw/src/common/storage_arrow.h index 3fe8998495b..76095f07a1a 100644 --- a/contrib/datalake_fdw/src/common/storage_arrow.h +++ b/contrib/datalake_fdw/src/common/storage_arrow.h @@ -69,18 +69,16 @@ class DlStatusDetail : public arrow::StatusDetail std::string type_; }; -/* The credential values to hide, taken from the mount options. */ -std::vector dl_storage_collect_secrets(const DlKeyValue *kv, - int nkv); - /* - * Turn an Arrow status into a DlErrCode and record it, hiding every value in - * `secrets` from both the message and the error class. Pass the secrets the - * handle carries; NULL only where no credentials exist yet. + * Hand the mount's credential values to the error layer, which removes them + * from everything recorded afterwards. Called once per mount; nothing below + * has to carry them around. */ +void dl_storage_remember_secrets(const DlKeyValue *kv, int nkv); + +/* Turn an Arrow status into a DlErrCode and record it. */ DlErrCode dl_storage_status_to_err(const arrow::Status &status, - const char *operation, - const std::vector *secrets); + const char *operation); /* * Whether a path may be joined onto a mount root: relative, and free of "." diff --git a/contrib/datalake_fdw/src/common/storage_backend_register.h b/contrib/datalake_fdw/src/common/storage_backend_register.h index cbb5c914844..eacb4e98912 100644 --- a/contrib/datalake_fdw/src/common/storage_backend_register.h +++ b/contrib/datalake_fdw/src/common/storage_backend_register.h @@ -35,18 +35,47 @@ extern "C" { #include "postgres.h" + #include "fmgr.h" +#include "utils/elog.h" } +/* + * Register a backend, whatever order the libraries were loaded in. + * + * Loading the extension is PostgreSQL's job, and PostgreSQL reports a missing + * library or symbol by raising an error, which unwinds with longjmp -- through + * this plugin's C++ frames, skipping their destructors. So the load happens + * inside PG_TRY and comes back as a value instead. + */ static inline DlErrCode datalake_storage_register(const DatalakeStorageBackend *backend) { typedef DlErrCode (*dl_register_fn) (const DatalakeStorageBackend *); - dl_register_fn fn = reinterpret_cast( - load_external_function("$libdir/datalake_fdw", - "datalake_register_storage_backend", true, NULL)); + volatile DlErrCode rc = DL_ERR_INTERNAL; + + PG_TRY(); + { + dl_register_fn fn = reinterpret_cast( + load_external_function("$libdir/datalake_fdw", + "datalake_register_storage_backend", + true, NULL)); + + rc = fn(backend); + } + PG_CATCH(); + { + /* + * The caller is a plugin's _PG_init, which is entitled to decide for + * itself whether it can carry on; what it must not get is an unwind + * through its own frames. + */ + FlushErrorState(); + rc = DL_ERR_NOT_SUPPORTED; + } + PG_END_TRY(); - return fn(backend); + return rc; } #endif /* STORAGE_BACKEND_REGISTER_H */ diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c index c21921790ae..6b5c85c3b89 100644 --- a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c @@ -151,7 +151,7 @@ iceberg_volume_fdw_validator(PG_FUNCTION_ARGS) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid iceberg volume %s \"%s\"", DATALAKE_ICEBERG_VOLUME_BASE_PATH, - volume_options.base_path), + pg_iceberg_redacted_location_uri(volume_options.base_path)), errdetail("%s", parse_detail))); PG_RETURN_VOID(); diff --git a/contrib/datalake_fdw/src/test/datalake_fdw_test.c b/contrib/datalake_fdw/src/test/datalake_fdw_test.c index f6d7504e0a4..a4b8954e795 100644 --- a/contrib/datalake_fdw/src/test/datalake_fdw_test.c +++ b/contrib/datalake_fdw/src/test/datalake_fdw_test.c @@ -109,30 +109,16 @@ storage_parse_uri(const char *uri, bool leaf, DatalakeLocation *location, char *detail = NULL; DlErrCode rc; - if (strncmp(uri, "dltest://", 9) == 0) - { - const char *path = uri + 9; - - if (path[0] != '/' || strchr(path, '?') != NULL || strchr(path, '#') != NULL) - { - dl_error_set(DL_ERR_INVALID_OPTION, "parse storage location", NULL, - "dltest location must have an absolute path and no query or fragment"); - dl_error_report(ERROR, DL_ERR_INVALID_OPTION, "parse storage location"); - } - memset(location, 0, sizeof(*location)); - location->abi_version = DATALAKE_LOCATION_ABI_VERSION; - location->scheme = pstrdup("dltest"); - location->authority = pstrdup(""); - location->path_prefix = pstrdup(path); - } - else + /* + * The production parser, dltest included: it accepts any scheme a backend + * has registered, so the test backend is addressed exactly the way a + * third party's would be. + */ + rc = pg_iceberg_parse_location(uri, NULL, NULL, location, &detail); + if (rc != DL_OK) { - rc = pg_iceberg_parse_location(uri, NULL, NULL, location, &detail); - if (rc != DL_OK) - { - dl_error_set(rc, "parse storage location", NULL, detail); - dl_error_report(ERROR, rc, "parse storage location"); - } + dl_error_set(rc, "parse storage location", NULL, detail); + dl_error_report(ERROR, rc, "parse storage location"); } *relative = pstrdup(""); @@ -178,13 +164,28 @@ storage_open_volume(const char *path_or_uri, const char *volume, bool leaf, if (volume != NULL) { DatalakeLocation volume_location; + Size prefix_len; iceberg_volume_resolve(volume, GetUserId(), &volume_location, &kv, &nkv); /* - * The volume says where and how; the URI says which object. Taking - * the endpoint and region from the volume is the point of naming one. + * A volume's credentials belong to the volume's storage. Without + * this, naming any volume would lend its keys to any bucket the + * caller cared to type. */ + prefix_len = strlen(volume_location.path_prefix); + if (strcmp(location.scheme, volume_location.scheme) != 0 || + strcmp(location.authority, volume_location.authority) != 0 || + strncmp(location.path_prefix, volume_location.path_prefix, + prefix_len) != 0 || + (location.path_prefix[prefix_len] != '\0' && + location.path_prefix[prefix_len] != '/')) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" is not inside volume \"%s\"", + path_or_uri, volume))); + + /* The volume says how to reach it; the URI says which object. */ location.endpoint = volume_location.endpoint; location.region = volume_location.region; } diff --git a/contrib/datalake_fdw/src/test/storage_test_backend.cpp b/contrib/datalake_fdw/src/test/storage_test_backend.cpp index ad729dab082..41e47275249 100644 --- a/contrib/datalake_fdw/src/test/storage_test_backend.cpp +++ b/contrib/datalake_fdw/src/test/storage_test_backend.cpp @@ -1,5 +1,31 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file for details. */ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * storage_test_backend.cpp + * A storage backend that exists so the registration contract can be + * tested from inside this module. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/test/storage_test_backend.cpp + * + *------------------------------------------------------------------------- + */ #include diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out index f12ee9f4c48..ea09037e381 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out @@ -417,7 +417,7 @@ CREATE SERVER dlskel_bad_scheme FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 'ftp://x/y'); ERROR: invalid iceberg volume base_path "ftp://x/y" -DETAIL: location URI "ftp://x/y" has unsupported scheme; expected s3 or file +DETAIL: location URI "ftp://x/y" names storage "ftp", which no backend is registered for CREATE SERVER dlskel_bad_authority FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://'); @@ -426,12 +426,12 @@ DETAIL: location URI "s3://" has an empty authority CREATE SERVER dlskel_bad_query FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://b/p?versionId=3'); -ERROR: invalid iceberg volume base_path "s3://b/p?versionId=3" +ERROR: invalid iceberg volume base_path "s3://b/p?***" DETAIL: location URI must not contain a query CREATE SERVER dlskel_bad_userinfo FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS (base_path 's3://user@b/p'); -ERROR: invalid iceberg volume base_path "s3://user@b/p" +ERROR: invalid iceberg volume base_path "s3://***@b/p" DETAIL: location URI must not contain userinfo CREATE SERVER dlskel_bad_bucket FOREIGN DATA WRAPPER iceberg_volume_fdw diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out index 4a283463e9e..6de85c2788a 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_dltest.out @@ -26,7 +26,9 @@ COPY (SELECT 1) TO PROGRAM -- What a Parquet file holds is the business of the format_parquet cases. SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, - (i % 2 = 0) AS flag + (i % 2 = 0) AS flag, ('2024-01-01'::date + i) AS day, + ('2024-01-01 00:00:00+00'::timestamptz + i * interval '1 second') AS at, + decode(lpad(to_hex(i), 8, '0'), 'hex') AS raw FROM generate_series(1, 2000) i$q$, 500, 'snappy', :volume) AS rows_written; rows_written @@ -37,21 +39,35 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', SELECT count(*) AS rows_read, sum(id) AS id_sum, min(label) AS first_label, - count(*) FILTER (WHERE flag) AS flagged + count(*) FILTER (WHERE flag) AS flagged, + max(day) AS last_day, + max(at) AS last_at, + max(encode(raw, 'hex')) AS last_raw FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); - rows_read | id_sum | first_label | flagged ------------+---------+-------------+--------- - 2000 | 2001000 | row 1 | 1000 + AS t(id int, amount float8, label text, flag boolean, day date, + at timestamptz, raw bytea); + rows_read | id_sum | first_label | flagged | last_day | last_at | last_raw +-----------+---------+-------------+---------+------------+------------------------------+---------- + 2000 | 2001000 | row 1 | 1000 | 06-23-2029 | Sun Dec 31 16:33:20 2023 PST | 000007d0 +(1 row) + +-- Columns are matched by the field id each one carries, not by where it sits +-- in the file, so a projection can name them in any order and leave some out. +SELECT count(*) AS projected_rows, min(label) AS first_label, sum(id) AS id_sum +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, + '{3,1}', :volume) AS t(label text, id int); + projected_rows | first_label | id_sum +----------------+-------------+--------- + 2000 | row 1 | 2001000 (1 row) -- Written with 500-row groups, so a range of them is a range of the file. -SELECT count(*) AS rows_in_two_groups -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); - rows_in_two_groups --------------------- - 1000 +SELECT count(*) AS rows_in_two_groups, min(id) AS first_id, max(id) AS last_id +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{1}', :volume) + AS t(id int); + rows_in_two_groups | first_id | last_id +--------------------+----------+--------- + 1000 | 501 | 1500 (1 row) -- The file is there, and it is the only thing there. @@ -70,8 +86,8 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', ERROR: 42P07 \set VERBOSITY default SELECT count(*) AS rows_still_there -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{1}', :volume) + AS t(id int); rows_still_there ------------------ 2000 @@ -80,9 +96,15 @@ FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volum -- A write that fails partway leaves nothing behind: not a truncated file, not -- an empty one, and nothing for the next attempt at that name to trip over. \set VERBOSITY sqlstate +-- Each row carries a kilobyte, and the failure comes after twelve thousand of +-- them: past the point where object storage has begun a multipart upload, so +-- what this proves is that the upload is abandoned and not merely that no +-- object appears. SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', - $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, - 500, '', :volume); + $q$SELECT i AS id, repeat('x', 1024) AS padding, + 1 / (i - 12000) AS boom + FROM generate_series(1, 20000) i$q$, + 2000, '', :volume); ERROR: 22012 \set VERBOSITY default SELECT replace(path, :'root', '') AS object @@ -113,6 +135,12 @@ SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}' SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, '{}', :volume) AS t(id int); ERROR: 42704 +-- So does listing a prefix nothing was ever written under. Object storage has +-- no such thing as a directory and would answer an empty list; a filesystem +-- would answer that there is no such directory. The rule is the facade's, so +-- both say the same thing here. +SELECT count(*) FROM datalake_storage_list(:'prefix' || '/never-written', :kv); +ERROR: 42704 \set VERBOSITY default SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; @@ -121,4 +149,11 @@ SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleane t | t (1 row) +-- And a prefix that held objects until a moment ago is no different from one +-- that never did: on a filesystem the directory is still there and empty, and +-- that has to read the same way. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_storage_list(:'prefix', :kv); +ERROR: 42704 +\set VERBOSITY default COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_conformance_dltest'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out index de68c2716aa..de4c2e45dbf 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_file.out @@ -25,7 +25,9 @@ COPY (SELECT 1) TO PROGRAM -- What a Parquet file holds is the business of the format_parquet cases. SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, - (i % 2 = 0) AS flag + (i % 2 = 0) AS flag, ('2024-01-01'::date + i) AS day, + ('2024-01-01 00:00:00+00'::timestamptz + i * interval '1 second') AS at, + decode(lpad(to_hex(i), 8, '0'), 'hex') AS raw FROM generate_series(1, 2000) i$q$, 500, 'snappy', :volume) AS rows_written; rows_written @@ -36,21 +38,35 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', SELECT count(*) AS rows_read, sum(id) AS id_sum, min(label) AS first_label, - count(*) FILTER (WHERE flag) AS flagged + count(*) FILTER (WHERE flag) AS flagged, + max(day) AS last_day, + max(at) AS last_at, + max(encode(raw, 'hex')) AS last_raw FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); - rows_read | id_sum | first_label | flagged ------------+---------+-------------+--------- - 2000 | 2001000 | row 1 | 1000 + AS t(id int, amount float8, label text, flag boolean, day date, + at timestamptz, raw bytea); + rows_read | id_sum | first_label | flagged | last_day | last_at | last_raw +-----------+---------+-------------+---------+------------+------------------------------+---------- + 2000 | 2001000 | row 1 | 1000 | 06-23-2029 | Sun Dec 31 16:33:20 2023 PST | 000007d0 +(1 row) + +-- Columns are matched by the field id each one carries, not by where it sits +-- in the file, so a projection can name them in any order and leave some out. +SELECT count(*) AS projected_rows, min(label) AS first_label, sum(id) AS id_sum +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, + '{3,1}', :volume) AS t(label text, id int); + projected_rows | first_label | id_sum +----------------+-------------+--------- + 2000 | row 1 | 2001000 (1 row) -- Written with 500-row groups, so a range of them is a range of the file. -SELECT count(*) AS rows_in_two_groups -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); - rows_in_two_groups --------------------- - 1000 +SELECT count(*) AS rows_in_two_groups, min(id) AS first_id, max(id) AS last_id +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{1}', :volume) + AS t(id int); + rows_in_two_groups | first_id | last_id +--------------------+----------+--------- + 1000 | 501 | 1500 (1 row) -- The file is there, and it is the only thing there. @@ -69,8 +85,8 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', ERROR: 42P07 \set VERBOSITY default SELECT count(*) AS rows_still_there -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{1}', :volume) + AS t(id int); rows_still_there ------------------ 2000 @@ -79,9 +95,15 @@ FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volum -- A write that fails partway leaves nothing behind: not a truncated file, not -- an empty one, and nothing for the next attempt at that name to trip over. \set VERBOSITY sqlstate +-- Each row carries a kilobyte, and the failure comes after twelve thousand of +-- them: past the point where object storage has begun a multipart upload, so +-- what this proves is that the upload is abandoned and not merely that no +-- object appears. SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', - $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, - 500, '', :volume); + $q$SELECT i AS id, repeat('x', 1024) AS padding, + 1 / (i - 12000) AS boom + FROM generate_series(1, 20000) i$q$, + 2000, '', :volume); ERROR: 22012 \set VERBOSITY default SELECT replace(path, :'root', '') AS object @@ -112,6 +134,12 @@ SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}' SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, '{}', :volume) AS t(id int); ERROR: 42704 +-- So does listing a prefix nothing was ever written under. Object storage has +-- no such thing as a directory and would answer an empty list; a filesystem +-- would answer that there is no such directory. The rule is the facade's, so +-- both say the same thing here. +SELECT count(*) FROM datalake_storage_list(:'prefix' || '/never-written', :kv); +ERROR: 42704 \set VERBOSITY default SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; @@ -120,4 +148,11 @@ SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleane t | t (1 row) +-- And a prefix that held objects until a moment ago is no different from one +-- that never did: on a filesystem the directory is still there and empty, and +-- that has to read the same way. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_storage_list(:'prefix', :kv); +ERROR: 42704 +\set VERBOSITY default COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_conformance_file'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out index 6f9ff84b106..e3c9def9a75 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/conformance_s3.out @@ -35,7 +35,9 @@ FROM pg_foreign_server WHERE srvname = 'dlconf_volume'; -- What a Parquet file holds is the business of the format_parquet cases. SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, - (i % 2 = 0) AS flag + (i % 2 = 0) AS flag, ('2024-01-01'::date + i) AS day, + ('2024-01-01 00:00:00+00'::timestamptz + i * interval '1 second') AS at, + decode(lpad(to_hex(i), 8, '0'), 'hex') AS raw FROM generate_series(1, 2000) i$q$, 500, 'snappy', :volume) AS rows_written; rows_written @@ -46,21 +48,35 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', SELECT count(*) AS rows_read, sum(id) AS id_sum, min(label) AS first_label, - count(*) FILTER (WHERE flag) AS flagged + count(*) FILTER (WHERE flag) AS flagged, + max(day) AS last_day, + max(at) AS last_at, + max(encode(raw, 'hex')) AS last_raw FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); - rows_read | id_sum | first_label | flagged ------------+---------+-------------+--------- - 2000 | 2001000 | row 1 | 1000 + AS t(id int, amount float8, label text, flag boolean, day date, + at timestamptz, raw bytea); + rows_read | id_sum | first_label | flagged | last_day | last_at | last_raw +-----------+---------+-------------+---------+------------+------------------------------+---------- + 2000 | 2001000 | row 1 | 1000 | 06-23-2029 | Sun Dec 31 16:33:20 2023 PST | 000007d0 +(1 row) + +-- Columns are matched by the field id each one carries, not by where it sits +-- in the file, so a projection can name them in any order and leave some out. +SELECT count(*) AS projected_rows, min(label) AS first_label, sum(id) AS id_sum +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, + '{3,1}', :volume) AS t(label text, id int); + projected_rows | first_label | id_sum +----------------+-------------+--------- + 2000 | row 1 | 2001000 (1 row) -- Written with 500-row groups, so a range of them is a range of the file. -SELECT count(*) AS rows_in_two_groups -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); - rows_in_two_groups --------------------- - 1000 +SELECT count(*) AS rows_in_two_groups, min(id) AS first_id, max(id) AS last_id +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{1}', :volume) + AS t(id int); + rows_in_two_groups | first_id | last_id +--------------------+----------+--------- + 1000 | 501 | 1500 (1 row) -- The file is there, and it is the only thing there. @@ -79,8 +95,8 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', ERROR: 42P07 \set VERBOSITY default SELECT count(*) AS rows_still_there -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{1}', :volume) + AS t(id int); rows_still_there ------------------ 2000 @@ -89,9 +105,15 @@ FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volum -- A write that fails partway leaves nothing behind: not a truncated file, not -- an empty one, and nothing for the next attempt at that name to trip over. \set VERBOSITY sqlstate +-- Each row carries a kilobyte, and the failure comes after twelve thousand of +-- them: past the point where object storage has begun a multipart upload, so +-- what this proves is that the upload is abandoned and not merely that no +-- object appears. SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', - $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, - 500, '', :volume); + $q$SELECT i AS id, repeat('x', 1024) AS padding, + 1 / (i - 12000) AS boom + FROM generate_series(1, 20000) i$q$, + 2000, '', :volume); ERROR: 22012 \set VERBOSITY default SELECT replace(path, :'root', '') AS object @@ -122,6 +144,12 @@ SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}' SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, '{}', :volume) AS t(id int); ERROR: 42704 +-- So does listing a prefix nothing was ever written under. Object storage has +-- no such thing as a directory and would answer an empty list; a filesystem +-- would answer that there is no such directory. The rule is the facade's, so +-- both say the same thing here. +SELECT count(*) FROM datalake_storage_list(:'prefix' || '/never-written', :kv); +ERROR: 42704 \set VERBOSITY default SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; @@ -130,5 +158,12 @@ SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleane t | t (1 row) +-- And a prefix that held objects until a moment ago is no different from one +-- that never did: on a filesystem the directory is still there and empty, and +-- that has to read the same way. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_storage_list(:'prefix', :kv); +ERROR: 42704 +\set VERBOSITY default \set ECHO none NOTICE: drop cascades to user mapping for gpadmin on server dlconf_volume diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/volume_resolve.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/volume_resolve.out new file mode 100644 index 00000000000..95c82fc51f3 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/expected/volume_resolve.out @@ -0,0 +1,118 @@ +-- Resolving a volume: who may use it, whose credentials are used, and what +-- happens when a path does not belong to it. A file volume needs no service +-- to talk to, so this runs everywhere. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_volume && mkdir -p /tmp/datalake_fdw_volume/inside'; +DROP SERVER IF EXISTS dlvol CASCADE; +NOTICE: server "dlvol" does not exist, skipping +DROP SERVER IF EXISTS dlvol_other CASCADE; +NOTICE: server "dlvol_other" does not exist, skipping +DROP ROLE IF EXISTS dlvol_user; +CREATE ROLE dlvol_user LOGIN; +NOTICE: resource queue required -- using default resource queue "pg_default" +CREATE SERVER dlvol FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'file:///tmp/datalake_fdw_volume/inside'); +CREATE SERVER dlvol_other FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'file:///tmp/datalake_fdw_volume/elsewhere'); +-- Through the volume, as its owner. +SELECT datalake_parquet_write('file:///tmp/datalake_fdw_volume/inside/a.parquet', + 'SELECT 42 AS id', 0, '', 'dlvol') AS rows_written; + rows_written +-------------- + 1 +(1 row) + +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); + id +---- + 42 +(1 row) + +-- A path outside the volume is refused, even though the volume would have +-- been happy to lend its settings to it. +SELECT datalake_parquet_write('file:///tmp/datalake_fdw_volume/outside.parquet', + 'SELECT 1 AS id', 0, '', 'dlvol'); +ERROR: "file:///tmp/datalake_fdw_volume/outside.parquet" is not inside volume "dlvol" +-- And so is one under a different volume. +SELECT datalake_parquet_write('file:///tmp/datalake_fdw_volume/inside/b.parquet', + 'SELECT 1 AS id', 0, '', 'dlvol_other'); +ERROR: "file:///tmp/datalake_fdw_volume/inside/b.parquet" is not inside volume "dlvol_other" +-- A volume that does not exist. +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', + 'no_such_volume') AS t(id int); +ERROR: server "no_such_volume" does not exist +-- Using a volume takes USAGE on it. +GRANT EXECUTE ON FUNCTION datalake_parquet_read(text, int, int, int[], text) + TO dlvol_user; +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); +ERROR: permission denied for foreign server dlvol +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlvol TO dlvol_user; +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); + id +---- + 42 +(1 row) + +RESET ROLE; +-- A PUBLIC mapping is what a user without one of their own gets. The file +-- backend ignores credentials, so what is asserted is that resolution finds +-- the mapping and still reaches the file, not that the values did anything. +CREATE USER MAPPING FOR PUBLIC SERVER dlvol + OPTIONS (access_key_id 'public-key', secret_access_key 'public-secret-value'); +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); + id +---- + 42 +(1 row) + +RESET ROLE; +CREATE USER MAPPING FOR dlvol_user SERVER dlvol + OPTIONS (access_key_id 'user-key', secret_access_key 'user-secret-value'); +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); + id +---- + 42 +(1 row) + +RESET ROLE; +-- A base_path is quoted back when it is rejected, and a URI can carry a +-- password in its userinfo. Neither the message nor the detail may reproduce +-- one: if this case ever prints "hunter2", the rejection leaked a credential +-- into the server log. +CREATE SERVER dlvol_secret FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://reader:hunter2@bucket/prefix'); +ERROR: invalid iceberg volume base_path "s3://***@bucket/prefix" +DETAIL: location URI must not contain userinfo +CREATE SERVER dlvol_signed FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://bucket/prefix?X-Amz-Signature=deadbeef'); +ERROR: invalid iceberg volume base_path "s3://bucket/prefix?***" +DETAIL: location URI must not contain a query +DROP SERVER dlvol CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to user mapping for public on server dlvol +drop cascades to user mapping for dlvol_user on server dlvol +DROP SERVER dlvol_other CASCADE; +DROP ROLE dlvol_user; +ERROR: role "dlvol_user" cannot be dropped because some objects depend on it +DETAIL: privileges for function datalake_parquet_read(text,integer,integer,integer[],text) +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_volume'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql index e6c3fded417..f5241f4a99a 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_body.sql @@ -10,21 +10,33 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', $q$SELECT i AS id, (i * 1.5)::float8 AS amount, 'row ' || i AS label, - (i % 2 = 0) AS flag + (i % 2 = 0) AS flag, ('2024-01-01'::date + i) AS day, + ('2024-01-01 00:00:00+00'::timestamptz + i * interval '1 second') AS at, + decode(lpad(to_hex(i), 8, '0'), 'hex') AS raw FROM generate_series(1, 2000) i$q$, 500, 'snappy', :volume) AS rows_written; SELECT count(*) AS rows_read, sum(id) AS id_sum, min(label) AS first_label, - count(*) FILTER (WHERE flag) AS flagged + count(*) FILTER (WHERE flag) AS flagged, + max(day) AS last_day, + max(at) AS last_at, + max(encode(raw, 'hex')) AS last_raw FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); + AS t(id int, amount float8, label text, flag boolean, day date, + at timestamptz, raw bytea); + +-- Columns are matched by the field id each one carries, not by where it sits +-- in the file, so a projection can name them in any order and leave some out. +SELECT count(*) AS projected_rows, min(label) AS first_label, sum(id) AS id_sum +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, + '{3,1}', :volume) AS t(label text, id int); -- Written with 500-row groups, so a range of them is a range of the file. -SELECT count(*) AS rows_in_two_groups -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); +SELECT count(*) AS rows_in_two_groups, min(id) AS first_id, max(id) AS last_id +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 1, 2, '{1}', :volume) + AS t(id int); -- The file is there, and it is the only thing there. SELECT replace(path, :'root', '') AS object @@ -37,15 +49,21 @@ SELECT datalake_parquet_write(:'prefix' || '/roundtrip.parquet', 'SELECT 1 AS id', 0, '', :volume); \set VERBOSITY default SELECT count(*) AS rows_still_there -FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{}', :volume) - AS t(id int, amount float8, label text, flag boolean); +FROM datalake_parquet_read(:'prefix' || '/roundtrip.parquet', 0, 0, '{1}', :volume) + AS t(id int); -- A write that fails partway leaves nothing behind: not a truncated file, not -- an empty one, and nothing for the next attempt at that name to trip over. \set VERBOSITY sqlstate +-- Each row carries a kilobyte, and the failure comes after twelve thousand of +-- them: past the point where object storage has begun a multipart upload, so +-- what this proves is that the upload is abandoned and not merely that no +-- object appears. SELECT datalake_parquet_write(:'prefix' || '/aborted.parquet', - $q$SELECT i AS id, 1 / (i - 1500) AS boom FROM generate_series(1, 2000) i$q$, - 500, '', :volume); + $q$SELECT i AS id, repeat('x', 1024) AS padding, + 1 / (i - 12000) AS boom + FROM generate_series(1, 20000) i$q$, + 2000, '', :volume); \set VERBOSITY default SELECT replace(path, :'root', '') AS object @@ -62,7 +80,20 @@ SELECT id FROM datalake_parquet_read(:'prefix' || '/aborted.parquet', 0, 0, '{}' \set VERBOSITY sqlstate SELECT count(*) FROM datalake_parquet_read(:'prefix' || '/missing.parquet', 0, 0, '{}', :volume) AS t(id int); + +-- So does listing a prefix nothing was ever written under. Object storage has +-- no such thing as a directory and would answer an empty list; a filesystem +-- would answer that there is no such directory. The rule is the facade's, so +-- both say the same thing here. +SELECT count(*) FROM datalake_storage_list(:'prefix' || '/never-written', :kv); \set VERBOSITY default SELECT datalake_storage_delete(:'prefix' || '/roundtrip.parquet', :kv) AS cleaned_roundtrip, datalake_storage_delete(:'prefix' || '/aborted.parquet', :kv) AS cleaned_aborted; + +-- And a prefix that held objects until a moment ago is no different from one +-- that never did: on a filesystem the directory is still there and empty, and +-- that has to read the same way. +\set VERBOSITY sqlstate +SELECT count(*) FROM datalake_storage_list(:'prefix', :kv); +\set VERBOSITY default diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql index 9cef616001b..b826b455e61 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/conformance_s3.sql @@ -13,6 +13,10 @@ SET log_min_error_statement = 'panic'; -- they are read with the echo off: what this file asserts must not depend on -- where it ran. \set ECHO none +-- An unset psql variable interpolates as its own name, so the optional ones +-- are given a value before \getenv has the chance to leave them undefined. +\set region '' +\set path_style '' \getenv endpoint DATALAKE_TEST_S3_ENDPOINT \getenv bucket DATALAKE_TEST_S3_BUCKET \getenv access_key DATALAKE_TEST_S3_ACCESS_KEY @@ -32,14 +36,16 @@ SELECT format('s3://%s/datalake_conformance/%s', :'bucket', :'run') AS base_path 'endpoint=' || :'endpoint', 'region=' || :'region', 'path_style_access=' || coalesce(nullif(:'path_style', ''), 'true'), 'access_key_id=' || :'access_key', - 'secret_access_key=' || :'secret_key') AS kv \gset + 'secret_access_key=' || :'secret_key') AS kv, + coalesce(nullif(:'region', ''), 'us-east-1') AS server_region, + coalesce(nullif(:'path_style', ''), 'true') AS server_path_style \gset DROP SERVER IF EXISTS dlconf_volume CASCADE; CREATE SERVER dlconf_volume FOREIGN DATA WRAPPER iceberg_volume_fdw OPTIONS ( base_path :'base_path', endpoint :'endpoint', - region :'region', - path_style_access :'path_style'); + region :'server_region', + path_style_access :'server_path_style'); CREATE USER MAPPING FOR CURRENT_USER SERVER dlconf_volume OPTIONS ( access_key_id :'access_key', secret_access_key :'secret_key'); diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/volume_resolve.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/volume_resolve.sql new file mode 100644 index 00000000000..6334cdb7493 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_conformance/sql/volume_resolve.sql @@ -0,0 +1,89 @@ +-- Resolving a volume: who may use it, whose credentials are used, and what +-- happens when a path does not belong to it. A file volume needs no service +-- to talk to, so this runs everywhere. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +COPY (SELECT 1) TO PROGRAM + 'rm -rf /tmp/datalake_fdw_volume && mkdir -p /tmp/datalake_fdw_volume/inside'; + +DROP SERVER IF EXISTS dlvol CASCADE; +DROP SERVER IF EXISTS dlvol_other CASCADE; +DROP ROLE IF EXISTS dlvol_user; +CREATE ROLE dlvol_user LOGIN; + +CREATE SERVER dlvol FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'file:///tmp/datalake_fdw_volume/inside'); +CREATE SERVER dlvol_other FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'file:///tmp/datalake_fdw_volume/elsewhere'); + +-- Through the volume, as its owner. +SELECT datalake_parquet_write('file:///tmp/datalake_fdw_volume/inside/a.parquet', + 'SELECT 42 AS id', 0, '', 'dlvol') AS rows_written; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); + +-- A path outside the volume is refused, even though the volume would have +-- been happy to lend its settings to it. +SELECT datalake_parquet_write('file:///tmp/datalake_fdw_volume/outside.parquet', + 'SELECT 1 AS id', 0, '', 'dlvol'); +-- And so is one under a different volume. +SELECT datalake_parquet_write('file:///tmp/datalake_fdw_volume/inside/b.parquet', + 'SELECT 1 AS id', 0, '', 'dlvol_other'); + +-- A volume that does not exist. +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', + 'no_such_volume') AS t(id int); + +-- Using a volume takes USAGE on it. +GRANT EXECUTE ON FUNCTION datalake_parquet_read(text, int, int, int[], text) + TO dlvol_user; +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); +RESET ROLE; + +GRANT USAGE ON FOREIGN SERVER dlvol TO dlvol_user; +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); +RESET ROLE; + +-- A PUBLIC mapping is what a user without one of their own gets. The file +-- backend ignores credentials, so what is asserted is that resolution finds +-- the mapping and still reaches the file, not that the values did anything. +CREATE USER MAPPING FOR PUBLIC SERVER dlvol + OPTIONS (access_key_id 'public-key', secret_access_key 'public-secret-value'); +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); +RESET ROLE; + +CREATE USER MAPPING FOR dlvol_user SERVER dlvol + OPTIONS (access_key_id 'user-key', secret_access_key 'user-secret-value'); +SET ROLE dlvol_user; +SELECT id FROM datalake_parquet_read( + 'file:///tmp/datalake_fdw_volume/inside/a.parquet', 0, 0, '{}', 'dlvol') + AS t(id int); +RESET ROLE; + +-- A base_path is quoted back when it is rejected, and a URI can carry a +-- password in its userinfo. Neither the message nor the detail may reproduce +-- one: if this case ever prints "hunter2", the rejection leaked a credential +-- into the server log. +CREATE SERVER dlvol_secret FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://reader:hunter2@bucket/prefix'); +CREATE SERVER dlvol_signed FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://bucket/prefix?X-Amz-Signature=deadbeef'); + +DROP SERVER dlvol CASCADE; +DROP SERVER dlvol_other CASCADE; +DROP ROLE dlvol_user; +COPY (SELECT 1) TO PROGRAM 'rm -rf /tmp/datalake_fdw_volume'; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out index 479d0318d30..dd116f04d20 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3.out @@ -41,6 +41,18 @@ $$; CREATE FUNCTION s3_uri(name text) RETURNS text LANGUAGE sql AS $$ SELECT s3_prefix() || '/' || name $$; +CREATE FUNCTION s3_error(uri text, kv text[]) RETURNS text LANGUAGE plpgsql AS $fn$ +DECLARE + message text; + detail text; +BEGIN + PERFORM datalake_storage_read_text(uri, kv); + RETURN 'no error'; +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS message = MESSAGE_TEXT, detail = PG_EXCEPTION_DETAIL; + RETURN message || ' | ' || detail; +END +$fn$; -- Round trip. SELECT datalake_storage_write_text(s3_uri('a.txt'), 'first-object', s3_kv()); datalake_storage_write_text @@ -107,18 +119,6 @@ SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); -- What the user is told names the object but not the secret. The text of an -- SDK message varies between services, so this asks what must and must not be -- in it rather than pinning the whole string. -CREATE FUNCTION s3_error(uri text, kv text[]) RETURNS text LANGUAGE plpgsql AS $fn$ -DECLARE - message text; - detail text; -BEGIN - PERFORM datalake_storage_read_text(uri, kv); - RETURN 'no error'; -EXCEPTION WHEN OTHERS THEN - GET STACKED DIAGNOSTICS message = MESSAGE_TEXT, detail = PG_EXCEPTION_DETAIL; - RETURN message || ' | ' || detail; -END -$fn$; SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret'))) LIKE '%' || current_setting('datalake.s3_bad_secret') || '%' AS leaks_the_secret, @@ -131,15 +131,67 @@ SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret') f | t | t (1 row) --- An endpoint that answers nothing has to become an error while someone is --- still waiting for it, rather than a session that cannot be cancelled. +-- Redaction, tested where the service really does echo the value back: a +-- bucket name appears in the error about it, so a run whose secret IS that +-- bucket name must come back with the name masked. Remove the redaction and +-- this fails, which the wrong-secret case above cannot claim. +SELECT s3_error('s3://dl-redaction-probe-9f2b/x.txt', + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=dl-redaction-probe-9f2b']) + LIKE '%dl-redaction-probe-9f2b%' AS leaks_the_secret, + s3_error('s3://dl-redaction-probe-9f2b/x.txt', + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=dl-redaction-probe-9f2b']) + LIKE '%***%' AS masked_it; + leaks_the_secret | masked_it +------------------+----------- + f | t +(1 row) + +-- Half a credential pair is a mistake, not a reason to fall back to whatever +-- identity the host happens to have. \set VERBOSITY sqlstate SELECT datalake_storage_read_text(s3_uri('a.txt'), - ARRAY['endpoint=http://10.255.255.1:9000', + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), 'region=us-east-1', 'path_style_access=true', - 'access_key_id=x', 'secret_access_key=y']); -ERROR: 58030 + 'access_key_id=' || current_setting('datalake.s3_access_key')]); +ERROR: 22023 \set VERBOSITY default +-- Asking for host-style addressing against an endpoint that is an IP address +-- still works, because a bucket name cannot be prepended to an IP and the SDK +-- falls back to path style. Worth pinning: it is the reason a wrong +-- path_style_access setting does not fail loudly in a lab. +SELECT datalake_storage_read_text(s3_uri('a.txt'), + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=us-east-1', 'path_style_access=false', + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=' || current_setting('datalake.s3_secret')]) + AS host_style_against_an_ip; + host_style_against_an_ip +-------------------------- + first-object +(1 row) + +-- An endpoint that answers nothing has to become an error while someone is +-- still waiting for it, rather than a session that cannot be cancelled. +SELECT s3_error(s3_uri('a.txt'), + ARRAY['endpoint=http://10.255.255.1:9000', + 'region=us-east-1', 'path_style_access=true', + 'access_key_id=x', 'secret_access_key=y']) + <> 'no error' AS blackhole_reported, + clock_timestamp() - statement_timestamp() < interval '30 seconds' + AS within_the_bound; + blackhole_reported | within_the_bound +--------------------+------------------ + t | t +(1 row) + -- The session still works afterwards. SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); datalake_storage_read_text diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3_pagination.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3_pagination.out new file mode 100644 index 00000000000..315a0de825f --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/expected/storage_s3_pagination.out @@ -0,0 +1,58 @@ +-- Listing more objects than one page holds. Roughly four thousand requests +-- go over the wire here, which is minutes rather than seconds, so this is its +-- own case and runs where DATALAKE_TEST_S3_PAGINATION says to -- CI, not every +-- local build. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; +SET log_statement = 'none'; +SET log_min_duration_statement = -1; +SET log_min_error_statement = 'panic'; +\set ECHO none +CREATE FUNCTION s3_kv() RETURNS text[] LANGUAGE sql AS $$ + SELECT ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=' || current_setting('datalake.s3_secret')] +$$; +CREATE FUNCTION s3_uri(name text) RETURNS text LANGUAGE sql AS $$ + SELECT format('s3://%s/datalake_regress/%s/%s', + current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run'), name) +$$; +CREATE FUNCTION s3_prefix() RETURNS text LANGUAGE sql AS $$ + SELECT format('s3://%s/datalake_regress/%s', + current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run')) +$$; +-- More objects than one listing page holds. Each aggregate reads the value +-- the function returned rather than counting rows: a target list nothing +-- refers to can be optimised away, and then the writes never happen while the +-- count still looks right. +SELECT count(*) FILTER (WHERE bytes > 0) AS created FROM ( + SELECT datalake_storage_write_text(s3_uri('page/' || lpad(i::text, 5, '0')), + i::text, s3_kv()) AS bytes + FROM generate_series(1, 1100) i) AS w; + created +--------- + 1100 +(1 row) + +SELECT count(*) AS listed FROM datalake_storage_list(s3_prefix() || '/page', s3_kv()); + listed +-------- + 1100 +(1 row) + +SELECT count(*) FILTER (WHERE gone) AS deleted FROM ( + SELECT datalake_storage_delete(s3_uri('page/' || lpad(i::text, 5, '0')), + s3_kv()) AS gone + FROM generate_series(1, 1100) i) AS d; + deleted +--------- + 1100 +(1 row) + +DROP FUNCTION s3_prefix(), s3_uri(text), s3_kv(); diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql index d18feaf5844..bdec675cb9b 100644 --- a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3.sql @@ -19,6 +19,12 @@ SET log_min_error_statement = 'panic'; -- file asserts must not depend on where it ran. Objects also go under a -- prefix of this run's own, so two runs against one bucket cannot collide. \set ECHO none +-- \getenv leaves the variable unset when the environment does not define it, +-- and an unset variable interpolates as its own name, which is a syntax error +-- rather than a default. So the optional ones are given values first. +\set region '' +\set path_style '' +\set bad_secret '' \getenv endpoint DATALAKE_TEST_S3_ENDPOINT \getenv bucket DATALAKE_TEST_S3_BUCKET \getenv access_key DATALAKE_TEST_S3_ACCESS_KEY @@ -68,6 +74,19 @@ CREATE FUNCTION s3_uri(name text) RETURNS text LANGUAGE sql AS $$ SELECT s3_prefix() || '/' || name $$; +CREATE FUNCTION s3_error(uri text, kv text[]) RETURNS text LANGUAGE plpgsql AS $fn$ +DECLARE + message text; + detail text; +BEGIN + PERFORM datalake_storage_read_text(uri, kv); + RETURN 'no error'; +EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS message = MESSAGE_TEXT, detail = PG_EXCEPTION_DETAIL; + RETURN message || ' | ' || detail; +END +$fn$; + -- Round trip. SELECT datalake_storage_write_text(s3_uri('a.txt'), 'first-object', s3_kv()); SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); @@ -104,18 +123,6 @@ SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); -- What the user is told names the object but not the secret. The text of an -- SDK message varies between services, so this asks what must and must not be -- in it rather than pinning the whole string. -CREATE FUNCTION s3_error(uri text, kv text[]) RETURNS text LANGUAGE plpgsql AS $fn$ -DECLARE - message text; - detail text; -BEGIN - PERFORM datalake_storage_read_text(uri, kv); - RETURN 'no error'; -EXCEPTION WHEN OTHERS THEN - GET STACKED DIAGNOSTICS message = MESSAGE_TEXT, detail = PG_EXCEPTION_DETAIL; - RETURN message || ' | ' || detail; -END -$fn$; SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret'))) LIKE '%' || current_setting('datalake.s3_bad_secret') || '%' @@ -125,15 +132,55 @@ SELECT s3_error(s3_uri('a.txt'), s3_kv(current_setting('datalake.s3_bad_secret') s3_error(s3_uri('a.txt'), s3_kv()) = 'no error' AS good_credentials_still_work; --- An endpoint that answers nothing has to become an error while someone is --- still waiting for it, rather than a session that cannot be cancelled. +-- Redaction, tested where the service really does echo the value back: a +-- bucket name appears in the error about it, so a run whose secret IS that +-- bucket name must come back with the name masked. Remove the redaction and +-- this fails, which the wrong-secret case above cannot claim. +SELECT s3_error('s3://dl-redaction-probe-9f2b/x.txt', + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=dl-redaction-probe-9f2b']) + LIKE '%dl-redaction-probe-9f2b%' AS leaks_the_secret, + s3_error('s3://dl-redaction-probe-9f2b/x.txt', + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=dl-redaction-probe-9f2b']) + LIKE '%***%' AS masked_it; + +-- Half a credential pair is a mistake, not a reason to fall back to whatever +-- identity the host happens to have. \set VERBOSITY sqlstate SELECT datalake_storage_read_text(s3_uri('a.txt'), - ARRAY['endpoint=http://10.255.255.1:9000', + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), 'region=us-east-1', 'path_style_access=true', - 'access_key_id=x', 'secret_access_key=y']); + 'access_key_id=' || current_setting('datalake.s3_access_key')]); \set VERBOSITY default +-- Asking for host-style addressing against an endpoint that is an IP address +-- still works, because a bucket name cannot be prepended to an IP and the SDK +-- falls back to path style. Worth pinning: it is the reason a wrong +-- path_style_access setting does not fail loudly in a lab. +SELECT datalake_storage_read_text(s3_uri('a.txt'), + ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=us-east-1', 'path_style_access=false', + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=' || current_setting('datalake.s3_secret')]) + AS host_style_against_an_ip; + +-- An endpoint that answers nothing has to become an error while someone is +-- still waiting for it, rather than a session that cannot be cancelled. +SELECT s3_error(s3_uri('a.txt'), + ARRAY['endpoint=http://10.255.255.1:9000', + 'region=us-east-1', 'path_style_access=true', + 'access_key_id=x', 'secret_access_key=y']) + <> 'no error' AS blackhole_reported, + clock_timestamp() - statement_timestamp() < interval '30 seconds' + AS within_the_bound; + -- The session still works afterwards. SELECT datalake_storage_read_text(s3_uri('a.txt'), s3_kv()); diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3_pagination.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3_pagination.sql new file mode 100644 index 00000000000..99b20ca4b23 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/storage_s3/sql/storage_s3_pagination.sql @@ -0,0 +1,70 @@ +-- Listing more objects than one page holds. Roughly four thousand requests +-- go over the wire here, which is minutes rather than seconds, so this is its +-- own case and runs where DATALAKE_TEST_S3_PAGINATION says to -- CI, not every +-- local build. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +CREATE EXTENSION IF NOT EXISTS datalake_fdw_test; +RESET client_min_messages; + +SET log_statement = 'none'; +SET log_min_duration_statement = -1; +SET log_min_error_statement = 'panic'; + +\set ECHO none +\set region '' +\set path_style '' +\getenv endpoint DATALAKE_TEST_S3_ENDPOINT +\getenv bucket DATALAKE_TEST_S3_BUCKET +\getenv access_key DATALAKE_TEST_S3_ACCESS_KEY +\getenv secret_key DATALAKE_TEST_S3_SECRET_KEY +\getenv region DATALAKE_TEST_S3_REGION +\getenv path_style DATALAKE_TEST_S3_PATH_STYLE + +SELECT set_config('datalake.s3_endpoint', :'endpoint', false) AS endpoint, + set_config('datalake.s3_bucket', :'bucket', false) AS bucket, + set_config('datalake.s3_access_key', :'access_key', false) AS access_key, + set_config('datalake.s3_secret', :'secret_key', false) AS secret, + set_config('datalake.s3_region', + coalesce(nullif(:'region', ''), 'us-east-1'), false) AS region, + set_config('datalake.s3_path_style', + coalesce(nullif(:'path_style', ''), 'true'), false) AS path_style, + set_config('datalake.s3_run', + to_char(clock_timestamp(), 'YYYYMMDDHH24MISSMS') || '_' || + pg_backend_pid(), false) AS run +\gset +\set ECHO all + +CREATE FUNCTION s3_kv() RETURNS text[] LANGUAGE sql AS $$ + SELECT ARRAY['endpoint=' || current_setting('datalake.s3_endpoint'), + 'region=' || current_setting('datalake.s3_region'), + 'path_style_access=' || current_setting('datalake.s3_path_style'), + 'access_key_id=' || current_setting('datalake.s3_access_key'), + 'secret_access_key=' || current_setting('datalake.s3_secret')] +$$; +CREATE FUNCTION s3_uri(name text) RETURNS text LANGUAGE sql AS $$ + SELECT format('s3://%s/datalake_regress/%s/%s', + current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run'), name) +$$; +CREATE FUNCTION s3_prefix() RETURNS text LANGUAGE sql AS $$ + SELECT format('s3://%s/datalake_regress/%s', + current_setting('datalake.s3_bucket'), + current_setting('datalake.s3_run')) +$$; + +-- More objects than one listing page holds. Each aggregate reads the value +-- the function returned rather than counting rows: a target list nothing +-- refers to can be optimised away, and then the writes never happen while the +-- count still looks right. +SELECT count(*) FILTER (WHERE bytes > 0) AS created FROM ( + SELECT datalake_storage_write_text(s3_uri('page/' || lpad(i::text, 5, '0')), + i::text, s3_kv()) AS bytes + FROM generate_series(1, 1100) i) AS w; +SELECT count(*) AS listed FROM datalake_storage_list(s3_prefix() || '/page', s3_kv()); +SELECT count(*) FILTER (WHERE gone) AS deleted FROM ( + SELECT datalake_storage_delete(s3_uri('page/' || lpad(i::text, 5, '0')), + s3_kv()) AS gone + FROM generate_series(1, 1100) i) AS d; + +DROP FUNCTION s3_prefix(), s3_uri(text), s3_kv(); From dbd0f51e23be19aefb8597550762498dad021ae7 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 23 Sep 2026 17:24:52 +0800 Subject: [PATCH 9/9] ci: build the AWS SDK and run datalake_fdw's s3 cases The s3 half of contrib/datalake_fdw's regression had nothing to run against. Three things were missing. The AWS SDK for C++, which no distribution packages: built from source with BUILD_ONLY="s3;sts", and cached under the version, the distribution and the architecture, since nothing else changes it. A miss costs about three minutes; the cache is 4 MB and restores in one second. Only the build dependencies the image lacks are installed, asked for by capability -- naming a package it already has makes dnf try to upgrade it, and on Rocky 10 the newest libcurl-devel wants a libcurl no enabled repository carries. A service that speaks S3: SeaweedFS, one static binary, pinned, and started with an identity file -- without one it accepts any credentials, and the case that asserts a wrong secret is refused would pass by not being tested. Readiness waits on the S3 port itself, which begins listening seconds after the master elects itself. Its coordinates, which "su - gpadmin" drops along with the rest of the environment, so they are named on the command line. Every other test entry leaves that empty, and a leg where the service did not start skips the s3 cases rather than failing them. Arrow on Rocky 9 and 10 now comes from the Arrow project's repository pinned to 17.0.0 and 21.0.0, as it already did on Rocky 8. EPEL's moves when EPEL does, and Arrow is the library this extension's ABI is shared with. --- .github/workflows/build-cloudberry.yml | 181 ++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 83dcad822ad..e0a23385b53 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -1440,6 +1440,63 @@ jobs: echo "Exact KB:" df -k / + # contrib/datalake_fdw reads and writes s3 through the AWS SDK for C++, + # which no distribution packages, so it is built from source. That costs + # a few minutes, and the result depends on nothing but the version, the + # distribution and the architecture -- so it is cached under exactly + # those three, and a hit restores it in seconds. + - name: Restore the AWS SDK for C++ + id: aws-sdk-cache + if: success() && needs.check-skip.outputs.should_skip != 'true' && matrix.pgxs_extension == 'contrib/datalake_fdw' + uses: actions/cache@v4 + with: + path: /opt/datalake + key: aws-sdk-cpp-1.11.844-rocky${{ matrix.rocky_version }}-${{ runner.arch }} + + - name: Build the AWS SDK for C++ + if: success() && needs.check-skip.outputs.should_skip != 'true' && matrix.pgxs_extension == 'contrib/datalake_fdw' && steps.aws-sdk-cache.outputs.cache-hit != 'true' + run: | + set -eo pipefail + + # Only s3 and the sts it needs to assume a role: the whole SDK is + # some three hundred clients and none of the rest is reachable from + # here. Static and position-independent, because what consumes it is + # a shared library that promises to add no runtime dependency of its + # own beyond the system libraries the SDK itself needs. + . /etc/os-release + crb=crb + if [[ "${VERSION_ID%%.*}" == "8" ]]; then crb=powertools; fi + + # Only what is missing, asked for by capability so a package the + # distribution renamed still counts as present -- Rocky 10 ships + # zlib-devel as zlib-ng-compat-devel. Naming a package the image + # already has makes dnf try to upgrade it to the newest build in the + # repository, and that is how this step first failed: Rocky 10's + # newest libcurl-devel wants a libcurl no enabled repository + # carries. The intent is to add build dependencies, never to move + # the image's own packages. + missing="" + for pkg in cmake git ninja-build libcurl-devel openssl-devel zlib-devel; do + rpm -q --whatprovides "${pkg}" > /dev/null 2>&1 || missing="${missing} ${pkg}" + done + if [[ -n "${missing}" ]]; then + dnf install -y --enablerepo=epel --enablerepo=${crb} ${missing} + fi + + git clone --depth 1 --branch 1.11.844 \ + --recurse-submodules --shallow-submodules \ + https://github.com/aws/aws-sdk-cpp.git /tmp/aws-sdk-cpp + cmake -S /tmp/aws-sdk-cpp -B /tmp/aws-sdk-cpp/build -GNinja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/datalake \ + -DBUILD_ONLY="s3;sts" -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DENABLE_TESTING=OFF -DUSE_OPENSSL=ON + ninja -C /tmp/aws-sdk-cpp/build install + + # The cache stores what is under /opt/datalake; the tree it was built + # from is a gigabyte that nothing reads again. + rm -rf /tmp/aws-sdk-cpp + # Modules that ship as extensions are not built into the RPM. Build them # here the way a user would: against the server that was just installed, # through PGXS, with no configured source tree involved. This has to run @@ -1488,11 +1545,21 @@ jobs: dnf install -y --enablerepo=epel --enablerepo=powertools \ arrow-devel-17.0.0-1.el8 parquet-devel-17.0.0-1.el8 else - # From EPEL, which the image has enrolled but left disabled, - # exactly as it does for its own EPEL packages; CRB carries what - # they depend on. + # Apache's own repository rather than EPEL, and pinned. EPEL's + # Arrow moves when EPEL moves, so a version change there would + # arrive in this job with no commit of ours behind it -- and + # Arrow is the library this extension's ABI is shared with. The + # two versions differ on purpose: between them and the Arrow 9 + # the maintainers test locally, the three legs cover the range + # contrib/datalake_fdw claims to build against. + arrow_version=17.0.0-1.el9 + if [[ "${VERSION_ID%%.*}" == "10" ]]; then + arrow_version=21.0.0-1.el10 + fi + dnf install -y \ + https://apache.jfrog.io/artifactory/arrow/almalinux/${VERSION_ID%%.*}/apache-arrow-release-latest.rpm dnf install -y --enablerepo=epel --enablerepo=crb \ - libarrow-devel parquet-libs-devel + "arrow-devel-${arrow_version}" "parquet-devel-${arrow_version}" fi fi @@ -1540,6 +1607,96 @@ jobs: } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log + # The s3 half of contrib/datalake_fdw's regression needs something that + # speaks S3. SeaweedFS is one static binary, starts in a second, and + # answers the same requests as the object stores the maintainers test + # against by hand. Its coordinates go into the job environment, which is + # what the test step reads to decide whether the s3 cases run at all -- + # so every other test entry is unaffected, and a leg where this step did + # not run skips them rather than failing. + - name: Start an S3 service for datalake_fdw + if: success() && needs.check-skip.outputs.should_skip != 'true' && matrix.test == 'ic-datalake-fdw' + run: | + set -eo pipefail + + case "$(uname -m)" in + x86_64) weed_arch=linux_amd64 ;; + aarch64) weed_arch=linux_arm64 ;; + *) echo "::error::no SeaweedFS build for $(uname -m)"; exit 1 ;; + esac + + # Pinned, and to the version the maintainers ran the same suite + # against by hand: a service the tests assert against must not change + # underneath them. + curl -fsSL -o /tmp/seaweedfs.tar.gz \ + "https://github.com/seaweedfs/seaweedfs/releases/download/4.47/${weed_arch}.tar.gz" + tar -C /usr/local/bin -xzf /tmp/seaweedfs.tar.gz weed + rm -f /tmp/seaweedfs.tar.gz + + # One identity with a key pair, because the suite asserts that a + # wrong secret is refused. Started without this file SeaweedFS + # accepts anything, and that case would pass by not being tested. + mkdir -p /tmp/seaweedfs + printf '%s\n' '{"identities":[{"name":"datalake","credentials":[{"accessKey":"datalake","secretKey":"datalake-secret"}],"actions":["Admin","Read","Write","List","Tagging"]}]}' \ + > /tmp/seaweedfs/s3.json + + nohup weed server -dir=/tmp/seaweedfs -ip=127.0.0.1 \ + -master.port=9333 -volume.port=8080 -filer -filer.port=8888 \ + -s3 -s3.port=8333 -s3.config=/tmp/seaweedfs/s3.json \ + > /tmp/seaweedfs/weed.log 2>&1 & + disown + + # Up to a minute, checking rather than sleeping, and checking the S3 + # port itself: the master elects itself and the volume server + # registers seconds before the S3 gateway starts listening, so a + # master that answers is not yet a service the tests can use. An + # unsigned request is refused with 403 once the identity file is in + # effect -- which is the state the wrong-secret case needs -- so any + # HTTP status means listening, and only "000" means nothing answered. + for _ in $(seq 60); do + s3_status=$(curl -sS -o /dev/null -w '%{http_code}' \ + http://127.0.0.1:8333/ 2>/dev/null || true) + if [[ -n "${s3_status}" && "${s3_status}" != "000" ]]; then + break + fi + sleep 1 + done + if [[ -z "${s3_status:-}" || "${s3_status}" == "000" ]]; then + echo "::error::the SeaweedFS S3 gateway never started listening" + tail -50 /tmp/seaweedfs/weed.log + exit 1 + fi + echo "S3 gateway answers an unsigned GET with ${s3_status}" + + # "weed shell" exits 0 whatever happened, so what is checked is the + # bucket being listed afterwards, not an exit code. + echo 's3.bucket.create -name datalake-test' | + weed shell -master=127.0.0.1:9333 > /tmp/seaweedfs/shell.log 2>&1 + if ! echo 's3.bucket.list' | + weed shell -master=127.0.0.1:9333 2>/dev/null | + grep -q 'datalake-test'; then + echo "::error::SeaweedFS has no datalake-test bucket" + cat /tmp/seaweedfs/shell.log + tail -50 /tmp/seaweedfs/weed.log + exit 1 + fi + + { + echo "DATALAKE_TEST_S3_ENDPOINT=http://127.0.0.1:8333" + echo "DATALAKE_TEST_S3_BUCKET=datalake-test" + echo "DATALAKE_TEST_S3_ACCESS_KEY=datalake" + echo "DATALAKE_TEST_S3_SECRET_KEY=datalake-secret" + echo "DATALAKE_TEST_S3_REGION=us-east-1" + echo "DATALAKE_TEST_S3_PATH_STYLE=true" + # A value that is a secret only to this run: the suite asserts it + # never appears in an error message, and the logs are searched for + # it afterwards. + echo "DATALAKE_TEST_S3_BAD_SECRET=dl-canary-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + # Eleven hundred objects through a paginated listing: minutes, so + # it runs here rather than in every local build. + echo "DATALAKE_TEST_S3_PAGINATION=1" + } >> "${GITHUB_ENV}" + - name: "Run Tests: ${{ matrix.test }}" if: success() && needs.check-skip.outputs.should_skip != 'true' env: @@ -1634,6 +1791,21 @@ jobs: PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" fi + # A test that needs a service of its own carries its coordinates in + # the job environment. "su -" builds a login shell and drops all of + # it, so what the tests need has to be named on the command line + # below; every other entry leaves this empty and the command line is + # the one it always was. + EXTRA_TEST_ENV="" + if [[ -n "${DATALAKE_TEST_S3_ENDPOINT:-}" ]]; then + for v in DATALAKE_TEST_S3_ENDPOINT DATALAKE_TEST_S3_BUCKET \ + DATALAKE_TEST_S3_ACCESS_KEY DATALAKE_TEST_S3_SECRET_KEY \ + DATALAKE_TEST_S3_REGION DATALAKE_TEST_S3_PATH_STYLE \ + DATALAKE_TEST_S3_BAD_SECRET DATALAKE_TEST_S3_PAGINATION; do + EXTRA_TEST_ENV+="${v}='${!v}' " + done + fi + # Read configs into array IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" @@ -1660,6 +1832,7 @@ jobs: # Execute test script with proper environment setup if ! time su - gpadmin -c "cd ${SRC_DIR} && \ + ${EXTRA_TEST_ENV}\ MAKE_NAME='${{ matrix.test }}-config$i' \ MAKE_TARGET='$target' \ MAKE_DIRECTORY='-C $dir' \