From dd293fbd12e2ca19bfed7c00a3f41d2e7d24a31d Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 16:22:17 -0600 Subject: [PATCH 1/4] Make typed reads and parameter binding statically compilable `juliac --trim=safe` rejected typed queries: an application with a few hundred row structs hit 1,455 verifier errors, all in Postgres.jl. Typed fields that are not timestamps or decimals were decoded by the column's OID decoder, which returns `Any`, so every field's `StructUtils.make` call was a runtime dispatch. Reading a json/jsonb (or any non-text) column into a `String` field also failed with a MethodError, because the OID decoder produced lazy JSON, integers, UUIDs and so on. With the default `PostgresStyle`, fields whose type Postgres.jl parses from text (strings, symbols, enums, integers, floats, Bool, Char, Date, Time, DateTime, UUID, bytea, lazy JSON, timestamps, decimals, and vectors of these, optionally unioned with Nothing or Missing) now decode from the column text by their declared type. The source stays a `String`, so each read compiles to static calls. A `String` field receives any column's text. Decoders keep the conversions that worked before: integer fields accept integral numeric text such as "12.00" and throw InexactError when out of range, and a Date field reads the UTC date of a timestamptz. A parser registered for the column still applies when its declared type fits the field. Custom styles keep the OID decoder, so their lifts still receive decoded values; other field types keep it too. Also fixed while verifying under `--trim`: - Protocol strings containing a NUL byte threw UndefVarError (the check referred to `Postgres.PostgresInterfaceError` from inside the API module) instead of PostgresInterfaceError. - The startup message took an unused `statement_timeout` argument; with `options` passed as a String the call no longer needs runtime dispatch. - `writemessages` and tuple parameters to `execute` iterated heterogeneous tuples in a loop; both now unroll. - Date and DateTime parameters were formatted with `string(x)`, whose Dates printing is not trim-safe; they use an equivalent static formatter. Behavior changes for typed fields read with the default style: an Int or Float field over a bool column now errors (it previously read 1 or 0), and a registered parser whose declared type does not fit the field is no longer applied before converting; the field parses the column text instead. Co-Authored-By: Claude Opus 5.5 --- docs/src/manual.md | 2 ++ src/api/API.jl | 22 ++++++++------ src/api/types.jl | 64 +++++++++++++++++++++++++++++++++++------ src/array_parsing.jl | 10 +++---- src/execute.jl | 19 ++++++++++++ test/result_decoding.jl | 61 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 23 deletions(-) diff --git a/docs/src/manual.md b/docs/src/manual.md index 0124c35..42548d9 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -119,6 +119,8 @@ profiles = DBInterface.execute(conn, """ The `postgres=(name=:column_name,)` tag is only needed when a column should map to a differently named field. Columns such as `id` or `name` can be left untagged because they already match the Julia field name. +With the default `Postgres.PostgresStyle`, a field whose type Postgres.jl can parse from text (strings, symbols, enums, integers, floats, `Bool`, `Char`, `Date`, `Time`, `DateTime`, `UUID`, `Vector{UInt8}`, lazy JSON, timestamps, decimals, and vectors of these) is decoded from the column text by its declared type, optionally wrapped in `Union{Nothing, …}` or `Union{Missing, …}`. A `String` field receives any column's text, so a `json` or `jsonb` column can be read as its JSON text. An integer field also accepts integral `numeric` values such as `12.00`. A parser registered with `register_type!` still applies when it returns the field's type. Typed reads of these fields compile to static calls, so they work in `juliac --trim` builds. Other field types, and all fields read with a custom driver style, receive the value decoded by the column's type, so custom-style `lift` methods see decoded values. + ### Driver Styles Connection behavior such as query logging, server notices, and asynchronous diff --git a/src/api/API.jl b/src/api/API.jl index f4e0332..73279e6 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -229,14 +229,14 @@ _msgsizeof_parts(parts::Tuple) = msgsizeof(first(parts)) + _msgsizeof_parts(Base writepart(io, x) = write(io, x) function writepart(io, x::String) - occursin('\0', x) && throw(Postgres.PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) + occursin('\0', x) && throw(PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) write(io, x) write(io, UInt8(0)) end writepart(io, x::Integer) = write(io, hton(x)) function writepart(io, x::Tuple{String, String}) (occursin('\0', x[1]) || occursin('\0', x[2])) && - throw(Postgres.PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) + throw(PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) write(io, x[1]) write(io, UInt8(0)) write(io, x[2]) @@ -301,15 +301,14 @@ function writestartupmessage( user::String, dbname::String, application_name::Union{Nothing, String}, - options::Union{Nothing, String}, - statement_timeout::Union{Nothing, Int}, + options::String, )::Nothing # statement_timeout is applied with a SET after connect rather than through # the startup `options` parameter: poolers (pgbouncer) reject unknown # startup options outright, so sending it here fails the whole connection. # A caller-supplied `options` value is different: it is passed through as # given, like libpq does, and an empty value is not sent at all. - send_options = options !== nothing && !isempty(options) + send_options = !isempty(options) len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + msgsizeof(("client_encoding", "UTF8")) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) @@ -329,11 +328,16 @@ function writestartupmessage( return nothing end +_write_messages_to_buffer(buf, debug, ::Tuple{}) = nothing +function _write_messages_to_buffer(buf, debug, msgs::Tuple) + _write_message_to_buffer(buf, debug, first(msgs)) + _write_messages_to_buffer(buf, debug, Base.tail(msgs)) + return nothing +end + function writemessages(socket, debug::Bool, msgs::Vararg{Tuple, N}) where {N} buf = IOBuffer() - for msg in msgs - _write_message_to_buffer(buf, debug, msg) - end + _write_messages_to_buffer(buf, debug, msgs) write(socket, take!(buf)) flush(socket) return @@ -829,7 +833,7 @@ function _startup!(socket, debug::Bool, user::String, dbname::String, @nospecial application_name_v = application_name::Union{String, Nothing} options_v = options::Union{String, Nothing} statement_timeout_v = statement_timeout::Union{Int, Nothing} - writestartupmessage(socket, debug, user, dbname, application_name_v, options_v, statement_timeout_v) + writestartupmessage(socket, debug, user, dbname, application_name_v, something(options_v, "")) # read initial response mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') diff --git a/src/api/types.jl b/src/api/types.jl index 8341800..140cf97 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -407,6 +407,26 @@ function pg_parse_timestamp(s::AbstractString, ::Type{Durations.Timestamp{P}}) w return Durations.Timestamp{P}(Dates.UTInstant(P(Int64(ticks)))) end +# timestamp(tz) column into a Date field: the UTC date, as for DateTime fields +pg_parse_date_any(s::AbstractString)::Date = occursin(':', s) ? Date(pg_parse_datetime_any(s)) : pg_parse_date(s) + +# Integer fields also read integral numeric text such as "12.00" (numeric +# columns, or SUM over one). Out-of-range values throw InexactError, as when +# converting a decoded column value; other text throws the parser's error. +function pg_parse_integer(::Type{T}, s::AbstractString) where {T <: Integer} + x = Parsers.tryparse(T, s) + x === nothing || return x + wide = Parsers.tryparse(Int128, s) + if wide === nothing + dot = findfirst(==('.'), s) + if dot !== nothing && all(==('0'), SubString(s, dot + 1)) + wide = Parsers.tryparse(Int128, SubString(s, 1, dot - 1)) + end + end + wide === nothing && return Parsers.parse(T, s) + return convert(T, wide) +end + # timestamptz column into a DateTime field: sniff a trailing offset/Z function pg_parse_datetime_any(s::AbstractString)::DateTime isempty(s) && throw(ArgumentError("invalid postgres timestamp")) @@ -970,9 +990,35 @@ end const WireFieldScalar = Union{Durations.Timestamp, DataDecimals.AbstractDecimal, DateTime} _wire_field(::Type{T}) where {T} = T <: Union{Missing, WireFieldScalar} _wire_field(::Type{<:AbstractVector{T}}) where {T} = T <: WireFieldScalar -_field_source(::Type{T}, v::FieldValue) where {T} = _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) -StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(T, v)) -StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(T, v), tags) + +_field_source(::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = + _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) + +# With the default style, field types that the lifts below (and StructUtils' +# own String, Symbol and Enum lifts) parse from text decode from the column +# text. The source stays a String, so typed reads compile to static calls +# under `--trim`; OID decoding returns `Any`. Custom styles keep the OID decoder +# so their lifts still receive decoded values, and other field types keep it too. +const TextField = Union{String, Symbol, Bool, Char, Int8, Int16, Int32, Int64, Cuint, Float32, Float64, + Date, Time, DateTime, UUID, Vector{UInt8}, JSONType, Vector{String}, Vector{Int16}, Vector{Int32}, + Vector{Int64}, Vector{Float32}, Vector{Float64}, Vector{Bool}, Vector{Date}, Vector{Time}, + Vector{DateTime}, Vector{UUID}, Vector{Char}} +_text_field(::Type{S}) where {S} = S !== Union{} && (S <: Union{TextField, Enum} || S === IntervalType || _wire_field(S)) + +function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} + S = Base.nonnothingtype(Base.nonmissingtype(T)) + _text_field(S) || return parse_value(v.oid, v.text, v.registry) + info = get(v.registry, v.oid, nothing) + if !_wire_field(S) && info !== nothing && info.parser !== nothing && info.julia_type <: S + # a parser registered for this column still applies when it produces + # the field's type (timestamps and decimals always parse the text) + x = info.parser(_field_text(v.text), v.registry) + x isa S && return x + end + return _field_text(v.text) +end +StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(st, T, v)) +StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(st, T, v), tags) @static if isdefined(StructUtils, :InterpClosure) && isdefined(StructUtils, :HotStructClosure) @inline function applycast(f::Union{StructUtils.InterpClosure, StructUtils.HotStructClosure}, name, typeId, val, registry::Dict{Int, TypeInfo}) @@ -1000,16 +1046,16 @@ end return end -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = Parsers.parse(Int8, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = pg_parse_integer(Int8, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Bool}, s::String) = parse_boolean(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = pg_parse_char(s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = Parsers.parse(Int16, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = Parsers.parse(Int32, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = Parsers.parse(Int64, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = Parsers.parse(Cuint, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = pg_parse_integer(Int16, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = pg_parse_integer(Int32, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = pg_parse_integer(Int64, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = pg_parse_integer(Cuint, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float32}, s::String) = Parsers.parse(Float32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float64}, s::String) = Parsers.parse(Float64, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date(s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Time}, s::String) = pg_parse_time(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{DateTime}, s::String) = pg_parse_datetime_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{UUID}, s::String) = UUID(s), nothing diff --git a/src/array_parsing.jl b/src/array_parsing.jl index 7f27300..80390fa 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -4,7 +4,7 @@ using Parsers, Dates, UUIDs import Durations, DataDecimals import ..parse_decimal import ..pg_parse_timestamp -import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char +import ..pg_parse_date, ..pg_parse_date_any, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char, ..pg_parse_integer const BRACKET_OPEN = UInt8('[') const BRACKET_CLOSE = UInt8(']') @@ -88,12 +88,12 @@ function parse_scalar(token::String, inner_type::Type{T}, quoted::Bool) where {T !quoted && token == NULL_STR && return missing inner_type === String && return token inner_type === Bool && return parse_bool_token(token) - inner_type === Int16 && return Parsers.parse(Int16, token) - inner_type === Int32 && return Parsers.parse(Int32, token) - inner_type === Int64 && return Parsers.parse(Int64, token) + inner_type === Int16 && return pg_parse_integer(Int16, token) + inner_type === Int32 && return pg_parse_integer(Int32, token) + inner_type === Int64 && return pg_parse_integer(Int64, token) inner_type === Float32 && return Parsers.parse(Float32, token) inner_type === Float64 && return Parsers.parse(Float64, token) - inner_type === Date && return pg_parse_date(token) + inner_type === Date && return pg_parse_date_any(token) inner_type === Time && return pg_parse_time(token) inner_type <: Durations.Timestamp && return pg_parse_timestamp(token, inner_type) inner_type === DateTime && return pg_parse_datetime_any(token) diff --git a/src/execute.jl b/src/execute.jl index 6668b71..8f6761a 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -261,6 +261,19 @@ Base.close(cursor::Cursor) = DBInterface.close!(cursor) _param(x::AbstractString)::String = String(x) _param(x)::String = string(x) +# Same text as `string(x)`, whose Dates formatting is not trim-safe. +function _padded(n::Integer, width::Int) + digits = string(abs(n)) + for _ in ncodeunits(digits)+1:width + digits = string('0', digits) # lpad is not trim-safe + end + return n < 0 ? string('-', digits) : digits +end +_param(x::Date)::String = string(_padded(year(x), 4), '-', _padded(month(x), 2), '-', _padded(day(x), 2)) +function _param(x::DateTime)::String + text = string(_param(Date(x)), 'T', _padded(hour(x), 2), ':', _padded(minute(x), 2), ':', _padded(second(x), 2)) + return iszero(millisecond(x)) ? text : string(text, '.', _padded(millisecond(x), 3)) +end function _param(x::Durations.Timestamp)::String # PostgreSQL would round finer fractions. Require an exact conversion. timestamp = x isa Durations.Timestamp{Dates.Nanosecond} ? convert(API.PGTimestamp, x) : x @@ -332,6 +345,12 @@ function build_params(params, nparams::Int, sql::AbstractString) return dest end +function build_unchecked_params(params::Tuple) + dest = Vector{Union{String, Missing}}(undef, length(params)) + bind_tuple_params!(dest, params) + return dest +end + function build_unchecked_params(params) dest = Union{String, Missing}[] params === nothing && return dest diff --git a/test/result_decoding.jl b/test/result_decoding.jl index f6022f5..bddf8d0 100644 --- a/test/result_decoding.jl +++ b/test/result_decoding.jl @@ -18,6 +18,19 @@ function decoding_values(row) end struct DecodingStyle <: Postgres.AbstractPostgresStyle end + +@enum DecodingColor decoding_red decoding_green + +struct StringFields + json::String + jsonb::String + int::String + id::String + at::Union{Nothing, String} + amount::Union{Missing, String} + tags::String + note::Union{Nothing, String} +end StructUtils.lift(::DecodingStyle, ::Type{Int64}, value::Int32) = Int64(value) + 1, nothing function test_result_decoding() @@ -82,6 +95,54 @@ function test_result_decoding() end end + @testset "String fields take the column text" begin + texts = ["{\"a\": [1, 2]}", "{\"a\": [1, 2]}", "42", "c8b1cf79-de6a-54ab-a142-682c06a0de6a", + "2024-01-02 03:04:05.123+02:30", "12.50", "{a,b}", nothing] + row = decoding_row(texts, [114, 3802, 23, 2950, 1184, 1700, 1009, 25]; names=collect(fieldnames(StringFields))) + @test StructUtils.make(StringFields, row, Postgres.PostgresStyle()) == StringFields(texts[1:7]..., nothing) + target = NamedTuple{(:json, :jsonb), Tuple{String, JSONType}} + typed = StructUtils.make(target, decoding_row(texts[1:2], [114, 3802]; names=[:json, :jsonb]), Postgres.PostgresStyle()) + @test typed.json == texts[1] + @test JSON.parse(typed.jsonb)["a"] == [1, 2] + # untyped results keep the OID decoders + values = decoding_values(row) + @test values[2] isa JSONType && values[3] === Int32(42) && values[7] == ["a", "b"] + # a custom parser registered for a String column still applies + row = decoding_row(["abc"], [999_999]; names=[:value]) + Postgres.API.register_type!(row.type_registry, 999_999, String; parser=(text::String, registry) -> uppercase(text)) + @test StructUtils.make(NamedTuple{(:value,), Tuple{String}}, row, Postgres.PostgresStyle()).value == "ABC" + end + + @testset "Typed fields decode the column text" begin + # values the OID decoders converted exactly before still convert + texts = ["12.00", "7.0", "2024-01-02 23:30:00-05", "{12.00,3}", "5", "decoding_red"] + names = [:total, :count, :day, :amounts, :maybe, :color] + row = decoding_row(texts, [1700, 1700, 1184, 1231, 20, 25]; names=names) + target = NamedTuple{Tuple(names), Tuple{Int64, Int32, Date, Vector{Int64}, Union{Nothing, Int64}, DecodingColor}} + typed = StructUtils.make(target, row, Postgres.PostgresStyle()) + @test typed == (total=12, count=Int32(7), day=Date(2024, 1, 3), amounts=[12, 3], maybe=5, color=decoding_red) + @test_throws ArgumentError StructUtils.make(NamedTuple{(:total,), Tuple{Int64}}, + decoding_row(["12.50"], [1700]; names=[:total]), Postgres.PostgresStyle()) + # a registered decoder applies when it produces the field's type; + # otherwise the field decodes the text by its declared type + row = decoding_row(["decoding_red", "decoding_red"], [999_998, 999_998]; names=[:symbol, :color]) + Postgres.API.register_type!(row.type_registry, 999_998, Symbol; parser=(text::String, registry) -> Symbol(text)) + typed = StructUtils.make(NamedTuple{(:symbol, :color), Tuple{Symbol, DecodingColor}}, row, Postgres.PostgresStyle()) + @test typed == (symbol=:decoding_red, color=decoding_red) + end + + @testset "Date and DateTime parameters" begin + for x in (Date(2024, 1, 2), Date(12345, 1, 2), Date(-5, 1, 2), DateTime(2024, 1, 2, 3, 4, 5), + DateTime(2024, 1, 2, 3, 4, 5, 7), DateTime(2024, 1, 2, 3, 4, 5, 120), DateTime(-44, 3, 15, 12)) + @test Postgres._param(x) == string(x) + end + end + + @testset "Protocol strings reject NUL bytes" begin + @test_throws Postgres.PostgresInterfaceError Postgres.API.writepart(IOBuffer(), "a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.API.writepart(IOBuffer(), ("key", "a\0b")) + end + @testset "Decoded values own retained text" begin row = decoding_row(["hello α🙂", "{\"value\":\"saved α\"}", "custom β", "{first,second}", raw"\x0001ff"], [25, 3802, 999_999, 1009, 17]) From b2cb7a8fe9a398c3562bd5a1fa1fb3262457bc02 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 16:22:17 -0600 Subject: [PATCH 2/4] Restore the juliac --trim compile test The 1.0 release removed test/trim_compile_tests.jl, so later changes to typed decoding regressed `--trim` builds without a failing test. The restored test compiles test/postgres_trim_queries.jl with JuliaC, requires zero verifier errors, and runs the executable against the test database (trust auth). The workload covers typed reads of nullable strings, dates and timestamps, numerics into Float64 and Int fields, enums, arrays, and jsonb into a String field, plus prepared statements, transactions and parameters of each bound type. JuliaC needs Julia 1.12, so it lives in test/trim/Project.toml and the test builds that environment only on released 1.12+ (64-bit, non-Windows), as JSON.jl does. Released Julia's Base64 pipes and Dates.CompoundPeriod constructor are not trim-safe; SASLAuth's SCRAM exchange and interval decoding reach them. Errors raised inside those stdlib files are reported and not counted, and the test compiles with `--trim=unsafe-warn` so the executable still builds. Co-Authored-By: Claude Opus 5.5 --- Project.toml | 4 +- test/postgres_trim_queries.jl | 151 ++++++++++++++++++++++++++++++++++ test/runtests.jl | 2 + test/trim/Project.toml | 4 + test/trim_compile_tests.jl | 109 ++++++++++++++++++++++++ 5 files changed, 269 insertions(+), 1 deletion(-) create mode 100644 test/postgres_trim_queries.jl create mode 100644 test/trim/Project.toml create mode 100644 test/trim_compile_tests.jl diff --git a/Project.toml b/Project.toml index b0b5016..8f06212 100644 --- a/Project.toml +++ b/Project.toml @@ -31,6 +31,7 @@ Harbor = "1.1" JSON = "1" MD5 = "0.2" Parsers = "2.7, 3" +Pkg = "1.10" Random = "1.10" # Reseau 1.5 adds non-blocking tryread! behind isvalid. Reseau = "1.5" @@ -47,8 +48,9 @@ julia = "1.10" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Distributed", "Harbor", "Sockets", "Test"] +test = ["Aqua", "Distributed", "Harbor", "Pkg", "Sockets", "Test"] diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl new file mode 100644 index 0000000..4b6dd11 --- /dev/null +++ b/test/postgres_trim_queries.jl @@ -0,0 +1,151 @@ +using Dates +using DBInterface +using Postgres +using StructUtils +using UUIDs + +@enum TrimStatus trim_active trim_paused + +struct TrimId + profile_id::Int32 +end + +StructUtils.@tags struct TrimProfile + profileId::Int32 &(postgres=(name=:profile_id,),) + displayName::String &(postgres=(name=:display_name,),) + nickname::Union{Nothing, String} + createdAt::DateTime &(postgres=(name=:created_at,),) + deletedAt::Union{Nothing, DateTime} &(postgres=(name=:deleted_at,),) + birthday::Union{Nothing, Date} + active::Bool + score::Union{Missing, Int32} + balance::Float64 + status::TrimStatus + uid::UUID + flags::Vector{Int32} + tags::Vector{String} + settings::String +end + +struct TrimTotals + total::Int64 + rows::Union{Nothing, Int64} +end + +const TRIM_PROFILE_SELECT = """ + SELECT profile_id, display_name, nickname, created_at, deleted_at, birthday, active, + score, balance, status, uid, flags, tags, settings + FROM trim_compile_profiles + """ + +function _postgres_trim_connect() + port = Base.parse(Int, get(ENV, "POSTGRES_TRIM_PORT", "5432")) + return DBInterface.connect( + Postgres.Connection, + get(ENV, "POSTGRES_TRIM_HOST", "127.0.0.1"), + get(ENV, "POSTGRES_TRIM_USER", "postgres"), + get(ENV, "POSTGRES_TRIM_PASSWORD", "postgres"); + dbname=get(ENV, "POSTGRES_TRIM_DBNAME", "postgres"), + port=port, + sslmode="disable", + connect_timeout=5, + application_name="postgres_trim", + options="-c search_path=public", + statement_timeout=30_000, + statement_cache_maxsize=4, + ) +end + +function _check(cond::Bool, msg::String)::Nothing + cond || error(msg) + return nothing +end + +function _check_profile(p::TrimProfile, id::Int32, name::String)::Nothing + _check(p.profileId == id, "unexpected profile id") + _check(p.displayName == name, "unexpected profile name") + _check(p.active, "expected active profile") + _check(!isempty(p.flags) && !isempty(p.tags), "expected non-empty arrays") + _check(startswith(p.settings, "{"), "expected jsonb text in a String field") + return nothing +end + +function run_postgres_trim_queries()::Nothing + conn = _postgres_trim_connect() + try + DBInterface.execute(conn, "DROP TYPE IF EXISTS trim_status") + DBInterface.execute(conn, "CREATE TYPE trim_status AS ENUM ('trim_active', 'trim_paused')") + DBInterface.execute(conn, """ + CREATE TEMP TABLE trim_compile_profiles ( + profile_id integer PRIMARY KEY, + display_name text NOT NULL, + nickname text, + created_at timestamp NOT NULL, + deleted_at timestamptz, + birthday date, + active boolean NOT NULL, + score integer, + balance numeric(10, 2) NOT NULL, + status trim_status NOT NULL, + uid uuid NOT NULL, + flags integer[] NOT NULL, + tags text[] NOT NULL, + settings jsonb NOT NULL + ) + """) + insert_sql = raw""" + INSERT INTO trim_compile_profiles VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9::numeric, $10::trim_status, $11, + $12::integer[], $13::text[], $14::jsonb + ) + RETURNING profile_id + """ + first_id = DBInterface.execute(conn, insert_sql, ( + Int32(1), "Ada", "ada", DateTime(2024, 1, 2, 3, 4, 5), missing, Date(1815, 12, 10), true, + Int32(99), "12.50", "trim_active", UUID("12345678-1234-5678-1234-567812345678"), + "{1,2,3}", "{math,poetry}", "{\"theme\": \"dark\"}", + ), TrimId) + _check(first_id.profile_id == 1, "unexpected inserted id") + DBInterface.transaction(conn) do + DBInterface.execute(conn, insert_sql, ( + Int32(2), "Grace", missing, DateTime(2024, 1, 3, 4, 5, 6), DateTime(2024, 2, 1), missing, true, + missing, "0.00", "trim_paused", UUID("87654321-4321-8765-4321-876543218765"), + "{4,5}", "{navy}", "{}", + )) + end + + stmt = DBInterface.prepare(conn, TRIM_PROFILE_SELECT * raw"WHERE profile_id = $1") + try + profile = DBInterface.execute(stmt, (Int32(1),), TrimProfile) + _check_profile(profile, Int32(1), "Ada") + _check(profile.nickname == "ada" && profile.deletedAt === nothing, "unexpected nullable fields") + _check(profile.balance == 12.5 && profile.status == trim_active, "unexpected numeric or enum field") + finally + DBInterface.close!(stmt) + end + + profiles = DBInterface.execute(conn, TRIM_PROFILE_SELECT * "ORDER BY profile_id", (), Vector{TrimProfile}) + _check(length(profiles) == 2, "expected two profiles") + _check_profile(profiles[2], Int32(2), "Grace") + _check(profiles[2].score === missing && profiles[2].deletedAt == DateTime(2024, 2, 1), "unexpected nullable fields") + + totals = DBInterface.execute(conn, + "SELECT sum(balance)::numeric(10, 2) AS total, count(*) AS rows FROM trim_compile_profiles WHERE balance = 0", + (), TrimTotals) + _check(totals.total == 0 && totals.rows == 1, "unexpected aggregate fields") + + result = DBInterface.execute(conn, "UPDATE trim_compile_profiles SET score = coalesce(score, 0) + 1") + _check(Postgres.rows_affected(result) == 2, "unexpected rows affected") + _check(occursin("UPDATE", Postgres.command_tag(result)), "unexpected command tag") + finally + DBInterface.close!(conn) + end + return nothing +end + +function @main(args::Vector{String})::Cint + run_postgres_trim_queries() + return 0 +end + +Base.Experimental.entrypoint(main, (Vector{String},)) diff --git a/test/runtests.jl b/test/runtests.jl index 71a4f5a..2581d74 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -500,6 +500,7 @@ include("isvalid_fake_server.jl") include("execute_fake_server.jl") include("executemany.jl") include("notification_deadlines.jl") +include("trim_compile_tests.jl") @testset "Postgres" begin test_timestamps() @@ -1050,6 +1051,7 @@ include("notification_deadlines.jl") conn = wait_for_connection(cfg) try test_executemany(conn) + run_postgres_trim_compile_tests(cfg) @testset "Auth" begin if occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS) conn_trust = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, nothing; dbname=cfg.dbname, port=cfg.port) diff --git a/test/trim/Project.toml b/test/trim/Project.toml new file mode 100644 index 0000000..fdf4b1d --- /dev/null +++ b/test/trim/Project.toml @@ -0,0 +1,4 @@ +[deps] +DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" +JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2" +StructUtils = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl new file mode 100644 index 0000000..3eb790f --- /dev/null +++ b/test/trim_compile_tests.jl @@ -0,0 +1,109 @@ +# Compiles test/postgres_trim_queries.jl with juliac, requires a clean verifier +# report apart from known stdlib gaps, and runs the executable against the test +# database. `--trim=unsafe-warn` reports the same verifier errors as `safe` but +# still builds the executable, so the stdlib gaps don't prevent the run. +import Pkg + +const _TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" && isempty(VERSION.prerelease) +const _JULIAC_ENTRYPOINT_EXPR = "using JuliaC; if isdefined(JuliaC, :main); JuliaC.main(ARGS); else JuliaC._main_cli(ARGS); end" +const _TRIM_COMPILE_TIMEOUT_S = parse(Float64, get(ENV, "POSTGRES_TRIM_COMPILE_TIMEOUT_S", "600")) +const _TRIM_RUN_TIMEOUT_S = parse(Float64, get(ENV, "POSTGRES_TRIM_EXE_TIMEOUT_S", "60")) + +function _prepare_trim_project(trim_project::String)::Nothing + mkpath(trim_project) + cp(joinpath(@__DIR__, "trim", "Project.toml"), joinpath(trim_project, "Project.toml")) + original_project = Base.active_project() + try + Pkg.activate(trim_project) + Pkg.develop(Pkg.PackageSpec(path = pkgdir(Postgres))) + Pkg.instantiate() + finally + original_project === nothing || Pkg.activate(original_project) + end + return nothing +end + +function _run_with_timeout(cmd::Cmd; timeout_s::Float64, label::String) + output_path = tempname() + exit_code, timed_out = -1, false + open(output_path, "w") do out + proc = run(pipeline(ignorestatus(cmd), stdout = out, stderr = out); wait = false) + started_at = time() + while Base.process_running(proc) + if time() - started_at >= timeout_s + timed_out = true + kill(proc) + break + end + sleep(0.1) + end + timed_out || wait(proc) + exit_code = something(proc.exitcode, -1) + end + output = read(output_path, String) + rm(output_path; force = true) + timed_out && error("trim $(label) timed out after $(timeout_s)s:\n$(output)") + return exit_code, output +end + +# Released Julia's Base64 pipes and Dates.CompoundPeriod constructor are not +# trim-safe (SASLAuth's SCRAM exchange and interval decoding reach them). An +# error whose stdlib frames, before the first package frame, pass through those +# files is reported but not counted. +function _stdlib_trim_gap(error_text::AbstractString)::Bool + for m in eachmatch(r"\n\s+@ (\S+) (\S+):\d+", error_text) + mod, path = m.captures + occursin(r"/Base64/src/|/Dates/src/periods\.jl", path) && return true + (mod == "Core" || mod == "Base" || startswith(mod, "Base.")) || return false + end + return false +end + +function _trim_verify_counts(output::String) + blocks = split(output, r"\nVerifier (?=error #\d+:)")[2:end] + gaps = count(_stdlib_trim_gap, blocks) + warnings = length(collect(eachmatch(r"Verifier warning #\d+:", output))) + return length(blocks) - gaps, gaps, warnings +end + +function run_postgres_trim_compile_tests(cfg)::Nothing + @testset "Trim compile" begin + if Sys.iswindows() || Sys.WORD_SIZE != 64 || !_TRIM_SUPPORTED + println("[trim] skipped: JuliaC trim compilation runs on 64-bit Linux/macOS with released Julia 1.12+") + @test true + return nothing + end + mktempdir() do tmpdir + trim_project = joinpath(tmpdir, "trim_project") + _prepare_trim_project(trim_project) + julia = joinpath(Sys.BINDIR, Base.julia_exename()) + script = joinpath(@__DIR__, "postgres_trim_queries.jl") + exe = joinpath(tmpdir, "postgres_trim_queries") + # --output-exe takes a bare name, written to the working directory + compile = Cmd(`$julia --startup-file=no --history-file=no --code-coverage=none --project=$trim_project -e $(_JULIAC_ENTRYPOINT_EXPR) -- --output-exe postgres_trim_queries --project=$trim_project --experimental --trim=unsafe-warn $script`; dir = tmpdir) + exit_code, output = _run_with_timeout(compile; timeout_s = _TRIM_COMPILE_TIMEOUT_S, label = "compile") + errors, stdlib_errors, warnings = _trim_verify_counts(output) + println("[trim] verifier errors=$(errors) stdlib_errors=$(stdlib_errors) warnings=$(warnings)") + (errors > 0 || warnings > 0 || exit_code != 0) && println(output) + @test errors == 0 + @test warnings == 0 + @test exit_code == 0 + (errors == 0 && exit_code == 0) || return nothing + # password auth would run SASLAuth's SCRAM exchange, one of the stdlib gaps + if !(occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS)) + println("[trim] executable run skipped: needs trust auth") + return nothing + end + env = copy(ENV) + env["POSTGRES_TRIM_HOST"] = cfg.host + env["POSTGRES_TRIM_PORT"] = string(cfg.port) + env["POSTGRES_TRIM_USER"] = cfg.user + env["POSTGRES_TRIM_PASSWORD"] = cfg.password + env["POSTGRES_TRIM_DBNAME"] = cfg.dbname + run_exit, run_output = _run_with_timeout(setenv(`$exe`, env); timeout_s = _TRIM_RUN_TIMEOUT_S, label = "run") + run_exit == 0 || println(run_output) + @test run_exit == 0 + end + end + return nothing +end From 46294c59180d4d5429b4f947111a7ff2cd85f086 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 16:52:48 -0600 Subject: [PATCH 3/4] Bump version to 2.3.0 Co-Authored-By: Claude Opus 5.5 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8f06212..e7fefc1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Postgres" uuid = "8f23287e-300e-4f50-bc2b-9f1dfe95da84" -version = "2.2.2" +version = "2.3.0" [deps] ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" From 36feebc668cfe762c3f1d979dbe5e7aae63cfd26 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Mon, 5 Oct 2026 17:05:53 -0600 Subject: [PATCH 4/4] test: wait for krbuser backends to exit before dropping the role The Kerberos fixture's cleanup ran `DROP ROLE krbuser` right after closing the last krbuser connection. That connection created a temp table, and PostgreSQL drops a session's temp tables only as its backend exits, which can trail `close!`. On a fast runner the DROP ROLE won the race and failed with 2BP01 ("owner of table pg_temp_4.bulk_gss"). Poll pg_stat_activity until the role has no sessions (up to 10s) before dropping it. Co-Authored-By: Claude Opus 5.5 --- test/gssapi.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/gssapi.jl b/test/gssapi.jl index f0317f7..85e7453 100644 --- a/test/gssapi.jl +++ b/test/gssapi.jl @@ -668,6 +668,12 @@ function test_kerberos_integration() err = connect_err(() -> DBInterface.connect(Postgres.Connection, cfg.host, KRB_USER, nothing; dbname=cfg.dbname, port=cfg.port, gssencmode="prefer", krbsrvname="nosuchservice")) @test err isa GSSAPI.GSSError finally + # a closed session's temp tables are dropped as its backend exits, + # which can trail close!; DROP ROLE fails while the role owns them + for _ in 1:100 + only(DBInterface.execute(admin, "SELECT count(*) AS n FROM pg_stat_activity WHERE usename = '$KRB_USER'")).n == 0 && break + sleep(0.1) + end DBInterface.execute(admin, "DROP ROLE IF EXISTS $KRB_USER") DBInterface.close!(admin) end