diff --git a/Project.toml b/Project.toml index b0b5016..e7fefc1 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Postgres" uuid = "8f23287e-300e-4f50-bc2b-9f1dfe95da84" -version = "2.2.2" +version = "2.3.0" [deps] ConcurrentUtilities = "f0e56b4a-5159-44fe-b623-3e5288b988bb" @@ -31,6 +31,7 @@ Harbor = "1.1" JSON = "1" MD5 = "0.2" Parsers = "2.7, 3" +Pkg = "1.10" Random = "1.10" # Reseau 1.5 adds non-blocking tryread! behind isvalid. Reseau = "1.5" @@ -47,8 +48,9 @@ julia = "1.10" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Harbor = "af79dbb9-1a80-47ad-8928-192a4af69376" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Distributed", "Harbor", "Sockets", "Test"] +test = ["Aqua", "Distributed", "Harbor", "Pkg", "Sockets", "Test"] diff --git a/docs/src/manual.md b/docs/src/manual.md index 0124c35..42548d9 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -119,6 +119,8 @@ profiles = DBInterface.execute(conn, """ The `postgres=(name=:column_name,)` tag is only needed when a column should map to a differently named field. Columns such as `id` or `name` can be left untagged because they already match the Julia field name. +With the default `Postgres.PostgresStyle`, a field whose type Postgres.jl can parse from text (strings, symbols, enums, integers, floats, `Bool`, `Char`, `Date`, `Time`, `DateTime`, `UUID`, `Vector{UInt8}`, lazy JSON, timestamps, decimals, and vectors of these) is decoded from the column text by its declared type, optionally wrapped in `Union{Nothing, …}` or `Union{Missing, …}`. A `String` field receives any column's text, so a `json` or `jsonb` column can be read as its JSON text. An integer field also accepts integral `numeric` values such as `12.00`. A parser registered with `register_type!` still applies when it returns the field's type. Typed reads of these fields compile to static calls, so they work in `juliac --trim` builds. Other field types, and all fields read with a custom driver style, receive the value decoded by the column's type, so custom-style `lift` methods see decoded values. + ### Driver Styles Connection behavior such as query logging, server notices, and asynchronous diff --git a/src/api/API.jl b/src/api/API.jl index f4e0332..73279e6 100644 --- a/src/api/API.jl +++ b/src/api/API.jl @@ -229,14 +229,14 @@ _msgsizeof_parts(parts::Tuple) = msgsizeof(first(parts)) + _msgsizeof_parts(Base writepart(io, x) = write(io, x) function writepart(io, x::String) - occursin('\0', x) && throw(Postgres.PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) + occursin('\0', x) && throw(PostgresInterfaceError("PostgreSQL protocol strings cannot contain a NUL byte")) write(io, x) write(io, UInt8(0)) end writepart(io, x::Integer) = write(io, hton(x)) function writepart(io, x::Tuple{String, String}) (occursin('\0', x[1]) || occursin('\0', x[2])) && - throw(Postgres.PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) + throw(PostgresInterfaceError("PostgreSQL startup parameters cannot contain a NUL byte")) write(io, x[1]) write(io, UInt8(0)) write(io, x[2]) @@ -301,15 +301,14 @@ function writestartupmessage( user::String, dbname::String, application_name::Union{Nothing, String}, - options::Union{Nothing, String}, - statement_timeout::Union{Nothing, Int}, + options::String, )::Nothing # statement_timeout is applied with a SET after connect rather than through # the startup `options` parameter: poolers (pgbouncer) reject unknown # startup options outright, so sending it here fails the whole connection. # A caller-supplied `options` value is different: it is passed through as # given, like libpq does, and an empty value is not sent at all. - send_options = options !== nothing && !isempty(options) + send_options = !isempty(options) len = 8 + msgsizeof(("user", user)) + msgsizeof(("database", dbname)) + msgsizeof(("client_encoding", "UTF8")) + 1 application_name !== nothing && (len += msgsizeof(("application_name", application_name))) @@ -329,11 +328,16 @@ function writestartupmessage( return nothing end +_write_messages_to_buffer(buf, debug, ::Tuple{}) = nothing +function _write_messages_to_buffer(buf, debug, msgs::Tuple) + _write_message_to_buffer(buf, debug, first(msgs)) + _write_messages_to_buffer(buf, debug, Base.tail(msgs)) + return nothing +end + function writemessages(socket, debug::Bool, msgs::Vararg{Tuple, N}) where {N} buf = IOBuffer() - for msg in msgs - _write_message_to_buffer(buf, debug, msg) - end + _write_messages_to_buffer(buf, debug, msgs) write(socket, take!(buf)) flush(socket) return @@ -829,7 +833,7 @@ function _startup!(socket, debug::Bool, user::String, dbname::String, @nospecial application_name_v = application_name::Union{String, Nothing} options_v = options::Union{String, Nothing} statement_timeout_v = statement_timeout::Union{Int, Nothing} - writestartupmessage(socket, debug, user, dbname, application_name_v, options_v, statement_timeout_v) + writestartupmessage(socket, debug, user, dbname, application_name_v, something(options_v, "")) # read initial response mt, len = readheader(socket, debug, MAX_PREAUTH_MESSAGE_LEN) if mt == UInt8('E') diff --git a/src/api/types.jl b/src/api/types.jl index 8341800..140cf97 100644 --- a/src/api/types.jl +++ b/src/api/types.jl @@ -407,6 +407,26 @@ function pg_parse_timestamp(s::AbstractString, ::Type{Durations.Timestamp{P}}) w return Durations.Timestamp{P}(Dates.UTInstant(P(Int64(ticks)))) end +# timestamp(tz) column into a Date field: the UTC date, as for DateTime fields +pg_parse_date_any(s::AbstractString)::Date = occursin(':', s) ? Date(pg_parse_datetime_any(s)) : pg_parse_date(s) + +# Integer fields also read integral numeric text such as "12.00" (numeric +# columns, or SUM over one). Out-of-range values throw InexactError, as when +# converting a decoded column value; other text throws the parser's error. +function pg_parse_integer(::Type{T}, s::AbstractString) where {T <: Integer} + x = Parsers.tryparse(T, s) + x === nothing || return x + wide = Parsers.tryparse(Int128, s) + if wide === nothing + dot = findfirst(==('.'), s) + if dot !== nothing && all(==('0'), SubString(s, dot + 1)) + wide = Parsers.tryparse(Int128, SubString(s, 1, dot - 1)) + end + end + wide === nothing && return Parsers.parse(T, s) + return convert(T, wide) +end + # timestamptz column into a DateTime field: sniff a trailing offset/Z function pg_parse_datetime_any(s::AbstractString)::DateTime isempty(s) && throw(ArgumentError("invalid postgres timestamp")) @@ -970,9 +990,35 @@ end const WireFieldScalar = Union{Durations.Timestamp, DataDecimals.AbstractDecimal, DateTime} _wire_field(::Type{T}) where {T} = T <: Union{Missing, WireFieldScalar} _wire_field(::Type{<:AbstractVector{T}}) where {T} = T <: WireFieldScalar -_field_source(::Type{T}, v::FieldValue) where {T} = _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) -StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(T, v)) -StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(T, v), tags) + +_field_source(::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = + _wire_field(T) ? _field_text(v.text) : parse_value(v.oid, v.text, v.registry) + +# With the default style, field types that the lifts below (and StructUtils' +# own String, Symbol and Enum lifts) parse from text decode from the column +# text. The source stays a String, so typed reads compile to static calls +# under `--trim`; OID decoding returns `Any`. Custom styles keep the OID decoder +# so their lifts still receive decoded values, and other field types keep it too. +const TextField = Union{String, Symbol, Bool, Char, Int8, Int16, Int32, Int64, Cuint, Float32, Float64, + Date, Time, DateTime, UUID, Vector{UInt8}, JSONType, Vector{String}, Vector{Int16}, Vector{Int32}, + Vector{Int64}, Vector{Float32}, Vector{Float64}, Vector{Bool}, Vector{Date}, Vector{Time}, + Vector{DateTime}, Vector{UUID}, Vector{Char}} +_text_field(::Type{S}) where {S} = S !== Union{} && (S <: Union{TextField, Enum} || S === IntervalType || _wire_field(S)) + +function _field_source(::PostgresStyle, ::Type{T}, v::FieldValue) where {T} + S = Base.nonnothingtype(Base.nonmissingtype(T)) + _text_field(S) || return parse_value(v.oid, v.text, v.registry) + info = get(v.registry, v.oid, nothing) + if !_wire_field(S) && info !== nothing && info.parser !== nothing && info.julia_type <: S + # a parser registered for this column still applies when it produces + # the field's type (timestamps and decimals always parse the text) + x = info.parser(_field_text(v.text), v.registry) + x isa S && return x + end + return _field_text(v.text) +end +StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue) where {T} = StructUtils.make(st, T, _field_source(st, T, v)) +StructUtils.make(st::AbstractPostgresStyle, ::Type{T}, v::FieldValue, tags) where {T} = StructUtils.make(st, T, _field_source(st, T, v), tags) @static if isdefined(StructUtils, :InterpClosure) && isdefined(StructUtils, :HotStructClosure) @inline function applycast(f::Union{StructUtils.InterpClosure, StructUtils.HotStructClosure}, name, typeId, val, registry::Dict{Int, TypeInfo}) @@ -1000,16 +1046,16 @@ end return end -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = Parsers.parse(Int8, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int8}, s::String) = pg_parse_integer(Int8, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Bool}, s::String) = parse_boolean(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Char}, s::String) = pg_parse_char(s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = Parsers.parse(Int16, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = Parsers.parse(Int32, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = Parsers.parse(Int64, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = Parsers.parse(Cuint, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int16}, s::String) = pg_parse_integer(Int16, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int32}, s::String) = pg_parse_integer(Int32, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Int64}, s::String) = pg_parse_integer(Int64, s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Cuint}, s::String) = pg_parse_integer(Cuint, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float32}, s::String) = Parsers.parse(Float32, s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Float64}, s::String) = Parsers.parse(Float64, s), nothing -StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date(s), nothing +StructUtils.lift(::AbstractPostgresStyle, ::Type{Date}, s::String) = pg_parse_date_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{Time}, s::String) = pg_parse_time(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{DateTime}, s::String) = pg_parse_datetime_any(s), nothing StructUtils.lift(::AbstractPostgresStyle, ::Type{UUID}, s::String) = UUID(s), nothing diff --git a/src/array_parsing.jl b/src/array_parsing.jl index 7f27300..80390fa 100644 --- a/src/array_parsing.jl +++ b/src/array_parsing.jl @@ -4,7 +4,7 @@ using Parsers, Dates, UUIDs import Durations, DataDecimals import ..parse_decimal import ..pg_parse_timestamp -import ..pg_parse_date, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char +import ..pg_parse_date, ..pg_parse_date_any, ..pg_parse_time, ..pg_parse_datetime_any, ..pg_parse_char, ..pg_parse_integer const BRACKET_OPEN = UInt8('[') const BRACKET_CLOSE = UInt8(']') @@ -88,12 +88,12 @@ function parse_scalar(token::String, inner_type::Type{T}, quoted::Bool) where {T !quoted && token == NULL_STR && return missing inner_type === String && return token inner_type === Bool && return parse_bool_token(token) - inner_type === Int16 && return Parsers.parse(Int16, token) - inner_type === Int32 && return Parsers.parse(Int32, token) - inner_type === Int64 && return Parsers.parse(Int64, token) + inner_type === Int16 && return pg_parse_integer(Int16, token) + inner_type === Int32 && return pg_parse_integer(Int32, token) + inner_type === Int64 && return pg_parse_integer(Int64, token) inner_type === Float32 && return Parsers.parse(Float32, token) inner_type === Float64 && return Parsers.parse(Float64, token) - inner_type === Date && return pg_parse_date(token) + inner_type === Date && return pg_parse_date_any(token) inner_type === Time && return pg_parse_time(token) inner_type <: Durations.Timestamp && return pg_parse_timestamp(token, inner_type) inner_type === DateTime && return pg_parse_datetime_any(token) diff --git a/src/execute.jl b/src/execute.jl index 6668b71..8f6761a 100644 --- a/src/execute.jl +++ b/src/execute.jl @@ -261,6 +261,19 @@ Base.close(cursor::Cursor) = DBInterface.close!(cursor) _param(x::AbstractString)::String = String(x) _param(x)::String = string(x) +# Same text as `string(x)`, whose Dates formatting is not trim-safe. +function _padded(n::Integer, width::Int) + digits = string(abs(n)) + for _ in ncodeunits(digits)+1:width + digits = string('0', digits) # lpad is not trim-safe + end + return n < 0 ? string('-', digits) : digits +end +_param(x::Date)::String = string(_padded(year(x), 4), '-', _padded(month(x), 2), '-', _padded(day(x), 2)) +function _param(x::DateTime)::String + text = string(_param(Date(x)), 'T', _padded(hour(x), 2), ':', _padded(minute(x), 2), ':', _padded(second(x), 2)) + return iszero(millisecond(x)) ? text : string(text, '.', _padded(millisecond(x), 3)) +end function _param(x::Durations.Timestamp)::String # PostgreSQL would round finer fractions. Require an exact conversion. timestamp = x isa Durations.Timestamp{Dates.Nanosecond} ? convert(API.PGTimestamp, x) : x @@ -332,6 +345,12 @@ function build_params(params, nparams::Int, sql::AbstractString) return dest end +function build_unchecked_params(params::Tuple) + dest = Vector{Union{String, Missing}}(undef, length(params)) + bind_tuple_params!(dest, params) + return dest +end + function build_unchecked_params(params) dest = Union{String, Missing}[] params === nothing && return dest diff --git a/test/gssapi.jl b/test/gssapi.jl index f0317f7..85e7453 100644 --- a/test/gssapi.jl +++ b/test/gssapi.jl @@ -668,6 +668,12 @@ function test_kerberos_integration() err = connect_err(() -> DBInterface.connect(Postgres.Connection, cfg.host, KRB_USER, nothing; dbname=cfg.dbname, port=cfg.port, gssencmode="prefer", krbsrvname="nosuchservice")) @test err isa GSSAPI.GSSError finally + # a closed session's temp tables are dropped as its backend exits, + # which can trail close!; DROP ROLE fails while the role owns them + for _ in 1:100 + only(DBInterface.execute(admin, "SELECT count(*) AS n FROM pg_stat_activity WHERE usename = '$KRB_USER'")).n == 0 && break + sleep(0.1) + end DBInterface.execute(admin, "DROP ROLE IF EXISTS $KRB_USER") DBInterface.close!(admin) end diff --git a/test/postgres_trim_queries.jl b/test/postgres_trim_queries.jl new file mode 100644 index 0000000..4b6dd11 --- /dev/null +++ b/test/postgres_trim_queries.jl @@ -0,0 +1,151 @@ +using Dates +using DBInterface +using Postgres +using StructUtils +using UUIDs + +@enum TrimStatus trim_active trim_paused + +struct TrimId + profile_id::Int32 +end + +StructUtils.@tags struct TrimProfile + profileId::Int32 &(postgres=(name=:profile_id,),) + displayName::String &(postgres=(name=:display_name,),) + nickname::Union{Nothing, String} + createdAt::DateTime &(postgres=(name=:created_at,),) + deletedAt::Union{Nothing, DateTime} &(postgres=(name=:deleted_at,),) + birthday::Union{Nothing, Date} + active::Bool + score::Union{Missing, Int32} + balance::Float64 + status::TrimStatus + uid::UUID + flags::Vector{Int32} + tags::Vector{String} + settings::String +end + +struct TrimTotals + total::Int64 + rows::Union{Nothing, Int64} +end + +const TRIM_PROFILE_SELECT = """ + SELECT profile_id, display_name, nickname, created_at, deleted_at, birthday, active, + score, balance, status, uid, flags, tags, settings + FROM trim_compile_profiles + """ + +function _postgres_trim_connect() + port = Base.parse(Int, get(ENV, "POSTGRES_TRIM_PORT", "5432")) + return DBInterface.connect( + Postgres.Connection, + get(ENV, "POSTGRES_TRIM_HOST", "127.0.0.1"), + get(ENV, "POSTGRES_TRIM_USER", "postgres"), + get(ENV, "POSTGRES_TRIM_PASSWORD", "postgres"); + dbname=get(ENV, "POSTGRES_TRIM_DBNAME", "postgres"), + port=port, + sslmode="disable", + connect_timeout=5, + application_name="postgres_trim", + options="-c search_path=public", + statement_timeout=30_000, + statement_cache_maxsize=4, + ) +end + +function _check(cond::Bool, msg::String)::Nothing + cond || error(msg) + return nothing +end + +function _check_profile(p::TrimProfile, id::Int32, name::String)::Nothing + _check(p.profileId == id, "unexpected profile id") + _check(p.displayName == name, "unexpected profile name") + _check(p.active, "expected active profile") + _check(!isempty(p.flags) && !isempty(p.tags), "expected non-empty arrays") + _check(startswith(p.settings, "{"), "expected jsonb text in a String field") + return nothing +end + +function run_postgres_trim_queries()::Nothing + conn = _postgres_trim_connect() + try + DBInterface.execute(conn, "DROP TYPE IF EXISTS trim_status") + DBInterface.execute(conn, "CREATE TYPE trim_status AS ENUM ('trim_active', 'trim_paused')") + DBInterface.execute(conn, """ + CREATE TEMP TABLE trim_compile_profiles ( + profile_id integer PRIMARY KEY, + display_name text NOT NULL, + nickname text, + created_at timestamp NOT NULL, + deleted_at timestamptz, + birthday date, + active boolean NOT NULL, + score integer, + balance numeric(10, 2) NOT NULL, + status trim_status NOT NULL, + uid uuid NOT NULL, + flags integer[] NOT NULL, + tags text[] NOT NULL, + settings jsonb NOT NULL + ) + """) + insert_sql = raw""" + INSERT INTO trim_compile_profiles VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9::numeric, $10::trim_status, $11, + $12::integer[], $13::text[], $14::jsonb + ) + RETURNING profile_id + """ + first_id = DBInterface.execute(conn, insert_sql, ( + Int32(1), "Ada", "ada", DateTime(2024, 1, 2, 3, 4, 5), missing, Date(1815, 12, 10), true, + Int32(99), "12.50", "trim_active", UUID("12345678-1234-5678-1234-567812345678"), + "{1,2,3}", "{math,poetry}", "{\"theme\": \"dark\"}", + ), TrimId) + _check(first_id.profile_id == 1, "unexpected inserted id") + DBInterface.transaction(conn) do + DBInterface.execute(conn, insert_sql, ( + Int32(2), "Grace", missing, DateTime(2024, 1, 3, 4, 5, 6), DateTime(2024, 2, 1), missing, true, + missing, "0.00", "trim_paused", UUID("87654321-4321-8765-4321-876543218765"), + "{4,5}", "{navy}", "{}", + )) + end + + stmt = DBInterface.prepare(conn, TRIM_PROFILE_SELECT * raw"WHERE profile_id = $1") + try + profile = DBInterface.execute(stmt, (Int32(1),), TrimProfile) + _check_profile(profile, Int32(1), "Ada") + _check(profile.nickname == "ada" && profile.deletedAt === nothing, "unexpected nullable fields") + _check(profile.balance == 12.5 && profile.status == trim_active, "unexpected numeric or enum field") + finally + DBInterface.close!(stmt) + end + + profiles = DBInterface.execute(conn, TRIM_PROFILE_SELECT * "ORDER BY profile_id", (), Vector{TrimProfile}) + _check(length(profiles) == 2, "expected two profiles") + _check_profile(profiles[2], Int32(2), "Grace") + _check(profiles[2].score === missing && profiles[2].deletedAt == DateTime(2024, 2, 1), "unexpected nullable fields") + + totals = DBInterface.execute(conn, + "SELECT sum(balance)::numeric(10, 2) AS total, count(*) AS rows FROM trim_compile_profiles WHERE balance = 0", + (), TrimTotals) + _check(totals.total == 0 && totals.rows == 1, "unexpected aggregate fields") + + result = DBInterface.execute(conn, "UPDATE trim_compile_profiles SET score = coalesce(score, 0) + 1") + _check(Postgres.rows_affected(result) == 2, "unexpected rows affected") + _check(occursin("UPDATE", Postgres.command_tag(result)), "unexpected command tag") + finally + DBInterface.close!(conn) + end + return nothing +end + +function @main(args::Vector{String})::Cint + run_postgres_trim_queries() + return 0 +end + +Base.Experimental.entrypoint(main, (Vector{String},)) diff --git a/test/result_decoding.jl b/test/result_decoding.jl index f6022f5..bddf8d0 100644 --- a/test/result_decoding.jl +++ b/test/result_decoding.jl @@ -18,6 +18,19 @@ function decoding_values(row) end struct DecodingStyle <: Postgres.AbstractPostgresStyle end + +@enum DecodingColor decoding_red decoding_green + +struct StringFields + json::String + jsonb::String + int::String + id::String + at::Union{Nothing, String} + amount::Union{Missing, String} + tags::String + note::Union{Nothing, String} +end StructUtils.lift(::DecodingStyle, ::Type{Int64}, value::Int32) = Int64(value) + 1, nothing function test_result_decoding() @@ -82,6 +95,54 @@ function test_result_decoding() end end + @testset "String fields take the column text" begin + texts = ["{\"a\": [1, 2]}", "{\"a\": [1, 2]}", "42", "c8b1cf79-de6a-54ab-a142-682c06a0de6a", + "2024-01-02 03:04:05.123+02:30", "12.50", "{a,b}", nothing] + row = decoding_row(texts, [114, 3802, 23, 2950, 1184, 1700, 1009, 25]; names=collect(fieldnames(StringFields))) + @test StructUtils.make(StringFields, row, Postgres.PostgresStyle()) == StringFields(texts[1:7]..., nothing) + target = NamedTuple{(:json, :jsonb), Tuple{String, JSONType}} + typed = StructUtils.make(target, decoding_row(texts[1:2], [114, 3802]; names=[:json, :jsonb]), Postgres.PostgresStyle()) + @test typed.json == texts[1] + @test JSON.parse(typed.jsonb)["a"] == [1, 2] + # untyped results keep the OID decoders + values = decoding_values(row) + @test values[2] isa JSONType && values[3] === Int32(42) && values[7] == ["a", "b"] + # a custom parser registered for a String column still applies + row = decoding_row(["abc"], [999_999]; names=[:value]) + Postgres.API.register_type!(row.type_registry, 999_999, String; parser=(text::String, registry) -> uppercase(text)) + @test StructUtils.make(NamedTuple{(:value,), Tuple{String}}, row, Postgres.PostgresStyle()).value == "ABC" + end + + @testset "Typed fields decode the column text" begin + # values the OID decoders converted exactly before still convert + texts = ["12.00", "7.0", "2024-01-02 23:30:00-05", "{12.00,3}", "5", "decoding_red"] + names = [:total, :count, :day, :amounts, :maybe, :color] + row = decoding_row(texts, [1700, 1700, 1184, 1231, 20, 25]; names=names) + target = NamedTuple{Tuple(names), Tuple{Int64, Int32, Date, Vector{Int64}, Union{Nothing, Int64}, DecodingColor}} + typed = StructUtils.make(target, row, Postgres.PostgresStyle()) + @test typed == (total=12, count=Int32(7), day=Date(2024, 1, 3), amounts=[12, 3], maybe=5, color=decoding_red) + @test_throws ArgumentError StructUtils.make(NamedTuple{(:total,), Tuple{Int64}}, + decoding_row(["12.50"], [1700]; names=[:total]), Postgres.PostgresStyle()) + # a registered decoder applies when it produces the field's type; + # otherwise the field decodes the text by its declared type + row = decoding_row(["decoding_red", "decoding_red"], [999_998, 999_998]; names=[:symbol, :color]) + Postgres.API.register_type!(row.type_registry, 999_998, Symbol; parser=(text::String, registry) -> Symbol(text)) + typed = StructUtils.make(NamedTuple{(:symbol, :color), Tuple{Symbol, DecodingColor}}, row, Postgres.PostgresStyle()) + @test typed == (symbol=:decoding_red, color=decoding_red) + end + + @testset "Date and DateTime parameters" begin + for x in (Date(2024, 1, 2), Date(12345, 1, 2), Date(-5, 1, 2), DateTime(2024, 1, 2, 3, 4, 5), + DateTime(2024, 1, 2, 3, 4, 5, 7), DateTime(2024, 1, 2, 3, 4, 5, 120), DateTime(-44, 3, 15, 12)) + @test Postgres._param(x) == string(x) + end + end + + @testset "Protocol strings reject NUL bytes" begin + @test_throws Postgres.PostgresInterfaceError Postgres.API.writepart(IOBuffer(), "a\0b") + @test_throws Postgres.PostgresInterfaceError Postgres.API.writepart(IOBuffer(), ("key", "a\0b")) + end + @testset "Decoded values own retained text" begin row = decoding_row(["hello α🙂", "{\"value\":\"saved α\"}", "custom β", "{first,second}", raw"\x0001ff"], [25, 3802, 999_999, 1009, 17]) diff --git a/test/runtests.jl b/test/runtests.jl index 71a4f5a..2581d74 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -500,6 +500,7 @@ include("isvalid_fake_server.jl") include("execute_fake_server.jl") include("executemany.jl") include("notification_deadlines.jl") +include("trim_compile_tests.jl") @testset "Postgres" begin test_timestamps() @@ -1050,6 +1051,7 @@ include("notification_deadlines.jl") conn = wait_for_connection(cfg) try test_executemany(conn) + run_postgres_trim_compile_tests(cfg) @testset "Auth" begin if occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS) conn_trust = DBInterface.connect(Postgres.Connection, cfg.host, cfg.user, nothing; dbname=cfg.dbname, port=cfg.port) diff --git a/test/trim/Project.toml b/test/trim/Project.toml new file mode 100644 index 0000000..fdf4b1d --- /dev/null +++ b/test/trim/Project.toml @@ -0,0 +1,4 @@ +[deps] +DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" +JuliaC = "acedd4c2-ced6-4a15-accc-2607eb759ba2" +StructUtils = "ec057cc2-7a8d-4b58-b3b3-92acb9f63b42" diff --git a/test/trim_compile_tests.jl b/test/trim_compile_tests.jl new file mode 100644 index 0000000..3eb790f --- /dev/null +++ b/test/trim_compile_tests.jl @@ -0,0 +1,109 @@ +# Compiles test/postgres_trim_queries.jl with juliac, requires a clean verifier +# report apart from known stdlib gaps, and runs the executable against the test +# database. `--trim=unsafe-warn` reports the same verifier errors as `safe` but +# still builds the executable, so the stdlib gaps don't prevent the run. +import Pkg + +const _TRIM_SUPPORTED = VERSION >= v"1.12.0-rc1" && isempty(VERSION.prerelease) +const _JULIAC_ENTRYPOINT_EXPR = "using JuliaC; if isdefined(JuliaC, :main); JuliaC.main(ARGS); else JuliaC._main_cli(ARGS); end" +const _TRIM_COMPILE_TIMEOUT_S = parse(Float64, get(ENV, "POSTGRES_TRIM_COMPILE_TIMEOUT_S", "600")) +const _TRIM_RUN_TIMEOUT_S = parse(Float64, get(ENV, "POSTGRES_TRIM_EXE_TIMEOUT_S", "60")) + +function _prepare_trim_project(trim_project::String)::Nothing + mkpath(trim_project) + cp(joinpath(@__DIR__, "trim", "Project.toml"), joinpath(trim_project, "Project.toml")) + original_project = Base.active_project() + try + Pkg.activate(trim_project) + Pkg.develop(Pkg.PackageSpec(path = pkgdir(Postgres))) + Pkg.instantiate() + finally + original_project === nothing || Pkg.activate(original_project) + end + return nothing +end + +function _run_with_timeout(cmd::Cmd; timeout_s::Float64, label::String) + output_path = tempname() + exit_code, timed_out = -1, false + open(output_path, "w") do out + proc = run(pipeline(ignorestatus(cmd), stdout = out, stderr = out); wait = false) + started_at = time() + while Base.process_running(proc) + if time() - started_at >= timeout_s + timed_out = true + kill(proc) + break + end + sleep(0.1) + end + timed_out || wait(proc) + exit_code = something(proc.exitcode, -1) + end + output = read(output_path, String) + rm(output_path; force = true) + timed_out && error("trim $(label) timed out after $(timeout_s)s:\n$(output)") + return exit_code, output +end + +# Released Julia's Base64 pipes and Dates.CompoundPeriod constructor are not +# trim-safe (SASLAuth's SCRAM exchange and interval decoding reach them). An +# error whose stdlib frames, before the first package frame, pass through those +# files is reported but not counted. +function _stdlib_trim_gap(error_text::AbstractString)::Bool + for m in eachmatch(r"\n\s+@ (\S+) (\S+):\d+", error_text) + mod, path = m.captures + occursin(r"/Base64/src/|/Dates/src/periods\.jl", path) && return true + (mod == "Core" || mod == "Base" || startswith(mod, "Base.")) || return false + end + return false +end + +function _trim_verify_counts(output::String) + blocks = split(output, r"\nVerifier (?=error #\d+:)")[2:end] + gaps = count(_stdlib_trim_gap, blocks) + warnings = length(collect(eachmatch(r"Verifier warning #\d+:", output))) + return length(blocks) - gaps, gaps, warnings +end + +function run_postgres_trim_compile_tests(cfg)::Nothing + @testset "Trim compile" begin + if Sys.iswindows() || Sys.WORD_SIZE != 64 || !_TRIM_SUPPORTED + println("[trim] skipped: JuliaC trim compilation runs on 64-bit Linux/macOS with released Julia 1.12+") + @test true + return nothing + end + mktempdir() do tmpdir + trim_project = joinpath(tmpdir, "trim_project") + _prepare_trim_project(trim_project) + julia = joinpath(Sys.BINDIR, Base.julia_exename()) + script = joinpath(@__DIR__, "postgres_trim_queries.jl") + exe = joinpath(tmpdir, "postgres_trim_queries") + # --output-exe takes a bare name, written to the working directory + compile = Cmd(`$julia --startup-file=no --history-file=no --code-coverage=none --project=$trim_project -e $(_JULIAC_ENTRYPOINT_EXPR) -- --output-exe postgres_trim_queries --project=$trim_project --experimental --trim=unsafe-warn $script`; dir = tmpdir) + exit_code, output = _run_with_timeout(compile; timeout_s = _TRIM_COMPILE_TIMEOUT_S, label = "compile") + errors, stdlib_errors, warnings = _trim_verify_counts(output) + println("[trim] verifier errors=$(errors) stdlib_errors=$(stdlib_errors) warnings=$(warnings)") + (errors > 0 || warnings > 0 || exit_code != 0) && println(output) + @test errors == 0 + @test warnings == 0 + @test exit_code == 0 + (errors == 0 && exit_code == 0) || return nothing + # password auth would run SASLAuth's SCRAM exchange, one of the stdlib gaps + if !(occursin("trust", DEFAULT_AUTH) || occursin("trust", DEFAULT_INITDB_ARGS)) + println("[trim] executable run skipped: needs trust auth") + return nothing + end + env = copy(ENV) + env["POSTGRES_TRIM_HOST"] = cfg.host + env["POSTGRES_TRIM_PORT"] = string(cfg.port) + env["POSTGRES_TRIM_USER"] = cfg.user + env["POSTGRES_TRIM_PASSWORD"] = cfg.password + env["POSTGRES_TRIM_DBNAME"] = cfg.dbname + run_exit, run_output = _run_with_timeout(setenv(`$exe`, env); timeout_s = _TRIM_RUN_TIMEOUT_S, label = "run") + run_exit == 0 || println(run_output) + @test run_exit == 0 + end + end + return nothing +end