diff --git a/Project.toml b/Project.toml index bc134714..3b5d05d3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LibPQ" uuid = "194296ae-ab2e-5f79-8cd4-7183a0a5a0d1" license = "MIT" -version = "1.6.2" +version = "1.7.0" [deps] BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" diff --git a/src/LibPQ.jl b/src/LibPQ.jl index 2cc1dac9..81b45088 100644 --- a/src/LibPQ.jl +++ b/src/LibPQ.jl @@ -1,8 +1,15 @@ module LibPQ -export status, reset!, execute, prepare, async_execute, cancel, - num_columns, num_rows, num_params, num_affected_rows - +export status, + reset!, + execute, + prepare, + async_execute, + cancel, + num_columns, + num_rows, + num_params, + num_affected_rows using Base: Semaphore, acquire, release using Base.Iterators: zip, product @@ -21,17 +28,17 @@ using Memento: Memento, getlogger, warn, info, error, debug using OffsetArrays using TimeZones -const Parameter = Union{String, Missing} +const Parameter = Union{String,Missing} const LOGGER = getlogger(@__MODULE__) function __init__() INTERVAL_REGEX[] = _interval_regex() Memento.register(LOGGER) + return nothing end # Docstring template for types using DocStringExtensions -@template TYPES = - """ +@template TYPES = """ $(TYPEDEF) $(DOCSTRING) @@ -50,7 +57,7 @@ module libpq_c include(joinpath(@__DIR__, "..", "deps", "deps.jl")) function __init__() - check_deps() + return check_deps() end else using LibPQ_jll @@ -83,6 +90,9 @@ LibPQ.jl. """ const LIBPQ_CONVERSIONS = PQConversions() +const BINARY = true +const TEXT = false + include("connections.jl") include("results.jl") include("statements.jl") diff --git a/src/asyncresults.jl b/src/asyncresults.jl index b6600e06..c02a4b0d 100644 --- a/src/asyncresults.jl +++ b/src/asyncresults.jl @@ -1,5 +1,5 @@ "An asynchronous PostgreSQL query" -mutable struct AsyncResult +mutable struct AsyncResult{BinaryFormat} "The LibPQ.jl Connection used for the query" jl_conn::Connection @@ -12,13 +12,15 @@ mutable struct AsyncResult "Task which errors or returns a LibPQ.jl Result which is created once available" result_task::Task - function AsyncResult(jl_conn::Connection, result_kwargs::Ref) - return new(jl_conn, result_kwargs, false) + function AsyncResult{BinaryFormat}( + jl_conn::Connection, result_kwargs::Ref + ) where BinaryFormat + return new{BinaryFormat}(jl_conn, result_kwargs, false) end end -function AsyncResult(jl_conn::Connection; kwargs...) - return AsyncResult(jl_conn, Ref(kwargs)) +function AsyncResult{BinaryFormat}(jl_conn::Connection; kwargs...) where BinaryFormat + return AsyncResult{BinaryFormat}(jl_conn, Ref(kwargs)) end function Base.show(io::IO, async_result::AsyncResult) @@ -31,7 +33,7 @@ function Base.show(io::IO, async_result::AsyncResult) else "in progress" end - print(io, typeof(async_result), " (", status, ")") + return print(io, typeof(async_result), " (", status, ")") end """ @@ -50,16 +52,16 @@ The result returned will be the [`Result`](@ref) of the last query run (the only using parameters). Any errors produced by the queries will be thrown together in a `CompositeException`. """ -function handle_result(async_result::AsyncResult; throw_error=true) +function handle_result( + async_result::AsyncResult{BinaryFormat}; throw_error=true +) where BinaryFormat errors = [] result = nothing for result_ptr in _consume(async_result.jl_conn) try result = handle_result( - Result( - result_ptr, - async_result.jl_conn; - async_result.result_kwargs[]... + Result{BinaryFormat}( + result_ptr, async_result.jl_conn; async_result.result_kwargs[]... ); throw_error=throw_error, ) @@ -102,9 +104,11 @@ function _consume(jl_conn::Connection) _cancel(jl_conn) end - last_log == curr && debug(LOGGER, "Waiting to read from connection $(jl_conn.conn)") + last_log == curr && + debug(LOGGER, "Waiting to read from connection $(jl_conn.conn)") wait(watcher) - last_log == curr && debug(LOGGER, "Consuming input from connection $(jl_conn.conn)") + last_log == curr && + debug(LOGGER, "Consuming input from connection $(jl_conn.conn)") success = libpq_c.PQconsumeInput(jl_conn.conn) == 1 !success && error(LOGGER, Errors.PQConnectionError(jl_conn)) @@ -116,8 +120,8 @@ function _consume(jl_conn::Connection) return result_ptrs else result_num = length(result_ptrs) + 1 - debug(LOGGER, - "Saving result $result_num from connection $(jl_conn.conn)" + debug( + LOGGER, "Saving result $result_num from connection $(jl_conn.conn)" ) push!(result_ptrs, result_ptr) end @@ -126,9 +130,12 @@ function _consume(jl_conn::Connection) catch err if err isa Base.IOError && err.code == -9 # EBADF debug(() -> sprint(showerror, err), LOGGER) - error(LOGGER, Errors.JLConnectionError( - "PostgreSQL connection socket was unexpectedly closed" - )) + error( + LOGGER, + Errors.JLConnectionError( + "PostgreSQL connection socket was unexpectedly closed" + ), + ) else rethrow(err) end @@ -147,7 +154,7 @@ function cancel(async_result::AsyncResult) # the actual cancellation will be triggered in the main loop of _consume # which will call `_cancel` on the `Connection` async_result.should_cancel = true - return + return nothing end function _cancel(jl_conn::Connection) @@ -158,9 +165,10 @@ function _cancel(jl_conn::Connection) errbuf = zeros(UInt8, errbuf_size) success = libpq_c.PQcancel(cancel_ptr, pointer(errbuf), errbuf_size) == 1 if !success - warn(LOGGER, Errors.JLConnectionError( - "Failed cancelling query: $(String(errbuf))" - )) + warn( + LOGGER, + Errors.JLConnectionError("Failed cancelling query: $(String(errbuf))"), + ) else debug(LOGGER, "Cancelled query for connection $(jl_conn.conn)") end @@ -180,6 +188,7 @@ Base.close(async_result::AsyncResult) = cancel(async_result) jl_conn::Connection, query::AbstractString, [parameters::Union{AbstractVector, Tuple},] + binary_format::Bool=false, kwargs... ) -> AsyncResult @@ -198,6 +207,10 @@ If multiple `AsyncResult`s use the same `Connection`, they will execute serially `async_execute` optionally takes a `parameters` vector which passes query parameters as strings to PostgreSQL. + +`binary_format`, when set to true, will transfer the data in binary format. +Support for binary transfer is currently limited to a subset of basic data types. + Queries without parameters can contain multiple SQL statements, and the result of the final statement is returned. Any errors which occur during executed statements will be bundled together in a @@ -207,7 +220,15 @@ As is normal for `Task`s, any exceptions will be thrown when calling `wait` or ` """ function async_execute end -function async_execute(jl_conn::Connection, query::AbstractString; kwargs...) +function async_execute(conn, query; binary_format=false, kwargs...) + if binary_format + async_execute(conn, query, []; binary_format=binary_format, kwargs...) + else + _multi_async_execute(conn, query; kwargs...) + end +end + +function _multi_async_execute(jl_conn::Connection, query::AbstractString; kwargs...) async_result = _async_execute(jl_conn; kwargs...) do jl_conn _async_submit(jl_conn.conn, query) end @@ -218,23 +239,28 @@ end function async_execute( jl_conn::Connection, query::AbstractString, - parameters::Union{AbstractVector, Tuple}; - kwargs... + parameters::Union{AbstractVector,Tuple}; + binary_format::Bool=false, + kwargs..., ) string_params = string_parameters(parameters) pointer_params = parameter_pointers(string_params) - async_result = _async_execute(jl_conn; kwargs...) do jl_conn - _async_submit(jl_conn.conn, query, pointer_params) - end + async_result = _async_execute(jl_conn; binary_format=binary_format, kwargs...) do jl_conn + _async_submit(jl_conn.conn, query, pointer_params; binary_format=binary_format) + end return async_result end function _async_execute( - submission_fn::Function, jl_conn::Connection; throw_error::Bool=true, kwargs... + submission_fn::Function, + jl_conn::Connection; + binary_format::Bool=false, + throw_error::Bool=true, + kwargs..., ) - async_result = AsyncResult(jl_conn; kwargs...) + async_result = AsyncResult{binary_format}(jl_conn; kwargs...) async_result.result_task = @async lock(jl_conn) do jl_conn.async_result = async_result @@ -262,7 +288,8 @@ end function _async_submit( conn_ptr::Ptr{libpq_c.PGconn}, query::AbstractString, - parameters::Vector{Ptr{UInt8}}, + parameters::Vector{Ptr{UInt8}}; + binary_format::Bool=false, ) num_params = length(parameters) @@ -274,7 +301,7 @@ function _async_submit( parameters, C_NULL, # paramLengths is ignored for text format parameters zeros(Cint, num_params), # all parameters in text format - zero(Cint), # return result in text format + Cint(binary_format), # return result in text or binary format ) return send_status == 1 diff --git a/src/parsing.jl b/src/parsing.jl index 2c798315..7f5062c0 100644 --- a/src/parsing.jl +++ b/src/parsing.jl @@ -1,5 +1,5 @@ "A wrapper for one value in a PostgreSQL result." -struct PQValue{OID} +struct PQValue{OID,BinaryFormat} "PostgreSQL result" jl_result::Result @@ -9,11 +9,16 @@ struct PQValue{OID} "Column index of the result (0-indexed)" col::Cint - function PQValue{OID}(jl_result::Result, row::Integer, col::Integer) where OID - return new{OID}(jl_result, row - 1, col - 1) + function PQValue{OID}( + jl_result::Result{BinaryFormat}, row::Integer, col::Integer + ) where {OID,BinaryFormat} + return new{OID,BinaryFormat}(jl_result, row - 1, col - 1) end end +const PQTextValue{OID} = PQValue{OID,TEXT} +const PQBinaryValue{OID} = PQValue{OID,BINARY} + """ PQValue(jl_result::Result, row::Integer, col::Integer) -> PQValue PQValue{OID}(jl_result::Result, row::Integer, col::Integer) -> PQValue{OID} @@ -42,7 +47,7 @@ end num_bytes(pqv::PQValue) -> Cint The length in bytes of the `PQValue`'s corresponding data. -LibPQ.jl currently always uses text format, so this is equivalent to C's `strlen`. +When a query uses `LibPQ.TEXT` format, this is equivalent to C's `strlen`. See also: [`data_pointer`](@ref) """ @@ -119,7 +124,7 @@ By default, this uses any existing `parse` method for parsing a value of type `T You can implement default PostgreSQL-specific parsing for a given type by overriding `pqparse`. """ -Base.parse(::Type{T}, pqv::PQValue) where {T} = pqparse(T, string_view(pqv)) +Base.parse(::Type{T}, pqv::PQValue) where T = pqparse(T, string_view(pqv)) """ LibPQ.pqparse(::Type{T}, str::AbstractString) -> T @@ -130,23 +135,39 @@ This is used to parse PostgreSQL's output format. function pqparse end # Fallback method -pqparse(::Type{T}, str::AbstractString) where {T} = parse(T, str) +pqparse(::Type{T}, str::AbstractString) where T = parse(T, str) # allow parsing as a Symbol anything which works as a String pqparse(::Type{Symbol}, str::AbstractString) = Symbol(str) +function generate_binary_parser(symbol) + @eval function Base.parse( + ::Type{T}, pqv::PQBinaryValue{$(oid(symbol))} + ) where T<:Number + return convert( + T, ntoh(unsafe_load(Ptr{$(_DEFAULT_TYPE_MAP[symbol])}(data_pointer(pqv)))) + ) + end +end + ## integers _DEFAULT_TYPE_MAP[:int2] = Int16 _DEFAULT_TYPE_MAP[:int4] = Int32 _DEFAULT_TYPE_MAP[:int8] = Int64 +foreach(generate_binary_parser, (:int2, :int4, :int8)) + ## floating point _DEFAULT_TYPE_MAP[:float4] = Float32 _DEFAULT_TYPE_MAP[:float8] = Float64 +foreach(generate_binary_parser, (:float4, :float8)) + ## oid _DEFAULT_TYPE_MAP[:oid] = Oid +generate_binary_parser(:oid) + ## numeric _DEFAULT_TYPE_MAP[:numeric] = Decimal @@ -168,8 +189,8 @@ pqparse(::Type{Char}, str::AbstractString) = Char(pqparse(PQChar, str)) _DEFAULT_TYPE_MAP[:bytea] = Vector{UInt8} # Needs it's own `parse` method as it uses bytes_view instead of string_view -function Base.parse(::Type{Vector{UInt8}}, pqv::PQValue{PQ_SYSTEM_TYPES[:bytea]}) - pqparse(Vector{UInt8}, bytes_view(pqv)) +function Base.parse(::Type{Vector{UInt8}}, pqv::PQTextValue{PQ_SYSTEM_TYPES[:bytea]}) + return pqparse(Vector{UInt8}, bytes_view(pqv)) end function pqparse(::Type{Vector{UInt8}}, bytes::Array{UInt8,1}) @@ -203,6 +224,10 @@ function pqparse(::Type{Bool}, str::AbstractString) end end +function Base.parse(::Type{Bool}, pqv::PQBinaryValue{oid(:bool)}) + return unsafe_load(Ptr{_DEFAULT_TYPE_MAP[:bool]}(data_pointer(pqv))) +end + ## dates and times # ISO, YMD @@ -245,7 +270,7 @@ function pqparse(::Type{ZonedDateTime}, str::AbstractString) return ZonedDateTime(typemin(DateTime), tz"UTC") end - for fmt in TIMESTAMPTZ_FORMATS[1:end-1] + for fmt in TIMESTAMPTZ_FORMATS[1:(end - 1)] parsed = tryparse(ZonedDateTime, str, fmt) parsed !== nothing && return parsed end @@ -280,7 +305,7 @@ function pqparse(::Type{Time}, str::AbstractString) end # InfExtendedTime support for Dates.TimeType -function pqparse(::Type{InfExtendedTime{T}}, str::AbstractString) where {T<:Dates.TimeType} +function pqparse(::Type{InfExtendedTime{T}}, str::AbstractString) where T<:Dates.TimeType if str == "infinity" return InfExtendedTime{T}(∞) elseif str == "-infinity" @@ -291,12 +316,12 @@ function pqparse(::Type{InfExtendedTime{T}}, str::AbstractString) where {T<:Date end # UNIX timestamps -function Base.parse(::Type{DateTime}, pqv::PQValue{PQ_SYSTEM_TYPES[:int8]}) - unix2datetime(parse(Int64, pqv)) +function Base.parse(::Type{DateTime}, pqv::PQTextValue{PQ_SYSTEM_TYPES[:int8]}) + return unix2datetime(parse(Int64, pqv)) end -function Base.parse(::Type{ZonedDateTime}, pqv::PQValue{PQ_SYSTEM_TYPES[:int8]}) - TimeZones.unix2zdt(parse(Int64, pqv)) +function Base.parse(::Type{ZonedDateTime}, pqv::PQTextValue{PQ_SYSTEM_TYPES[:int8]}) + return TimeZones.unix2zdt(parse(Int64, pqv)) end ## intervals @@ -320,7 +345,8 @@ function _interval_regex() for long_type in (Hour, Minute) print(io, _field_match(long_type)) end - print(io, + print( + io, _field_match(Second, "(?-?\\d+)(?:\\.(?\\d{1,9}))?"), ) print(io, ")?\$") @@ -368,7 +394,7 @@ function pqparse(::Type{Dates.CompoundPeriod}, str::AbstractString) period_coeff = fld1(len, 3) period_type = frac_periods[period_coeff] # field regex prevents BoundsError - frac_seconds = parse(Int, frac_seconds_str) * 10 ^ (3 * period_coeff - len) + frac_seconds = parse(Int, frac_seconds_str) * 10^(3 * period_coeff - len) if frac_seconds != 0 push!(periods, period_type(frac_seconds)) end @@ -386,7 +412,7 @@ _DEFAULT_TYPE_MAP[:tsrange] = Interval{DateTime} _DEFAULT_TYPE_MAP[:tstzrange] = Interval{ZonedDateTime} _DEFAULT_TYPE_MAP[:daterange] = Interval{Date} -function pqparse(::Type{Interval{T}}, str::AbstractString) where {T} +function pqparse(::Type{Interval{T}}, str::AbstractString) where T str == "empty" && return Interval{T}() return parse(Interval{T}, str; element_parser=pqparse) end @@ -395,15 +421,16 @@ end # numeric arrays never have double quotes and always use ',' as a separator parse_numeric_element(::Type{T}, str) where T = parse(T, str) -parse_numeric_element(::Type{Union{T, Missing}}, str) where T = - str == "NULL" ? missing : parse(T, str) +function parse_numeric_element(::Type{Union{T,Missing}}, str) where T + return str == "NULL" ? missing : parse(T, str) +end function parse_numeric_array(eltype::Type{T}, str::AbstractString) where T eq_ind = findfirst(isequal('='), str) if eq_ind !== nothing - offset_str = str[1:eq_ind-1] - range_strs = split(str[1:eq_ind-1], ['[',']']; keepempty=false) + offset_str = str[1:(eq_ind - 1)] + range_strs = split(str[1:(eq_ind - 1)], ['[', ']']; keepempty=false) ranges = map(range_strs) do range_str lower, upper = split(range_str, ':'; limit=2) @@ -411,7 +438,7 @@ function parse_numeric_array(eltype::Type{T}, str::AbstractString) where T end arr = OffsetArray{T}(undef, ranges...) - el_iter = eachmatch(r"[^\}\{,]+", str[eq_ind+1:end]) + el_iter = eachmatch(r"[^\}\{,]+", str[(eq_ind + 1):end]) else arr = Array{T}(undef, array_size(str)...) el_iter = eachmatch(r"[^\}\{,]+", str) @@ -464,7 +491,7 @@ end for pq_eltype in ("int2", "int4", "int8", "float4", "float8", "oid", "numeric") array_oid = PQ_SYSTEM_TYPES[Symbol("_$pq_eltype")] jl_type = _DEFAULT_TYPE_MAP[Symbol(pq_eltype)] - jl_missingtype = Union{jl_type, Missing} + jl_missingtype = Union{jl_type,Missing} # could be an OffsetArray or Array of any dimensionality _DEFAULT_TYPE_MAP[array_oid] = AbstractArray{jl_missingtype} @@ -472,24 +499,23 @@ for pq_eltype in ("int2", "int4", "int8", "float4", "float8", "oid", "numeric") for jl_eltype in (jl_type, jl_missingtype) @eval function pqparse( ::Type{A}, str::AbstractString - ) where A <: AbstractArray{$jl_eltype} - parse_numeric_array($jl_eltype, str)::A + ) where A<:AbstractArray{$jl_eltype} + return parse_numeric_array($jl_eltype, str)::A end end end -struct FallbackConversion <: AbstractDict{Tuple{Oid, Type}, Base.Callable} -end +struct FallbackConversion <: AbstractDict{Tuple{Oid,Type},Base.Callable} end -function Base.getindex(cmap::FallbackConversion, oid_typ::Tuple{Integer, Type}) +function Base.getindex(cmap::FallbackConversion, oid_typ::Tuple{Integer,Type}) _, typ = oid_typ return function parse_type(pqv::PQValue) - parse(typ, pqv) + return parse(typ, pqv) end end -Base.haskey(cmap::FallbackConversion, oid_typ::Tuple{Integer, Type}) = true +Base.haskey(cmap::FallbackConversion, oid_typ::Tuple{Integer,Type}) = true """ A fallback conversion mapping (like [`PQConversions`](@ref) which holds a single function diff --git a/src/results.jl b/src/results.jl index d171675e..04b59fe9 100644 --- a/src/results.jl +++ b/src/results.jl @@ -1,5 +1,5 @@ "A result from a PostgreSQL database query" -mutable struct Result +mutable struct Result{BinaryFormat} "A pointer to a libpq PGresult object (C_NULL if cleared)" result::Ptr{libpq_c.PGresult} @@ -22,21 +22,18 @@ mutable struct Result column_names::Vector{String} # TODO: attach encoding per https://wiki.postgresql.org/wiki/Driver_development#Result_object_and_client_encoding - function Result( + function Result{BinaryFormat}( result::Ptr{libpq_c.PGresult}, jl_conn::Connection; - column_types::Union{AbstractDict, AbstractVector}=ColumnTypeMap(), + column_types::Union{AbstractDict,AbstractVector}=ColumnTypeMap(), type_map::AbstractDict=PQTypeMap(), conversions::AbstractDict=PQConversions(), not_null=false, - ) - jl_result = new(result, Atomic{Bool}(result == C_NULL)) + ) where BinaryFormat + jl_result = new{BinaryFormat}(result, Atomic{Bool}(result == C_NULL)) type_lookup = LayerDict( - PQTypeMap(type_map), - jl_conn.type_map, - LIBPQ_TYPE_MAP, - _DEFAULT_TYPE_MAP, + PQTypeMap(type_map), jl_conn.type_map, LIBPQ_TYPE_MAP, _DEFAULT_TYPE_MAP ) func_lookup = LayerDict( @@ -48,8 +45,8 @@ mutable struct Result ) jl_result.column_oids = col_oids = map(1:num_columns(jl_result)) do col_num - libpq_c.PQftype(jl_result.result, col_num - 1) - end + libpq_c.PQftype(jl_result.result, col_num - 1) + end jl_result.column_names = map(1:num_columns(jl_result)) do col_num unsafe_string(libpq_c.PQfname(jl_result.result, col_num - 1)) @@ -60,16 +57,22 @@ mutable struct Result column_type_map[column_number(jl_result, k)] = v end - jl_result.column_types = col_types = collect(Type, imap(enumerate(col_oids)) do itr - col_num, col_oid = itr - get(column_type_map, col_num) do - get(type_lookup, col_oid, String) - end - end) - - jl_result.column_funcs = collect(Base.Callable, imap(col_oids, col_types) do oid, typ - func_lookup[(oid, typ)] - end) + jl_result.column_types = col_types = collect( + Type, + imap(enumerate(col_oids)) do itr + col_num, col_oid = itr + get(column_type_map, col_num) do + get(type_lookup, col_oid, String) + end + end, + ) + + jl_result.column_funcs = collect( + Base.Callable, + imap(col_oids, col_types) do oid, typ + func_lookup[(oid, typ)] + end, + ) # figure out which columns the user says may contain nulls if not_null isa Bool @@ -77,9 +80,11 @@ mutable struct Result elseif not_null isa AbstractArray if eltype(not_null) === Bool if length(not_null) != length(col_types) - throw(ArgumentError( - "The length of keyword argument not_null, when an array, must be equal to the number of columns" - )) + throw( + ArgumentError( + "The length of keyword argument not_null, when an array, must be equal to the number of columns", + ), + ) end jl_result.not_null = not_null @@ -95,9 +100,11 @@ mutable struct Result end end else - throw(ArgumentError( - "Unsupported type $(typeof(not_null)) for keyword argument not_null" - )) + throw( + ArgumentError( + "Unsupported type $(typeof(not_null)) for keyword argument not_null" + ), + ) end finalizer(close, jl_result) @@ -106,6 +113,8 @@ mutable struct Result end end +Result(args...; kwargs...) = Result{TEXT}(args...; kwargs...) + """ show(io::IO, jl_result::Result) @@ -146,15 +155,16 @@ end function _verbose_error_message(jl_result::Result) msg_ptr = libpq_c.PQresultVerboseErrorMessage( - jl_result.result, - libpq_c.PQERRORS_VERBOSE, - libpq_c.PQSHOW_CONTEXT_ALWAYS, + jl_result.result, libpq_c.PQERRORS_VERBOSE, libpq_c.PQSHOW_CONTEXT_ALWAYS ) if msg_ptr == C_NULL - error(LOGGER, Errors.JLResultError( - "libpq could not allocate memory for the result error message" - )) + error( + LOGGER, + Errors.JLResultError( + "libpq could not allocate memory for the result error message" + ), + ) end msg = unsafe_string(msg_ptr) @@ -179,7 +189,7 @@ julia> LibPQ.error_field(result, LibPQ.libpq_c.PG_DIAG_SEVERITY) "ERROR" ``` """ -function error_field(jl_result::Result, field_code::Union{Char, Integer}) +function error_field(jl_result::Result, field_code::Union{Char,Integer}) ret = libpq_c.PQresultErrorField(jl_result.result, field_code) return ret == C_NULL ? nothing : unsafe_string(ret) end @@ -245,6 +255,7 @@ end {jl_conn::Connection, query::AbstractString | stmt::Statement}, [parameters::Union{AbstractVector, Tuple},] throw_error::Bool=true, + binary_format::Bool=false, column_types::AbstractDict=ColumnTypeMap(), type_map::AbstractDict=LibPQ.PQTypeMap(), conversions::AbstractDict=LibPQ.PQConversions(), @@ -257,21 +268,29 @@ fatal error or unreadable response. The query may be passed as `Connection` and `AbstractString` (SQL) arguments, or as a `Statement`. -`execute` optionally takes a `parameters` vector which passes query parameters as strings to -PostgreSQL. +`execute` optionally takes a `parameters` vector which passes query parameters as +strings to PostgreSQL. `column_types` accepts type overrides for columns in the result which take priority over those in `type_map`. For information on the `column_types`, `type_map`, and `conversions` arguments, see [Type Conversions](@ref typeconv). + +`binary_format`, when set to true, will transfer the data in binary format. +Support for binary transfer is currently limited to a subset of basic data types. """ function execute end -function execute( - jl_conn::Connection, - query::AbstractString; - throw_error::Bool=true, - kwargs... +function execute(conn, query; binary_format=false, kwargs...) + if binary_format + execute(conn, query, []; binary_format=binary_format, kwargs...) + else + _multi_execute(conn, query; kwargs...) + end +end + +function _multi_execute( + jl_conn::Connection, query::AbstractString; throw_error::Bool=true, kwargs... ) result = lock(jl_conn) do _execute(jl_conn.conn, query) @@ -283,18 +302,21 @@ end function execute( jl_conn::Connection, query::AbstractString, - parameters::Union{AbstractVector, Tuple}; + parameters::Union{AbstractVector,Tuple}; throw_error::Bool=true, - kwargs... + binary_format::Bool=false, + kwargs..., ) string_params = string_parameters(parameters) pointer_params = parameter_pointers(string_params) result = lock(jl_conn) do - _execute(jl_conn.conn, query, pointer_params) + _execute(jl_conn.conn, query, pointer_params; binary_format=binary_format) end - return handle_result(Result(result, jl_conn; kwargs...); throw_error=throw_error) + return handle_result( + Result{binary_format}(result, jl_conn; kwargs...); throw_error=throw_error + ) end function _execute(conn_ptr::Ptr{libpq_c.PGconn}, query::AbstractString) @@ -304,7 +326,8 @@ end function _execute( conn_ptr::Ptr{libpq_c.PGconn}, query::AbstractString, - parameters::Vector{Ptr{UInt8}}, + parameters::Vector{Ptr{UInt8}}; + binary_format::Bool=false, ) num_params = length(parameters) @@ -316,7 +339,7 @@ function _execute( parameters, C_NULL, # paramLengths is ignored for text format parameters zeros(Cint, num_params), # all parameters in text format - zero(Cint), # return result in text format + Cint(binary_format), # return result in text format ) end @@ -337,11 +360,11 @@ string_parameters(parameters::AbstractVector) = map(string_parameter, parameters # vector which might contain missings function string_parameters(parameters::AbstractVector{>:Missing}) - collect( - Union{String, Missing}, + return collect( + Union{String,Missing}, imap(parameters) do parameter ismissing(parameter) ? missing : string_parameter(parameter) - end + end, ) end @@ -352,14 +375,13 @@ function string_parameter(parameter::AbstractVector) print(io, "{") join(io, (_array_element(el) for el in parameter), ",") print(io, "}") - String(take!(io)) + return String(take!(io)) end _array_element(el::AbstractString) = "\"$el\"" _array_element(el::Missing) = "NULL" _array_element(el) = string_parameter(el) - function string_parameter(interval::AbstractInterval) io = IOBuffer() L, R = bounds_types(interval) @@ -371,7 +393,7 @@ function string_parameter(interval::AbstractInterval) return String(take!(io)) end -function string_parameter(parameter::InfExtendedTime{T}) where {T<:Dates.TimeType} +function string_parameter(parameter::InfExtendedTime{T}) where T<:Dates.TimeType if isinf(parameter) return isposinf(parameter) ? "infinity" : "-infinity" else @@ -403,7 +425,7 @@ If this result did not come from the description of a prepared statement, return """ function num_params(jl_result::Result)::Int # todo: check cleared? - libpq_c.PQnparams(jl_result.result) + return libpq_c.PQnparams(jl_result.result) end """ @@ -414,7 +436,7 @@ This will be 0 if the query would never return data. """ function num_rows(jl_result::Result)::Int # todo: check cleared? - libpq_c.PQntuples(jl_result.result) + return libpq_c.PQntuples(jl_result.result) end """ @@ -443,7 +465,7 @@ This will be 0 if the query would never return data. """ function num_columns(jl_result::Result)::Int # todo: check cleared? - libpq_c.PQnfields(jl_result.result) + return libpq_c.PQnfields(jl_result.result) end """ @@ -467,7 +489,7 @@ column_names(jl_result::Result) = copy(jl_result.column_names) Return the index (1-based) of the column named `column_name`. """ -function column_number(jl_result::Result, column_name::Union{AbstractString, Symbol})::Int +function column_number(jl_result::Result, column_name::Union{AbstractString,Symbol})::Int return something(findfirst(isequal(String(column_name)), jl_result.column_names), 0) end diff --git a/src/statements.jl b/src/statements.jl index 832b8cd2..b1a05bd9 100644 --- a/src/statements.jl +++ b/src/statements.jl @@ -53,7 +53,7 @@ function prepare(jl_conn::Connection, query::AbstractString) description = handle_result(Result(result, jl_conn); throw_error=true) - Statement(jl_conn, uid, query, description, num_params(description)) + return Statement(jl_conn, uid, query, description, num_params(description)) end """ @@ -62,12 +62,8 @@ end Show a PostgreSQL prepared statement and its query. """ function Base.show(io::IO, stmt::Statement) - print( - io, - "PostgreSQL prepared statement named ", - stmt.name, - " with query ", - stmt.query, + return print( + io, "PostgreSQL prepared statement named ", stmt.name, " with query ", stmt.query ) end @@ -92,7 +88,7 @@ Return the name of the column at index `column_number` (1-based) that would be r executing the prepared statement. """ function column_name(stmt::Statement, column_number::Integer) - column_name(stmt.description, column_number) + return column_name(stmt.description, column_number) end """ @@ -110,42 +106,48 @@ Return the index (1-based) of the column named `column_name` that would be retur executing the prepared statement. """ function column_number(stmt::Statement, column_name::AbstractString) - column_number(stmt.description, column_name) + return column_number(stmt.description, column_name) end function execute( stmt::Statement, - parameters::Union{AbstractVector, Tuple}; + parameters::Union{AbstractVector,Tuple}; throw_error::Bool=true, - kwargs... + binary_format::Bool=false, + kwargs..., ) num_params = length(parameters) string_params = string_parameters(parameters) pointer_params = parameter_pointers(string_params) result = lock(stmt.jl_conn) do - _execute_prepared(stmt.jl_conn.conn, stmt.name, pointer_params) + _execute_prepared( + stmt.jl_conn.conn, stmt.name, pointer_params; binary_format=binary_format + ) end - return handle_result(Result(result, stmt.jl_conn; kwargs...); throw_error=throw_error) + return handle_result( + Result{binary_format}(result, stmt.jl_conn; kwargs...); throw_error=throw_error + ) end function execute( - stmt::Statement; - throw_error::Bool=true, - kwargs... + stmt::Statement; throw_error::Bool=true, binary_format::Bool=false, kwargs... ) result = lock(stmt.jl_conn) do - _execute_prepared(stmt.jl_conn.conn, stmt.name) + _execute_prepared(stmt.jl_conn.conn, stmt.name; binary_format=binary_format) end - return handle_result(Result(result, stmt.jl_conn; kwargs...); throw_error=throw_error) + return handle_result( + Result{binary_format}(result, stmt.jl_conn; kwargs...); throw_error=throw_error + ) end function _execute_prepared( conn_ptr::Ptr{libpq_c.PGconn}, stmt_name::AbstractString, - parameters::Vector{Ptr{UInt8}}=Ptr{UInt8}[], + parameters::Vector{Ptr{UInt8}}=Ptr{UInt8}[]; + binary_format::Bool=false, ) num_params = length(parameters) @@ -156,6 +158,6 @@ function _execute_prepared( num_params == 0 ? C_NULL : parameters, C_NULL, # paramLengths is ignored for text format parameters num_params == 0 ? C_NULL : zeros(Cint, num_params), # all parameters in text format - zero(Cint), # return result in text format + Cint(binary_format), # return result in text format ) end diff --git a/src/tables.jl b/src/tables.jl index 3ed1c121..1acf906d 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -143,7 +143,7 @@ julia> LibPQ.load!( julia> execute(conn, "COMMIT;"); ``` """ -function load!(table::T, connection::Connection, query::AbstractString) where {T} +function load!(table::T, connection::Connection, query::AbstractString) where T rows = Tables.rows(table) stmt = prepare(connection, query) state = iterate(rows) diff --git a/test/runtests.jl b/test/runtests.jl index 6e3c8812..f2498f93 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -222,21 +222,67 @@ end @test LibPQ.column_number(stmt, "no_nulls") == 0 @test LibPQ.column_names(stmt) == [] - result = execute( - conn, - "SELECT no_nulls, yes_nulls FROM libpqjl_test ORDER BY no_nulls DESC;"; - throw_error=true, - ) - @test status(result) == LibPQ.libpq_c.PGRES_TUPLES_OK - @test LibPQ.num_rows(result) == 2 - @test LibPQ.num_columns(result) == 2 + @testset for binary_format in (LibPQ.TEXT, LibPQ.BINARY) + result = execute( + conn, + "SELECT no_nulls, yes_nulls FROM libpqjl_test ORDER BY no_nulls DESC;"; + throw_error=true, + binary_format=binary_format, + ) + @test status(result) == LibPQ.libpq_c.PGRES_TUPLES_OK + @test LibPQ.num_rows(result) == 2 + @test LibPQ.num_columns(result) == 2 - table_data = columntable(result) - @test table_data[:no_nulls] == data[:no_nulls] - @test table_data[:yes_nulls][1] == data[:yes_nulls][1] - @test table_data[:yes_nulls][2] === missing + table_data = columntable(result) + @test table_data[:no_nulls] == data[:no_nulls] + @test table_data[:yes_nulls][1] == data[:yes_nulls][1] + @test table_data[:yes_nulls][2] === missing - close(result) + close(result) + + ar = async_execute( + conn, + "SELECT no_nulls, yes_nulls FROM libpqjl_test ORDER BY no_nulls DESC;"; + throw_error=true, + binary_format=binary_format, + ) + result = fetch(ar) + @test LibPQ.num_rows(result) == 2 + @test LibPQ.num_columns(result) == 2 + + table_data = columntable(result) + @test table_data[:no_nulls] == data[:no_nulls] + @test table_data[:yes_nulls][1] == data[:yes_nulls][1] + @test table_data[:yes_nulls][2] === missing + + stmt = prepare(conn, "SELECT no_nulls, yes_nulls FROM libpqjl_test ORDER BY no_nulls DESC;") + + result = execute(stmt; binary_format=binary_format, throw_error=true) + + @test LibPQ.num_rows(result) == 2 + @test LibPQ.num_columns(result) == 2 + + table_data = columntable(result) + @test table_data[:no_nulls] == data[:no_nulls] + @test table_data[:yes_nulls][1] == data[:yes_nulls][1] + @test table_data[:yes_nulls][2] === missing + + close(result) + + stmt = prepare(conn, "SELECT no_nulls, yes_nulls FROM libpqjl_test ORDER BY no_nulls DESC;") + + result = execute(stmt, []; binary_format=binary_format, throw_error=true) + + @test LibPQ.num_rows(result) == 2 + @test LibPQ.num_columns(result) == 2 + + table_data = columntable(result) + @test table_data[:no_nulls] == data[:no_nulls] + @test table_data[:yes_nulls][1] == data[:yes_nulls][1] + @test table_data[:yes_nulls][2] === missing + + close(result) + end result = execute( conn, @@ -1102,192 +1148,240 @@ end end @testset "Parsing" begin - @testset "Default Types" begin - conn = LibPQ.Connection("dbname=postgres user=$DATABASE_USER"; throw_error=true) - - test_data = [ - ("3", Cint(3)), - ("3::int8", Int64(3)), - ("3::int4", Int32(3)), - ("3::int2", Int16(3)), - ("3::float8", Float64(3)), - ("3::float4", Float32(3)), - ("3::oid", LibPQ.Oid(3)), - ("3::numeric", decimal("3")), - ("$(BigFloat(pi))::numeric", decimal(BigFloat(pi))), - ("$(big"4608230166434464229556241992703")::numeric", parse(Decimal, "4608230166434464229556241992703")), - ("E'\\\\xDEADBEEF'::bytea", hex2bytes("DEADBEEF")), - ("E'\\\\000'::bytea", UInt8[0o000]), - ("E'\\\\047'::bytea", UInt8[0o047]), - ("E'\\''::bytea", UInt8[0o047]), - ("E'\\\\134'::bytea", UInt8[0o134]), - ("E'\\\\\\\\'::bytea", UInt8[0o134]), - ("E'\\\\001'::bytea", UInt8[0o001]), - ("E'\\\\176'::bytea", UInt8[0o176]), - ("'hello'::char(10)", "hello"), - ("'hello '::char(10)", "hello"), - ("'hello '::varchar(10)", "hello "), - ("'3'::\"char\"", LibPQ.PQChar('3')), - ("'t'::bool", true), - ("'T'::bool", true), - ("'true'::bool", true), - ("'TRUE'::bool", true), - ("'tRuE'::bool", true), - ("'y'::bool", true), - ("'YEs'::bool", true), - ("'on'::bool", true), - ("1::bool", true), - ("true", true), - ("'f'::bool", false), - ("'F'::bool", false), - ("'false'::bool", false), - ("'FALSE'::bool", false), - ("'fAlsE'::bool", false), - ("'n'::bool", false), - ("'nO'::bool", false), - ("'off'::bool", false), - ("0::bool", false), - ("false", false), - ("TIMESTAMP '2004-10-19 10:23:54'", DateTime(2004, 10, 19, 10, 23, 54)), - ("TIMESTAMP '2004-10-19 10:23:54.123'", DateTime(2004, 10, 19, 10, 23, 54,123)), - ("TIMESTAMP '2004-10-19 10:23:54.1234'", DateTime(2004, 10, 19, 10, 23, 54,123)), - ("'infinity'::timestamp", typemax(DateTime)), - ("'-infinity'::timestamp", typemin(DateTime)), - ("'epoch'::timestamp", DateTime(1970, 1, 1, 0, 0, 0)), - ("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54-00'", ZonedDateTime(2004, 10, 19, 10, 23, 54, tz"UTC")), - ("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54-02'", ZonedDateTime(2004, 10, 19, 10, 23, 54, tz"UTC-2")), - ("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+10'", ZonedDateTime(2004, 10, 19, 10, 23, 54, tz"UTC+10")), - ("'infinity'::timestamptz", ZonedDateTime(typemax(DateTime), tz"UTC")), - ("'-infinity'::timestamptz", ZonedDateTime(typemin(DateTime), tz"UTC")), - ("'epoch'::timestamptz", ZonedDateTime(1970, 1, 1, 0, 0, 0, tz"UTC")), - ("DATE '2017-01-31'", Date(2017, 1, 31)), - ("'infinity'::date", typemax(Date)), - ("'-infinity'::date", typemin(Date)), - ("TIME '13:13:13.131'", Time(13, 13, 13, 131)), - ("TIME '13:13:13.131242'", Time(13, 13, 13, 131)), - ("TIME '01:01:01'", Time(1, 1, 1)), - ("'allballs'::time", Time(0, 0, 0)), - ("INTERVAL '1 year 2 months 3 days 4 hours 5 minutes 6 seconds'", Dates.CompoundPeriod(Period[Year(1), Month(2), Day(3), Hour(4), Minute(5), Second(6)])), - ("INTERVAL '6.1 seconds'", Dates.CompoundPeriod(Period[Second(6), Millisecond(100)])), - ("INTERVAL '6.01 seconds'", Dates.CompoundPeriod(Period[Second(6), Millisecond(10)])), - ("INTERVAL '6.001 seconds'", Dates.CompoundPeriod(Period[Second(6), Millisecond(1)])), - ("INTERVAL '6.0001 seconds'", Dates.CompoundPeriod(Period[Second(6), Microsecond(100)])), - ("INTERVAL '6.1001 seconds'", Dates.CompoundPeriod(Period[Second(6), Microsecond(100100)])), - ("INTERVAL '1000 years 7 weeks'", Dates.CompoundPeriod(Period[Year(1000), Day(7 * 7)])), - ("INTERVAL '1 day -1 hour'", Dates.CompoundPeriod(Period[Day(1), Hour(-1)])), - ("INTERVAL '-1 month 1 day'", Dates.CompoundPeriod(Period[Month(-1), Day(1)])), - ("'{{{1,2,3},{4,5,6}}}'::int2[]", Array{Union{Int16, Missing}}(reshape(Int16[1 2 3; 4 5 6], 1, 2, 3))), - ("'{}'::int2[]", Union{Missing, Int16}[]), - ("'{{{1,2,3},{4,5,6}}}'::int4[]", Array{Union{Int32, Missing}}(reshape(Int32[1 2 3; 4 5 6], 1, 2, 3))), - ("'{{{1,2,3},{4,5,6}}}'::int8[]", Array{Union{Int64, Missing}}(reshape(Int64[1 2 3; 4 5 6], 1, 2, 3))), - ("'{{{NULL,2,3},{4,NULL,6}}}'::int8[]", Array{Union{Int64, Missing}}(reshape(Union{Int64, Missing}[missing 2 3; 4 missing 6], 1, 2, 3))), - ("'{{{1,2,3},{4,5,6}}}'::float4[]", Array{Union{Float32, Missing}}(reshape(Float32[1 2 3; 4 5 6], 1, 2, 3))), - ("'{{{1,2,3},{4,5,6}}}'::float8[]", Array{Union{Float64, Missing}}(reshape(Float64[1 2 3; 4 5 6], 1, 2, 3))), - ("'{{{NULL,2,3},{4,NULL,6}}}'::float8[]", Array{Union{Float64, Missing}}(reshape(Union{Float64, Missing}[missing 2 3; 4 missing 6], 1, 2, 3))), - ("'{{{1,2,3},{4,5,6}}}'::oid[]", Array{Union{LibPQ.Oid, Missing}}(reshape(LibPQ.Oid[1 2 3; 4 5 6], 1, 2, 3))), - ("'{{{1,2,3},{4,5,6}}}'::numeric[]", Array{Union{Decimal, Missing}}(reshape(Decimal[1 2 3; 4 5 6], 1, 2, 3))), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int2[]", copyto!(OffsetArray{Union{Missing, Int16}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int4[]", copyto!(OffsetArray{Union{Missing, Int32}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int8[]", copyto!(OffsetArray{Union{Missing, Int64}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::float4[]", copyto!(OffsetArray{Union{Missing, Float32}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::float8[]", copyto!(OffsetArray{Union{Missing, Float64}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::oid[]", copyto!(OffsetArray{Union{Missing, LibPQ.Oid}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::numeric[]", copyto!(OffsetArray{Union{Missing, Decimal}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), - ("'[3,7)'::int4range", Interval{Int32, Closed, Open}(3, 7)), - ("'(3,7)'::int4range", Interval{Int32, Closed, Open}(4, 7)), - ("'[4,4]'::int4range", Interval{Int32, Closed, Open}(4, 5)), - ("'[4,4)'::int4range", Interval{Int32}()), # Empty interval - ("'[3,7)'::int8range", Interval{Int64, Closed, Open}(3, 7)), - ("'(3,7)'::int8range", Interval{Int64, Closed, Open}(4, 7)), - ("'[4,4]'::int8range", Interval{Int64, Closed, Open}(4, 5)), - ("'[4,4)'::int8range", Interval{Int64}()), # Empty interval - ("'[11.1,22.2)'::numrange", Interval{Decimal, Closed, Open}(11.1, 22.2)), - ("'[2010-01-01 14:30, 2010-01-01 15:30)'::tsrange", Interval{Closed, Open}(DateTime(2010, 1, 1, 14, 30), DateTime(2010, 1, 1, 15, 30))), - ("'[2010-01-01 14:30-00, 2010-01-01 15:30-00)'::tstzrange", Interval{Closed, Open}(ZonedDateTime(2010, 1, 1, 14, 30, tz"UTC"), ZonedDateTime(2010, 1, 1, 15, 30, tz"UTC"))), - ("'[2004-10-19 10:23:54-02, 2004-10-19 11:23:54-02)'::tstzrange", Interval{Closed, Open}(ZonedDateTime(2004, 10, 19, 12, 23, 54, tz"UTC"), ZonedDateTime(2004, 10, 19, 13, 23, 54, tz"UTC"))), - ("'[2004-10-19 10:23:54-02, Infinity)'::tstzrange", Interval{Closed, Open}(ZonedDateTime(2004, 10, 19, 12, 23, 54, tz"UTC"), ZonedDateTime(typemax(DateTime), tz"UTC"))), - ("'(-Infinity, Infinity)'::tstzrange", Interval{Open, Open}(ZonedDateTime(typemin(DateTime), tz"UTC"), ZonedDateTime(typemax(DateTime), tz"UTC"))), - ("'[2018/01/01, 2018/02/02)'::daterange", Interval{Closed, Open}(Date(2018, 1, 1), Date(2018, 2, 2))), - # Unbounded ranges - ("'[3,)'::int4range", Interval{Int32}(3, nothing)), - ("'[3,]'::int4range", Interval{Int32}(3, nothing)), - ("'(3,)'::int4range", Interval{Int32, Closed, Unbounded}(4, nothing)), # Postgres normalizes `(3,)` to `[4,)` - ("'(,3)'::int4range", Interval{Int32, Unbounded, Open}(nothing, 3)), - ("'(,3]'::int4range", Interval{Int32, Unbounded, Open}(nothing, 4)), # Postgres normalizes `(,3]` to `(,4)` - ("'[,]'::int4range", Interval{Int32, Unbounded, Unbounded}(nothing, nothing)), - ("'(,)'::int4range", Interval{Int32, Unbounded, Unbounded}(nothing, nothing)), - ("'[2010-01-01 14:30,)'::tsrange", Interval{Closed, Unbounded}(DateTime(2010, 1, 1, 14, 30), nothing)), - ("'[,2010-01-01 15:30-00)'::tstzrange", Interval{Unbounded, Open}(nothing, ZonedDateTime(2010, 1, 1, 15, 30, tz"UTC"))), - ("'[2018/01/01,]'::daterange", Interval{Closed, Unbounded}(Date(2018, 1, 1), nothing)), - ] - @testset for (test_str, data) in test_data - result = execute(conn, "SELECT $test_str;") - - try - @test LibPQ.num_rows(result) == 1 - @test LibPQ.num_columns(result) == 1 - @test LibPQ.column_types(result)[1] >: typeof(data) - - oid = LibPQ.column_oids(result)[1] - func = result.column_funcs[1] - parsed = func(LibPQ.PQValue{oid}(result, 1, 1)) - @test isequal(parsed, data) - @test typeof(parsed) == typeof(data) - finally - close(result) + binary_not_implemented_pgtypes = [ + "numeric", + "timestamp", + "timestamptz", + "tstzrange", + ] + binary_not_implemented_types = [ + Decimal, + DateTime, + ZonedDateTime, + Date, + Time, + Dates.CompoundPeriod, + Array, + OffsetArray, + Interval, + ] + + @testset for binary_format in (LibPQ.TEXT, LibPQ.BINARY) + @testset "Default Types" begin + conn = LibPQ.Connection("dbname=postgres user=$DATABASE_USER"; throw_error=true) + + test_data = [ + ("3", Cint(3)), + ("3::int8", Int64(3)), + ("3::int4", Int32(3)), + ("3::int2", Int16(3)), + ("3::float8", Float64(3)), + ("3::float4", Float32(3)), + ("3::oid", LibPQ.Oid(3)), + ("3::numeric", decimal("3")), + ("$(BigFloat(pi))::numeric", decimal(BigFloat(pi))), + ("$(big"4608230166434464229556241992703")::numeric", parse(Decimal, "4608230166434464229556241992703")), + ("E'\\\\xDEADBEEF'::bytea", hex2bytes("DEADBEEF")), + ("E'\\\\000'::bytea", UInt8[0o000]), + ("E'\\\\047'::bytea", UInt8[0o047]), + ("E'\\''::bytea", UInt8[0o047]), + ("E'\\\\134'::bytea", UInt8[0o134]), + ("E'\\\\\\\\'::bytea", UInt8[0o134]), + ("E'\\\\001'::bytea", UInt8[0o001]), + ("E'\\\\176'::bytea", UInt8[0o176]), + ("'hello'::char(10)", "hello"), + ("'hello '::char(10)", "hello"), + ("'hello '::varchar(10)", "hello "), + ("'3'::\"char\"", LibPQ.PQChar('3')), + ("'t'::bool", true), + ("'T'::bool", true), + ("'true'::bool", true), + ("'TRUE'::bool", true), + ("'tRuE'::bool", true), + ("'y'::bool", true), + ("'YEs'::bool", true), + ("'on'::bool", true), + ("1::bool", true), + ("true", true), + ("'f'::bool", false), + ("'F'::bool", false), + ("'false'::bool", false), + ("'FALSE'::bool", false), + ("'fAlsE'::bool", false), + ("'n'::bool", false), + ("'nO'::bool", false), + ("'off'::bool", false), + ("0::bool", false), + ("false", false), + ("TIMESTAMP '2004-10-19 10:23:54'", DateTime(2004, 10, 19, 10, 23, 54)), + ("TIMESTAMP '2004-10-19 10:23:54.123'", DateTime(2004, 10, 19, 10, 23, 54,123)), + ("TIMESTAMP '2004-10-19 10:23:54.1234'", DateTime(2004, 10, 19, 10, 23, 54,123)), + ("'infinity'::timestamp", typemax(DateTime)), + ("'-infinity'::timestamp", typemin(DateTime)), + ("'epoch'::timestamp", DateTime(1970, 1, 1, 0, 0, 0)), + ("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54-00'", ZonedDateTime(2004, 10, 19, 10, 23, 54, tz"UTC")), + ("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54-02'", ZonedDateTime(2004, 10, 19, 10, 23, 54, tz"UTC-2")), + ("TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54+10'", ZonedDateTime(2004, 10, 19, 10, 23, 54, tz"UTC+10")), + ("'infinity'::timestamptz", ZonedDateTime(typemax(DateTime), tz"UTC")), + ("'-infinity'::timestamptz", ZonedDateTime(typemin(DateTime), tz"UTC")), + ("'epoch'::timestamptz", ZonedDateTime(1970, 1, 1, 0, 0, 0, tz"UTC")), + ("DATE '2017-01-31'", Date(2017, 1, 31)), + ("'infinity'::date", typemax(Date)), + ("'-infinity'::date", typemin(Date)), + ("TIME '13:13:13.131'", Time(13, 13, 13, 131)), + ("TIME '13:13:13.131242'", Time(13, 13, 13, 131)), + ("TIME '01:01:01'", Time(1, 1, 1)), + ("'allballs'::time", Time(0, 0, 0)), + ("INTERVAL '1 year 2 months 3 days 4 hours 5 minutes 6 seconds'", Dates.CompoundPeriod(Period[Year(1), Month(2), Day(3), Hour(4), Minute(5), Second(6)])), + ("INTERVAL '6.1 seconds'", Dates.CompoundPeriod(Period[Second(6), Millisecond(100)])), + ("INTERVAL '6.01 seconds'", Dates.CompoundPeriod(Period[Second(6), Millisecond(10)])), + ("INTERVAL '6.001 seconds'", Dates.CompoundPeriod(Period[Second(6), Millisecond(1)])), + ("INTERVAL '6.0001 seconds'", Dates.CompoundPeriod(Period[Second(6), Microsecond(100)])), + ("INTERVAL '6.1001 seconds'", Dates.CompoundPeriod(Period[Second(6), Microsecond(100100)])), + ("INTERVAL '1000 years 7 weeks'", Dates.CompoundPeriod(Period[Year(1000), Day(7 * 7)])), + ("INTERVAL '1 day -1 hour'", Dates.CompoundPeriod(Period[Day(1), Hour(-1)])), + ("INTERVAL '-1 month 1 day'", Dates.CompoundPeriod(Period[Month(-1), Day(1)])), + ("'{{{1,2,3},{4,5,6}}}'::int2[]", Array{Union{Int16, Missing}}(reshape(Int16[1 2 3; 4 5 6], 1, 2, 3))), + ("'{}'::int2[]", Union{Missing, Int16}[]), + ("'{{{1,2,3},{4,5,6}}}'::int4[]", Array{Union{Int32, Missing}}(reshape(Int32[1 2 3; 4 5 6], 1, 2, 3))), + ("'{{{1,2,3},{4,5,6}}}'::int8[]", Array{Union{Int64, Missing}}(reshape(Int64[1 2 3; 4 5 6], 1, 2, 3))), + ("'{{{NULL,2,3},{4,NULL,6}}}'::int8[]", Array{Union{Int64, Missing}}(reshape(Union{Int64, Missing}[missing 2 3; 4 missing 6], 1, 2, 3))), + ("'{{{1,2,3},{4,5,6}}}'::float4[]", Array{Union{Float32, Missing}}(reshape(Float32[1 2 3; 4 5 6], 1, 2, 3))), + ("'{{{1,2,3},{4,5,6}}}'::float8[]", Array{Union{Float64, Missing}}(reshape(Float64[1 2 3; 4 5 6], 1, 2, 3))), + ("'{{{NULL,2,3},{4,NULL,6}}}'::float8[]", Array{Union{Float64, Missing}}(reshape(Union{Float64, Missing}[missing 2 3; 4 missing 6], 1, 2, 3))), + ("'{{{1,2,3},{4,5,6}}}'::oid[]", Array{Union{LibPQ.Oid, Missing}}(reshape(LibPQ.Oid[1 2 3; 4 5 6], 1, 2, 3))), + ("'{{{1,2,3},{4,5,6}}}'::numeric[]", Array{Union{Decimal, Missing}}(reshape(Decimal[1 2 3; 4 5 6], 1, 2, 3))), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int2[]", copyto!(OffsetArray{Union{Missing, Int16}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int4[]", copyto!(OffsetArray{Union{Missing, Int32}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int8[]", copyto!(OffsetArray{Union{Missing, Int64}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::float4[]", copyto!(OffsetArray{Union{Missing, Float32}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::float8[]", copyto!(OffsetArray{Union{Missing, Float64}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::oid[]", copyto!(OffsetArray{Union{Missing, LibPQ.Oid}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::numeric[]", copyto!(OffsetArray{Union{Missing, Decimal}}(undef, 1:1, -2:-1, 3:5), [1 2 3; 4 5 6])), + ("'[3,7)'::int4range", Interval{Int32, Closed, Open}(3, 7)), + ("'(3,7)'::int4range", Interval{Int32, Closed, Open}(4, 7)), + ("'[4,4]'::int4range", Interval{Int32, Closed, Open}(4, 5)), + ("'[4,4)'::int4range", Interval{Int32}()), # Empty interval + ("'[3,7)'::int8range", Interval{Int64, Closed, Open}(3, 7)), + ("'(3,7)'::int8range", Interval{Int64, Closed, Open}(4, 7)), + ("'[4,4]'::int8range", Interval{Int64, Closed, Open}(4, 5)), + ("'[4,4)'::int8range", Interval{Int64}()), # Empty interval + ("'[11.1,22.2)'::numrange", Interval{Decimal, Closed, Open}(11.1, 22.2)), + ("'[2010-01-01 14:30, 2010-01-01 15:30)'::tsrange", Interval{Closed, Open}(DateTime(2010, 1, 1, 14, 30), DateTime(2010, 1, 1, 15, 30))), + ("'[2010-01-01 14:30-00, 2010-01-01 15:30-00)'::tstzrange", Interval{Closed, Open}(ZonedDateTime(2010, 1, 1, 14, 30, tz"UTC"), ZonedDateTime(2010, 1, 1, 15, 30, tz"UTC"))), + ("'[2004-10-19 10:23:54-02, 2004-10-19 11:23:54-02)'::tstzrange", Interval{Closed, Open}(ZonedDateTime(2004, 10, 19, 12, 23, 54, tz"UTC"), ZonedDateTime(2004, 10, 19, 13, 23, 54, tz"UTC"))), + ("'[2004-10-19 10:23:54-02, Infinity)'::tstzrange", Interval{Closed, Open}(ZonedDateTime(2004, 10, 19, 12, 23, 54, tz"UTC"), ZonedDateTime(typemax(DateTime), tz"UTC"))), + ("'(-Infinity, Infinity)'::tstzrange", Interval{Open, Open}(ZonedDateTime(typemin(DateTime), tz"UTC"), ZonedDateTime(typemax(DateTime), tz"UTC"))), + ("'[2018/01/01, 2018/02/02)'::daterange", Interval{Closed, Open}(Date(2018, 1, 1), Date(2018, 2, 2))), + # Unbounded ranges + ("'[3,)'::int4range", Interval{Int32}(3, nothing)), + ("'[3,]'::int4range", Interval{Int32}(3, nothing)), + ("'(3,)'::int4range", Interval{Int32, Closed, Unbounded}(4, nothing)), # Postgres normalizes `(3,)` to `[4,)` + ("'(,3)'::int4range", Interval{Int32, Unbounded, Open}(nothing, 3)), + ("'(,3]'::int4range", Interval{Int32, Unbounded, Open}(nothing, 4)), # Postgres normalizes `(,3]` to `(,4)` + ("'[,]'::int4range", Interval{Int32, Unbounded, Unbounded}(nothing, nothing)), + ("'(,)'::int4range", Interval{Int32, Unbounded, Unbounded}(nothing, nothing)), + ("'[2010-01-01 14:30,)'::tsrange", Interval{Closed, Unbounded}(DateTime(2010, 1, 1, 14, 30), nothing)), + ("'[,2010-01-01 15:30-00)'::tstzrange", Interval{Unbounded, Open}(nothing, ZonedDateTime(2010, 1, 1, 15, 30, tz"UTC"))), + ("'[2018/01/01,]'::daterange", Interval{Closed, Unbounded}(Date(2018, 1, 1), nothing)), + ] + + @testset for (test_str, data) in test_data + result = execute( + conn, + "SELECT $test_str;"; + binary_format=binary_format, + ) + + try + @test LibPQ.num_rows(result) == 1 + @test LibPQ.num_columns(result) == 1 + @test LibPQ.column_types(result)[1] >: typeof(data) + + oid = LibPQ.column_oids(result)[1] + func = result.column_funcs[1] + if binary_format && any(T -> data isa T, binary_not_implemented_types) + @test_broken parsed = func(LibPQ.PQValue{oid}(result, 1, 1)) + @test_broken isequal(parsed, data) + @test_broken typeof(parsed) == typeof(data) + @test_broken parsed_no_oid = func(LibPQ.PQValue(result, 1, 1)) + @test_broken isequal(parsed_no_oid, data) + @test_broken typeof(parsed_no_oid) == typeof(data) + else + parsed = func(LibPQ.PQValue{oid}(result, 1, 1)) + @test isequal(parsed, data) + @test typeof(parsed) == typeof(data) + parsed_no_oid = func(LibPQ.PQValue(result, 1, 1)) + @test isequal(parsed_no_oid, data) + @test typeof(parsed_no_oid) == typeof(data) + end + finally + close(result) + end end - end - - close(conn) - end - - @testset "Specified Types" begin - conn = LibPQ.Connection("dbname=postgres user=$DATABASE_USER"; throw_error=true) - - test_data = [ - ("3", UInt, UInt(3)), - ("3::int8", UInt16, UInt16(3)), - ("3::int4", Int32, Int32(3)), - ("3::int2", UInt8, UInt8(3)), - ("3::oid", UInt32, UInt32(3)), - ("3::numeric", Float64, 3.0), - ("'3'::\"char\"", Char, '3'), - ("'foobar'", Symbol, :foobar), - ("0::int8", DateTime, DateTime(1970, 1, 1, 0)), - ("0::int8", ZonedDateTime, ZonedDateTime(1970, 1, 1, 0, tz"UTC")), - ("'{{{1,2,3},{4,5,6}}}'::int2[]", AbstractArray{Int16}, reshape(Int16[1 2 3; 4 5 6], 1, 2, 3)), - ("'infinity'::timestamp", InfExtendedTime{Date}, InfExtendedTime{Date}(∞)), - ("'-infinity'::timestamp", InfExtendedTime{Date}, InfExtendedTime{Date}(-∞)), - ("'infinity'::timestamptz", InfExtendedTime{ZonedDateTime}, InfExtendedTime{ZonedDateTime}(∞)), - ("'-infinity'::timestamptz", InfExtendedTime{ZonedDateTime}, InfExtendedTime{ZonedDateTime}(-∞)), - ("'[2004-10-19 10:23:54-02, infinity)'::tstzrange", Interval{InfExtendedTime{ZonedDateTime}}, Interval{Closed, Open}(ZonedDateTime(2004, 10, 19, 12, 23, 54, tz"UTC"), ∞)), - ("'(-infinity, infinity)'::tstzrange", Interval{InfExtendedTime{ZonedDateTime}}, Interval{InfExtendedTime{ZonedDateTime}, Open, Open}(-∞, ∞)), - ] - for (test_str, typ, data) in test_data - result = execute( - conn, - "SELECT $test_str;", - column_types=Dict(1 => typ), - ) + close(conn) + end - try - @test LibPQ.num_rows(result) == 1 - @test LibPQ.num_columns(result) == 1 - @test LibPQ.column_types(result)[1] >: typeof(data) - - oid = LibPQ.column_oids(result)[1] - func = result.column_funcs[1] - parsed = func(LibPQ.PQValue{oid}(result, 1, 1)) - @test parsed == data - @test typeof(parsed) == typeof(data) - finally - close(result) + @testset "Specified Types" begin + conn = LibPQ.Connection("dbname=postgres user=$DATABASE_USER"; throw_error=true) + + test_data = [ + ("3", UInt, UInt(3)), + ("3::int8", UInt16, UInt16(3)), + ("3::int4", Int32, Int32(3)), + ("3::int2", UInt8, UInt8(3)), + ("3::oid", UInt32, UInt32(3)), + ("3::numeric", Float64, 3.0), + ("'3'::\"char\"", Char, '3'), + ("'foobar'", Symbol, :foobar), + ("0::int8", DateTime, DateTime(1970, 1, 1, 0)), + ("0::int8", ZonedDateTime, ZonedDateTime(1970, 1, 1, 0, tz"UTC")), + ("'{{{1,2,3},{4,5,6}}}'::int2[]", AbstractArray{Int16}, reshape(Int16[1 2 3; 4 5 6], 1, 2, 3)), + ("'infinity'::timestamp", InfExtendedTime{Date}, InfExtendedTime{Date}(∞)), + ("'-infinity'::timestamp", InfExtendedTime{Date}, InfExtendedTime{Date}(-∞)), + ("'infinity'::timestamptz", InfExtendedTime{ZonedDateTime}, InfExtendedTime{ZonedDateTime}(∞)), + ("'-infinity'::timestamptz", InfExtendedTime{ZonedDateTime}, InfExtendedTime{ZonedDateTime}(-∞)), + ("'[2004-10-19 10:23:54-02, infinity)'::tstzrange", Interval{InfExtendedTime{ZonedDateTime}}, Interval{Closed, Open}(ZonedDateTime(2004, 10, 19, 12, 23, 54, tz"UTC"), ∞)), + ("'(-infinity, infinity)'::tstzrange", Interval{InfExtendedTime{ZonedDateTime}}, Interval{InfExtendedTime{ZonedDateTime}, Open, Open}(-∞, ∞)), + ] + + for (test_str, typ, data) in test_data + result = execute( + conn, + "SELECT $test_str;", + column_types=Dict(1 => typ), + binary_format=binary_format, + ) + + try + @test LibPQ.num_rows(result) == 1 + @test LibPQ.num_columns(result) == 1 + @test LibPQ.column_types(result)[1] >: typeof(data) + + oid = LibPQ.column_oids(result)[1] + func = result.column_funcs[1] + + if binary_format && ( + any(T -> data isa T, binary_not_implemented_types) || + any(occursin.(binary_not_implemented_pgtypes, test_str)) + ) + @test_broken parsed = func(LibPQ.PQValue{oid}(result, 1, 1)) + @test_broken parsed == data + @test_broken typeof(parsed) == typeof(data) + else + parsed = func(LibPQ.PQValue{oid}(result, 1, 1)) + @test parsed == data + @test typeof(parsed) == typeof(data) + end + finally + close(result) + end end - end - close(conn) + close(conn) + end end @testset "Interval Regex" begin