Implement LZ4 transport compression

.gitignore

@@ -2,3 +2,6 @@
 docs/build
 docs/Manifest.toml
 Manifest.toml
+
+# editor(s)
+.vscode

Project.toml

@@ -5,6 +5,7 @@ version = "0.2.2"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DecFP = "55939f99-70c6-5e9b-8bb0-5071ed7d61fd"

docs/src/index.md

@@ -31,7 +31,6 @@ Usage examples can be found on the [usage page](@ref Usage).
 
 ## Limitations
 
-- Transfer compression is currently not implemented
 - Timezone conversion of `DateTime` / `DateTime64` for columns that have a
   timezone assigned in ClickHouse doesn't happen automatically. All DateTime
   objects are naive, meaning they aren't timezone aware. For reasoning, see

src/ClickHouse.jl

@@ -8,6 +8,10 @@ include("columns/columns.jl")
 include("Connect.jl")
 include("Query.jl")
 
+export Compression
+export COMPRESSION_NONE
+export COMPRESSION_LZ4
+export COMPRESSION_CHECKSUM_ONLY
 export ClickHouseSock
 export Block
 export select

src/Connect.jl

@@ -95,7 +95,8 @@ function connect(
     password::AbstractString = "",
     connection_timeout = DBMS_DEFAULT_CONNECT_TIMEOUT,
     max_insert_block_size = DBMS_DEFAULT_MAX_INSERT_BLOCK,
-    send_buffer_size = DBMS_DEFAULT_BUFFER_SIZE
+    send_buffer_size = DBMS_DEFAULT_BUFFER_SIZE,
+    compression::Compression = COMPRESSION_NONE
 )::ClickHouseSock
     sock = ClickHouseSock(
         nothing,
@@ -107,7 +108,8 @@ function connect(
             password = password,
             connection_timeout = connection_timeout,
             max_insert_block_size = max_insert_block_size,
-            send_buffer_size = send_buffer_size
+            send_buffer_size = send_buffer_size,
+            compression = compression,
         )
     )
 

src/Exceptions.jl

@@ -3,4 +3,7 @@ struct ClickHouseServerException <: Exception
     code::Int
     name::String
     message::String
-end
+end
+
+"""checksum (compressed block hash values) don't match"""
+struct ChecksumError <: Exception end

src/Query.jl

@@ -8,7 +8,8 @@ import Sockets
 # ============================================================================ #
 
 function write_query(sock::ClickHouseSock, query::AbstractString)::Nothing
-    query = ClientQuery("", ClientInfo(), "", 2, 0, query)
+    compression = compression_enabled(sock.settings)
+    query = ClientQuery("", ClientInfo(), "", 2, compression, query)
     write_packet(sock, query)
     write_packet(sock, make_block())
     nothing

src/tcp/BasicIO.jl

@@ -71,4 +71,15 @@ chwrite(sock::ClickHouseSock, x::AbstractVector{T}) where T <: Number =
     write(sock.io, x)
 
 chwrite(sock::ClickHouseSock, x::AbstractVector{String}) =
-    foreach(x -> chwrite(sock, x), x)
+    foreach(x -> chwrite(sock, x), x)
+
+
+# Compression bytes
+
+function chwrite(sock::ClickHouseSock, compression::ClickHouse.Compression)
+    chwrite(sock, UInt8(compression))
+end
+
+function chread(sock::ClickHouseSock, ::Type{ClickHouse.Compression})
+    Compression(chread(sock, UInt8))
+end