Skip to content

Commit

Permalink
reduce amount of small Strings we make
Browse files Browse the repository at this point in the history
  • Loading branch information
Moelf committed Mar 15, 2024
1 parent 0d42f7b commit f6d62f5
Show file tree
Hide file tree
Showing 4 changed files with 14 additions and 15 deletions.
9 changes: 4 additions & 5 deletions src/RNTuple/bootstrap.jl
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ function decompress_bytes!(uncomp_data, compbytes, NTarget::Integer)
compression_header = unpack(io, CompressionHeader)
cname, _, compbytes, uncompbytes = unpack(compression_header)
rawbytes = read(io, compbytes)
if cname == "L4"
if cname == @SVector UInt8['L', '4']
# skip checksum which is 8 bytes
# original: lz4_decompress(rawbytes[9:end], uncompbytes)
input = @view rawbytes[9:end]
Expand All @@ -82,17 +82,16 @@ function decompress_bytes!(uncomp_data, compbytes, NTarget::Integer)
output_ptr = pointer(uncomp_data) + fulfilled
output_size = uncompbytes
_decompress_lz4!(input_ptr, input_size, output_ptr, output_size)
elseif cname == "ZL"
elseif cname == @SVector UInt8['Z', 'L']
output = @view(uncomp_data[fulfilled+1:fulfilled+uncompbytes])
zlib_decompress!(Decompressor(), output, rawbytes, uncompbytes)
elseif cname == "XZ"
elseif cname == @SVector UInt8['X', 'Z']
@view(uncomp_data[fulfilled+1:fulfilled+uncompbytes]) .= transcode(XzDecompressor, rawbytes)
elseif cname == "ZS"
elseif cname == @SVector UInt8['Z', 'S']
@view(uncomp_data[fulfilled+1:fulfilled+uncompbytes]) .= transcode(ZstdDecompressor, rawbytes)
else
error("Unsupported compression type '$(String(compression_header.algo))'")
end

fulfilled += uncompbytes
end
return uncomp_data
Expand Down
10 changes: 5 additions & 5 deletions src/streamers.jl
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,17 @@ function Streamers(io)
# notice our `TKey` size is not the same as official TKey, can't use sizeof()
skipped = position(io) - start
compressedbytes = read(io, tkey.fNbytes - skipped)
cname = String(compression_header.algo)
cname = compression_header.algo

if cname == "ZL"
if cname == @SVector UInt8['Z', 'L']
output = Vector{UInt8}(undef, tkey.fObjlen)
zlib_decompress!(Decompressor(), output, compressedbytes, length(output))
IOBuffer(output)
elseif cname == "XZ"
elseif cname == @SVector UInt8['X', 'Z']
IOBuffer(transcode(XzDecompressor, compressedbytes))
elseif cname == "ZS"
elseif cname == @SVector UInt8['Z', 'S']
IOBuffer(transcode(ZstdDecompressor, compressedbytes))
elseif cname == "L4"
elseif cname == @SVector UInt8['L', '4']
IOBuffer(lz4_decompress(compressedbytes[9:end], tkey.fObjlen))
else
error("Unsupported compression type '$(String(compression_header.algo))'")
Expand Down
8 changes: 4 additions & 4 deletions src/types.jl
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ function decompress_datastreambytes(compbytes, tkey)
@debug "Compression type: $(cname)"
@debug "Compressed/uncompressed size in bytes: $(compbytes) / $(uncompbytes)"

if cname == "L4"
if cname == @SVector UInt8['L', '4']
# skip checksum which is 8 bytes
# original: lz4_decompress(rawbytes[9:end], uncompbytes)
input = @view rawbytes[9:end]
Expand All @@ -163,12 +163,12 @@ function decompress_datastreambytes(compbytes, tkey)
output_ptr = pointer(uncomp_data) + fulfilled
output_size = uncompbytes
_decompress_lz4!(input_ptr, input_size, output_ptr, output_size)
elseif cname == "ZL"
elseif cname == @SVector UInt8['Z', 'L']
output = @view(uncomp_data[fulfilled+1:fulfilled+uncompbytes])
zlib_decompress!(Decompressor(), output, rawbytes, uncompbytes)
elseif cname == "XZ"
elseif cname == @SVector UInt8['X', 'Z']
@view(uncomp_data[fulfilled+1:fulfilled+uncompbytes]) .= transcode(XzDecompressor, rawbytes)
elseif cname == "ZS"
elseif cname == @SVector UInt8['Z', 'S']
@view(uncomp_data[fulfilled+1:fulfilled+uncompbytes]) .= transcode(ZstdDecompressor, rawbytes)
else
error("Unsupported compression type '$(String(compression_header.algo))'")
Expand Down
2 changes: 1 addition & 1 deletion src/utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ Return the following information:
- compressedbytes and uncompressedbytes according to [uproot3](https://github.com/scikit-hep/uproot3/blob/54f5151fb7c686c3a161fbe44b9f299e482f346b/uproot3/source/compressed.py#L132)
"""
function unpack(x::CompressionHeader)
algname = String(x.algo)
algname = x.algo
# shift without casting to `Int` will give you 0x00 because we're shifting 0 bits into UInt8
compressedbytes = x.c1 + (Int(x.c2) << 8) + (Int(x.c3) << 16)
uncompressedbytes = x.u1 + (Int(x.u2) << 8) + (Int(x.u3) << 16)
Expand Down

0 comments on commit f6d62f5

Please sign in to comment.