From 24ece08ad549b24eb85865135d87f9de2fc86752 Mon Sep 17 00:00:00 2001 From: Moelf Date: Fri, 31 May 2024 22:56:41 -0400 Subject: [PATCH 01/14] tests --- src/RNTuple/Writing/TFileWriter.jl | 60 ++++++++++++++++++++++++++++++ src/RNTuple/bootstrap.jl | 3 ++ 2 files changed, 63 insertions(+) create mode 100644 src/RNTuple/Writing/TFileWriter.jl diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl new file mode 100644 index 00000000..8fe91c57 --- /dev/null +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -0,0 +1,60 @@ + +""" +struct FilePreamble + identifier::SVector{4, UInt8} # Root file identifier ("root") + fVersion::Int32 # File format version +end +""" +dummy_FilePreamble = UnROOT.FilePreamble(b"root", 0xf745) + +""" +struct FileHeader32 + fBEGIN::Int32 # Pointer to first data record + fEND::UInt32 # Pointer to first free word at the EOF + fSeekFree::UInt32 # Pointer to FREE data record + fNbytesFree::Int32 # Number of bytes in FREE data record + nfree::Int32 # Number of free data records + fNbytesName::Int32 # Number of bytes in TNamed at creation time + fUnits::UInt8 # Number of bytes for file pointers + fCompress::Int32 # Compression level and algorithm + fSeekInfo::UInt32 # Pointer to TStreamerInfo record + fNbytesInfo::Int32 # Number of bytes in TStreamerInfo record + fUUID::SVector{18, UInt8} # Universal Unique ID +end +""" +# dummy_FileHeader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +dummy_FileHeader = [ + 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +dummy_padding1 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] + +""" +struct ROOTDirectoryHeader32 + fVersion::Int16 + fDatimeC::UInt32 + fDatimeM::UInt32 + fNbytesKeys::Int32 + fNbytesName::Int32 + fSeekDir::Int32 + fSeekParent::Int32 + fSeekKeys::Int32 +end +""" +# dummy_DirectoryHeader32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) +dummy_DirectoryHeader32 = [ + 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, + 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x01, +] + +dummy_padding2 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] diff --git a/src/RNTuple/bootstrap.jl b/src/RNTuple/bootstrap.jl index b2130e88..a226cb6a 100644 --- a/src/RNTuple/bootstrap.jl +++ b/src/RNTuple/bootstrap.jl @@ -15,6 +15,8 @@ end function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) local_io = datastream(io, tkey) + @show position(io) + @show dump(tkey) skip(local_io, 6) anchor = ROOT_3a3a_Experimental_3a3a_RNTuple(; fVersionEpoch = readtype(local_io, UInt16), @@ -30,6 +32,7 @@ function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) fChecksum = readtype(local_io, UInt64), ) + @show dump(anchor) header_bytes = decompress_bytes(read_seek_nb(io, anchor.fSeekHeader, anchor.fNBytesHeader), anchor.fLenHeader) header_io = IOBuffer(header_bytes) header = _rntuple_read(header_io, RNTupleEnvelope{RNTupleHeader}) From e38145817864756c27991db959b860e778265683 Mon Sep 17 00:00:00 2001 From: Pere Mato Date: Mon, 3 Jun 2024 17:34:37 +0200 Subject: [PATCH 02/14] Replacing xrootdgo by XRootD.jl (#340) * Replace xrootdgo_jll by XRootD.jl --- Project.toml | 4 ++-- src/root.jl | 6 +----- src/streamsource.jl | 42 ++++++++++++++++-------------------------- test/runtests.jl | 42 ++++++++++++++++++++++-------------------- 4 files changed, 41 insertions(+), 53 deletions(-) diff --git a/Project.toml b/Project.toml index 5be877dc..162ec744 100644 --- a/Project.toml +++ b/Project.toml @@ -27,8 +27,8 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +XRootD = "164e3b87-dc46-4cbc-97a6-5b310108fce0" XXHashNative = "e5d8e439-e7fa-4681-9c12-1c64bda517be" -xrootdgo_jll = "9d84c17e-11f2-50ef-8cc9-e9701362097f" [compat] AbstractTrees = "^0.4" @@ -61,9 +61,9 @@ StructArrays = "0.6" TOML = "^1.0" Tables = "^1.9" Test = "^1.0" +XRootD = "^0.1" XXHashNative = "^1.0.1" julia = "^1.7" -xrootdgo_jll = "=0.32.1, =0.34.1" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" diff --git a/src/root.jl b/src/root.jl index e4273bc6..a65f5659 100644 --- a/src/root.jl +++ b/src/root.jl @@ -66,11 +66,7 @@ function ROOTFile(filename::AbstractString; customstructs = Dict("TLorentzVector fobj = if startswith(filename, r"https?://") HTTPStream(filename) elseif startswith(filename, "root://") - length(findall("//", filename)) < 2 && error("The xrootd URL is illegal: missing the '//' separator between the server and the path (e.g. 'root://server:1234//path/to/file.root')") - sep_idx = findall("//", filename)[2] - baseurl = filename[8:first(sep_idx)-1] - filepath = filename[last(sep_idx):end] - XRDStream(baseurl, filepath, "go") + XRDStream(filename) else !isfile(filename) && throw(SystemError("opening file $filename", 2)) MmapStream(filename) diff --git a/src/streamsource.jl b/src/streamsource.jl index 048cafb0..65ef3d9b 100644 --- a/src/streamsource.jl +++ b/src/streamsource.jl @@ -1,8 +1,8 @@ -using xrootdgo_jll +using XRootD.XrdCl import HTTP mutable struct XRDStream - gofile_id::Cstring # used as key to a global `map` on the Go side + file::File # encapsulates a XRootD.XrdCl!File object seekloc::Int size::Int end @@ -130,50 +130,40 @@ function Base.read(fobj::SourceStream) read(fobj, fobj.size - fobj.seekloc + 1) end -function XRDStream(urlbase::AbstractString, filepath::AbstractString, username::AbstractString) - file_id = @ccall xrootdgo.Open(urlbase::Cstring, filepath::Cstring, username::Cstring)::Cstring - if unsafe_string(file_id) == "error" - error("xrootd Go library errored.") - end - # file_id = @threadcall((:Open, xrootdgo), Cstring, (Cstring, Cstring, Cstring), urlbase, filepath, username) - size = @ccall xrootdgo.Size(file_id::Cstring)::Int - XRDStream(file_id, 0, size) +function XRDStream(url::AbstractString) + file = File() + st, _ = open(file, url, OpenFlags.Read) + isError(st) && error("XRootD file open error: $st") + st, statinfo = stat(file) + isError(st) && error("XRootD file stat error: $st") + XRDStream(file, 0, statinfo.size) end function Base.close(fobj::XRDStream) - xrootdgo.Close(fobj.gofile_id) + close(fobj.file) end function read_seek_nb(fobj::XRDStream, seek, nb) - buffer = Vector{UInt8}(undef, nb) - # @threadcall((:ReadAt, xrootdgo), Cvoid, (Ptr{UInt8}, Cstring, Clong, Clong), buffer, fobj.gofile_id, nb, seek) - @ccall xrootdgo.ReadAt(buffer::Ptr{UInt8}, fobj.gofile_id::Cstring, nb::Clong, seek::Clong)::Cvoid + st, buffer = read(fobj.file, nb, seek) + isError(st) && error("XRootD file read error: $st") return buffer end -function _read!(ptr, fobj, nb, seekloc) - @ccall xrootdgo.ReadAt(ptr::Ptr{UInt8}, - fobj.gofile_id::Cstring, nb::Clong, seekloc::Clong)::Cvoid -end - -function _read!(ptr, fobj, nb) - _read!(ptr, fobj, nb, fobj.seekloc) -end function Base.read(fobj::XRDStream, ::Type{T}) where T @debug @show T, sizeof(T) nb = sizeof(T) output = Ref{T}() tko = Base.@_gc_preserve_begin output - po = Ptr{UInt8}(pointer_from_objref(output)) - _read!(po, fobj, nb, fobj.seekloc) + po = pointer_from_objref(output) + unsafe_read(fobj.file, po, nb, fobj.seekloc) Base.@_gc_preserve_end tko fobj.seekloc += nb return output[] end function Base.read(fobj::XRDStream, nb::Integer) - buffer = Vector{UInt8}(undef, nb) - GC.@preserve buffer _read!(buffer, fobj, nb, fobj.seekloc) + st, buffer = read(fobj.file, nb, fobj.seekloc) + isError(st) && error("XRootD file read error: $st") fobj.seekloc += nb return buffer end diff --git a/test/runtests.jl b/test/runtests.jl index 14315a6f..0221f962 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,27 +4,29 @@ using UnROOT nthreads = UnROOT._maxthreadid() nthreads == 1 && @warn "Running on a single thread. Please re-run the test suite with at least two threads (`julia --threads 2 ...`)" -include("Aqua.jl") -include("bootstrapping.jl") -include("compressions.jl") -include("jagged.jl") -include("lazy.jl") -include("histograms.jl") -include("views.jl") -include("multithreading.jl") -include("remote.jl") -include("displays.jl") -include("type_stability.jl") -include("utils.jl") -include("misc.jl") +@testset "UnROOT tests" verbose = true begin + include("Aqua.jl") + include("bootstrapping.jl") + include("compressions.jl") + include("jagged.jl") + include("lazy.jl") + include("histograms.jl") + include("views.jl") + include("multithreading.jl") + include("remote.jl") + include("displays.jl") + include("type_stability.jl") + include("utils.jl") + include("misc.jl") -include("type_support.jl") -include("custom_bootstrapping.jl") -include("lorentzvectors.jl") -include("NanoAOD.jl") + include("type_support.jl") + include("custom_bootstrapping.jl") + include("lorentzvectors.jl") + include("NanoAOD.jl") -include("issues.jl") + include("issues.jl") -if VERSION >= v"1.9" - include("rntuple.jl") + if VERSION >= v"1.9" + include("rntuple.jl") + end end From 40f35203ff78d5afe38bc959b23394651cfed235 Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Tue, 4 Jun 2024 08:24:41 -0400 Subject: [PATCH 03/14] bump to 0.10.31 (#341) Release for XRootD.jl --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 162ec744..c264a955 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "UnROOT" uuid = "3cd96dde-e98d-4713-81e9-a4a1b0235ce9" authors = ["Tamas Gal", "Jerry Ling", "Johannes Schumann", "Nick Amin"] -version = "0.10.30" +version = "0.10.31" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" From a2864bfb4cc140150611e6480eeeade0c5eff9b3 Mon Sep 17 00:00:00 2001 From: Moelf Date: Tue, 18 Jun 2024 11:48:25 -0400 Subject: [PATCH 04/14] clean up --- src/RNTuple/Writing/TFileWriter.jl | 385 +++++++++++++++++++++++++++-- src/RNTuple/bootstrap.jl | 2 +- 2 files changed, 372 insertions(+), 15 deletions(-) diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index 8fe91c57..6b9b5aa0 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -1,3 +1,108 @@ +using StaticArrays + +const REFERENCE_BYTES = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, + 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, + 0x46, 0x69, 0x6C, 0x65, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, + 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x18, 0x74, + 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, + 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, + 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, + 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, 0x20, 0x76, + 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, + 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, + 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, + 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, 0xCE, 0xCE, 0xCE, 0xCE, 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, + 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, + 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, + 0x25, 0x1E, 0x55, 0x4C, 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, + 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, + 0xFE, 0x3C, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, + 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, + 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, + 0x3A, 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, + 0x65, 0x00, 0x40, 0x00, 0x00, 0x42, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, + 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, + 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, + 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, + 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, + 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, + 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, + 0x00, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, + 0x40, 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, + 0x74, 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, + 0x6F, 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, + 0x74, 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, + 0xC2, 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, + 0x15, 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, + 0xDA, 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, + 0x04, 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, + 0xDE, 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, + 0x2A, 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, + 0x37, 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, + 0x64, 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, + 0xD6, 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, + 0xC5, 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, + 0x0C, 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, + 0x8A, 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, + 0x17, 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, + 0x92, 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, + 0x0F, 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, + 0x7C, 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, + 0x58, 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, + 0xF2, 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, + 0xC7, 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, + 0x90, 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, + 0x2E, 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, + 0x15, 0x3D, 0xC1, 0xCA, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, + 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, + 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, + 0x35, 0x94, 0x00, +] """ struct FilePreamble @@ -5,7 +110,10 @@ struct FilePreamble fVersion::Int32 # File format version end """ -dummy_FilePreamble = UnROOT.FilePreamble(b"root", 0xf745) +# file_preamble = UnROOT.FilePreamble(b"root", 0xf745) +file_preamble = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, +] """ struct FileHeader32 @@ -24,18 +132,31 @@ end """ # dummy_FileHeader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) dummy_FileHeader = [ - 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] dummy_padding1 = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] + +tkey32_tfile = [ + 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, + 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, + 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, +] + +tfile = [ + 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, + 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] + """ struct ROOTDirectoryHeader32 fVersion::Int16 @@ -48,13 +169,249 @@ struct ROOTDirectoryHeader32 fSeekKeys::Int32 end """ -# dummy_DirectoryHeader32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) -dummy_DirectoryHeader32 = [ - 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, - 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x01, +# tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) +tdirectory32 = [ + 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, + 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, ] dummy_padding2 = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +struct RBlob + fNbytes::Int32 + fVersion::Int16 + fObjLen::Int32 + fDatime::UInt32 + fKeyLen::Int16 + fCycle::Int16 + fSeekKey::Int64 + fClassName::SVector{5,UInt8} + fName::SVector{1,UInt8} + fTitle::SVector{1,UInt8} +end + +# RBlob1 = RBlob(220, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 0x000000F4, b"RBlob", b"\0", b"\0") + +RBlob1 = [ + 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, ] + + +""" +UnROOT.RNTupleHeader + feature_flag: UInt64 0x0000000000000000 + name: String "myntuple" + ntuple_description: String "" + writer_identifier: String "ROOT v6.33.01" + field_records: Array{UnROOT.FieldRecord}((1,)) + 1: UnROOT.FieldRecord + field_version: UInt32 0x00000000 + type_version: UInt32 0x00000000 + parent_field_id: UInt32 0x00000000 + struct_role: UInt16 0x0000 + flags: UInt16 0x0000 + repetition: Int64 0 + field_name: String "one_uint" + type_name: String "std::uint32_t" + type_alias: String "" + field_desc: String "" + column_records: Array{UnROOT.ColumnRecord}((1,)) + 1: UnROOT.ColumnRecord + type: UInt16 0x0014 + nbits: UInt16 0x0020 + field_id: UInt32 0x00000000 + flags: UInt32 0x00000000 + first_ele_idx: Int64 0 + alias_columns: Array{UnROOT.AliasRecord}((0,)) + extra_type_infos: Array{UnROOT.ExtraTypeInfo}((0,)) +""" +# rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "UnROOT v6.33.", [ +# UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), +# ], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + +rnt_header = [ + 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, + 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, 0x20, 0x76, 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, + 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, + 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, + 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, + 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, +] + +RBlob2 = [ + 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +page1 = [ + 0xCE, 0xCE, 0xCE, 0xCE, +] + +RBlob3 = [ + 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +linkedlist_envelope = [ + 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, 0x25, 0x1E, 0x55, 0x4C, +] + + +RBlob4 = [ + 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +rnt_footer = [ + 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, + 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, 0xFE, 0x3C, +] + +""" +UnROOT.TKey32 + fNbytes: Int32 134 + fVersion: Int16 4 + fObjlen: Int32 70 + fDatime: UInt32 0x7567176d + fKeylen: Int16 64 + fCycle: Int16 1 + fSeekKey: Int32 866 + fSeekPdir: Int32 100 + fClassName: String "ROOT::Experimental::RNTuple" + fName: String "myntuple" + fTitle: String "" +""" +tkey32_anchor = [ + 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, + 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, + 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, +] +# these 6 bytes are between tkey32_anchor and the actual anchor +magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04] + +""" +UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple + fVersionEpoch: UInt16 0x0000 + fVersionMajor: UInt16 0x0002 + fVersionMinor: UInt16 0x0000 + fVersionPatch: UInt16 0x0000 + fSeekHeader: UInt64 0x0000000000000116 + fNBytesHeader: UInt64 0x00000000000000ba + fLenHeader: UInt64 0x00000000000000ba + fSeekFooter: UInt64 0x00000000000002b6 + fNBytesFooter: UInt64 0x00000000000000ac + fLenFooter: UInt64 0x00000000000000ac + fChecksum: UInt64 0xdc495fd01479af1b +""" +rnt_anchor = [ + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, +] + +tkey32_TDirectory = [ + 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, +] + +# 1 key, and it is the RNTuple Anchor +n_keys = [ + 0x00, 0x00, 0x00, 0x01, +] + +@assert tkey32_anchor == [ + 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, + 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, + 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, +] + + +tkey32_TStreamerInfo = [ + 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, 0x74, + 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, 0x6F, + 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, 0x74, +] + +tsreamerinfo_compressed = [ + 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, + 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15, + 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA, + 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04, + 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE, + 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A, + 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37, + 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64, + 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6, + 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5, + 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C, + 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A, + 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17, + 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92, + 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F, + 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C, + 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58, + 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2, + 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7, + 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90, + 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E, + 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15, + 0x3D, 0xC1, 0xCA, +] + +tfile_end = [ + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00, +] + + +MINE = [ + file_preamble; + dummy_FileHeader; dummy_padding1; + tkey32_tfile; tfile; + tdirectory32; dummy_padding2; + RBlob1; rnt_header; + RBlob2; page1; + RBlob3; linkedlist_envelope; + RBlob4; rnt_footer; + tkey32_anchor; magic_6bytes; rnt_anchor; + tkey32_TDirectory; n_keys; tkey32_anchor; + tkey32_TStreamerInfo; tsreamerinfo_compressed; + tfile_end +] + +@assert MINE == REFERENCE_BYTES diff --git a/src/RNTuple/bootstrap.jl b/src/RNTuple/bootstrap.jl index a226cb6a..4cb85db8 100644 --- a/src/RNTuple/bootstrap.jl +++ b/src/RNTuple/bootstrap.jl @@ -32,7 +32,7 @@ function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) fChecksum = readtype(local_io, UInt64), ) - @show dump(anchor) + @show dump(anchor) header_bytes = decompress_bytes(read_seek_nb(io, anchor.fSeekHeader, anchor.fNBytesHeader), anchor.fLenHeader) header_io = IOBuffer(header_bytes) header = _rntuple_read(header_io, RNTupleEnvelope{RNTupleHeader}) From 05ab59a11a97cb84698be1d806aebcfb7f0f169c Mon Sep 17 00:00:00 2001 From: Moelf Date: Wed, 19 Jun 2024 01:35:07 -0400 Subject: [PATCH 05/14] gradually replace dummy with function write add Envelope, FieldRecord, ColumnRecord fix temp_io and RNTupleHeader without envelope id_length (8 bytes) and checksum (8 bytes) --- src/RNTuple/Writing/TFileWriter.jl | 380 ++++++++++++++++++++++++++--- src/RNTuple/bootstrap.jl | 3 - 2 files changed, 347 insertions(+), 36 deletions(-) diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index 6b9b5aa0..bcea6155 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -1,4 +1,7 @@ using StaticArrays +using UnROOT +using UnROOT: RNTupleFrame +using XXHashNative: xxh3_64 const REFERENCE_BYTES = [ 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, @@ -104,6 +107,70 @@ const REFERENCE_BYTES = [ 0x35, 0x94, 0x00, ] +function color_diff(ary1, ary2) + if length(ary1) != length(ary2) + printstyled("!!! Length mismatch !!!: length(ary1)=$(length(ary1)), length(ary2)=$(length(ary2))\n", color=:red) + end + print("[") + x = 0 + for (i,j) in zip(ary1, ary2) + if x % 8 == 0 + print(" ") + end + if x % 16 == 0 + println() + x = 0 + end + if i != j + printstyled("$(repr(i))/$(repr(j)), ", color=:red) + else + printstyled("$(repr(i)), ", color=:green) + end + x += 1 + end + println() + println("]") +end + +function rnt_write(io::IO, x::AbstractString; legacy=false) + L = length(x) + if legacy + if L > typemax(UInt8) + error("String longer than 255 not implemented") + end + write(io, UInt8(L)) + write(io, codeunits(x)) + else + write(io, UInt32(L)) + write(io, codeunits(x)) + end +end + +function rnt_write(io::IO, x; legacy=false) + if legacy + write(io, bswap(x)) + else + write(io, x) + end +end + +function rnt_write(io::IO, x::AbstractVector{UInt8}; legacy=false) + if legacy + write(io, reverse(x)) + else + write(io, x) + end +end + +function test_io(obj, expected) + a = IOBuffer() + rnt_write(a, obj) + ours = take!(a) + if ours != expected + color_diff(ours, expected) + end +end + """ struct FilePreamble identifier::SVector{4, UInt8} # Root file identifier ("root") @@ -130,7 +197,21 @@ struct FileHeader32 fUUID::SVector{18, UInt8} # Universal Unique ID end """ -# dummy_FileHeader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +function rnt_write(io::IO, x::UnROOT.FileHeader32) + rnt_write(io, x.fBEGIN; legacy=true) + rnt_write(io, x.fEND; legacy=true) + rnt_write(io, x.fSeekFree; legacy=true) + rnt_write(io, x.fNbytesFree; legacy=true) + rnt_write(io, x.nfree; legacy=true) + rnt_write(io, x.fNbytesName; legacy=true) + rnt_write(io, x.fUnits; legacy=true) + rnt_write(io, x.fCompress; legacy=true) + rnt_write(io, x.fSeekInfo; legacy=true) + rnt_write(io, x.fNbytesInfo; legacy=true) + rnt_write(io, x.fUUID; legacy=true) +end + dummy_FileHeader = [ 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, @@ -138,23 +219,67 @@ dummy_FileHeader = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] +test_io(fileheader, dummy_FileHeader) + dummy_padding1 = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] -tkey32_tfile = [ +""" +struct TKey32 + fNbytes::Int32 + fVersion::Int16 + fObjlen::Int32 + fDatime::UInt32 + fKeylen::Int16 + fCycle::Int16 + fSeekKey::Int32 + fSeekPdir::Int32 + fClassName::String + fName::String + fTitle::String +end +""" +tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") +function rnt_write(io::IO, x::UnROOT.TKey32) + rnt_write(io, x.fNbytes; legacy=true) + rnt_write(io, x.fVersion; legacy=true) + rnt_write(io, x.fObjlen; legacy=true) + rnt_write(io, x.fDatime; legacy=true) + rnt_write(io, x.fKeylen; legacy=true) + rnt_write(io, x.fCycle; legacy=true) + rnt_write(io, x.fSeekKey; legacy=true) + rnt_write(io, x.fSeekPdir; legacy=true) + rnt_write(io, x.fClassName; legacy=true) + rnt_write(io, x.fName; legacy=true) + rnt_write(io, x.fTitle; legacy=true) +end + +dummy_tkey32_tfile = [ 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] -tfile = [ +test_io(tkey32_tfile, dummy_tkey32_tfile) + +struct TFile_write + filename::String + unknown::String +end +tfile = TFile_write("test_ntuple_minimal.root", "") +function rnt_write(io::IO, x::TFile_write) + rnt_write(io, x.filename; legacy=true) + rnt_write(io, x.unknown; legacy=true) +end +dummy_tfile = [ 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] +test_io(tfile, dummy_tfile) """ @@ -169,11 +294,22 @@ struct ROOTDirectoryHeader32 fSeekKeys::Int32 end """ -# tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) -tdirectory32 = [ +tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) +function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) + rnt_write(io, x.fVersion; legacy=true) + rnt_write(io, x.fDatimeC; legacy=true) + rnt_write(io, x.fDatimeM; legacy=true) + rnt_write(io, x.fNbytesKeys; legacy=true) + rnt_write(io, x.fNbytesName; legacy=true) + rnt_write(io, x.fSeekDir; legacy=true) + rnt_write(io, x.fSeekParent; legacy=true) + rnt_write(io, x.fSeekKeys; legacy=true) +end +dummy_tdirectory32 = [ 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, ] +test_io(tdirectory32, dummy_tdirectory32) dummy_padding2 = [ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -187,20 +323,35 @@ struct RBlob fDatime::UInt32 fKeyLen::Int16 fCycle::Int16 - fSeekKey::Int64 - fClassName::SVector{5,UInt8} - fName::SVector{1,UInt8} - fTitle::SVector{1,UInt8} + fSeekKey::Int32 + fSeekPdir::Int32 + fClassName::String + fName::String + fTitle::String +end +RBlob1 = RBlob(220, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") +function rnt_write(io::IO, x::RBlob) + rnt_write(io, x.fNbytes; legacy=true) + rnt_write(io, x.fVersion; legacy=true) + rnt_write(io, x.fObjLen; legacy=true) + rnt_write(io, x.fDatime; legacy=true) + rnt_write(io, x.fKeyLen; legacy=true) + rnt_write(io, x.fCycle; legacy=true) + rnt_write(io, x.fSeekKey; legacy=true) + rnt_write(io, x.fSeekPdir; legacy=true) + rnt_write(io, x.fClassName; legacy=true) + rnt_write(io, x.fName; legacy=true) + rnt_write(io, x.fTitle; legacy=true) end -# RBlob1 = RBlob(220, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 0x000000F4, b"RBlob", b"\0", b"\0") - -RBlob1 = [ +dummy_RBlob1 = [ 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] +test_io(RBlob1, dummy_RBlob1) + """ UnROOT.RNTupleHeader @@ -230,23 +381,156 @@ UnROOT.RNTupleHeader alias_columns: Array{UnROOT.AliasRecord}((0,)) extra_type_infos: Array{UnROOT.ExtraTypeInfo}((0,)) """ -# rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "UnROOT v6.33.", [ -# UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), -# ], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) - -rnt_header = [ - 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, - 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, 0x20, 0x76, 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, - 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, - 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, - 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, - 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, - 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, +rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ + UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), +], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + + +# ==================================== side tests ==================================== +function rnt_write(io::IO, x::UnROOT.FieldRecord) + rnt_write(io, x.field_version) + rnt_write(io, x.type_version) + rnt_write(io, x.parent_field_id) + rnt_write(io, x.struct_role) + rnt_write(io, x.flags) + if !iszero(x.repetition) + if x.flags != 0x01 + error("Repetition is set but flags is not 0x01") + end + rnt_write(io, x.repetition) + end + rnt_write(io, x.field_name) + rnt_write(io, x.type_name) + rnt_write(io, x.type_alias) + rnt_write(io, x.field_desc) +end +field_record = UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", "") +dummy_field_record = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, + 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(field_record, dummy_field_record) + +function rnt_write(io::IO, x::UnROOT.ColumnRecord) + rnt_write(io, x.type) + rnt_write(io, x.nbits) + rnt_write(io, x.field_id) + rnt_write(io, x.flags) + if !iszero(x.first_ele_idx) + if x.flags != 0x08 + error("First element index is set but flags is not 0x08") + end + rnt_write(io, x.first_ele_idx) + end +end +column_record = UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0) +dummy_column_record = [ + 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(column_record, dummy_column_record) + +function rnt_write(io::IO, x::UnROOT.RNTupleHeader) + temp_io = IOBuffer() + id_type = 0x0001 + _rnt_write(temp_io, x) + envelope_size = temp_io.size + 18 + payload_ary = take!(temp_io) + id_length = (UInt64(envelope_size & 0xff) << 24) | id_type + write(io, id_length) + write(io, payload_ary) + write(io, xxh3_64(payload_ary)) +end +# reference from reading code +# function _rntuple_read(io, ::Type{Vector{T}}) where T +# pos = position(io) +# Size = read(io, Int64) +# @assert Size < 0 +# NumItems = read(io, Int32) +# end_pos = pos - Size +# res = T[_rntuple_read(io, RNTupleFrame{T}).payload for _=1:NumItems] +# seek(io, end_pos) +# return res +# end +# +# struct RNTupleFrame{T} + # payload::T +# end +# function _rntuple_read(io, ::Type{RNTupleFrame{T}}) where T + # pos = position(io) + # Size = read(io, Int64) + # end_pos = pos + Size + # @assert Size >= 0 + # res = _rntuple_read(io, T) + # seek(io, end_pos) + # return RNTupleFrame(res) +# end + +function rnt_write(io::IO, x::RNTupleFrame{T}) where T + temp_io = IOBuffer() + rnt_write(temp_io, x.payload) + size = temp_io.size + 8 + write(io, Int64(size)) + seekstart(temp_io) + write(io, temp_io) +end + +function rnt_write(io::IO, ary::AbstractArray) + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, RNTupleFrame(x)) + end + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end + +envelope_frame_field_record = [field_record] +dummy_envelope_frame_field_record = [ + 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, + 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +test_io(envelope_frame_field_record, dummy_envelope_frame_field_record) + +function rnt_write(io::IO, x::UnROOT.RNTupleHeader) + rnt_write(io, x.feature_flag) + rnt_write(io, x.name) + rnt_write(io, x.ntuple_description) + rnt_write(io, x.writer_identifier) + rnt_write(io, x.field_records) + rnt_write(io, x.column_records) + rnt_write(io, x.alias_columns) + rnt_write(io, x.extra_type_infos) +end +dummy_rnt_header_payload = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, + 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, + 0x20, 0x76, 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, + 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, +] + +test_io(rnt_header, dummy_rnt_header_payload) + +# ==================================================================================== +# ==================================== side tests ==================================== + +dummy_rnt_header = [ + 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, ] RBlob2 = [ @@ -402,9 +686,9 @@ tfile_end = [ MINE = [ file_preamble; dummy_FileHeader; dummy_padding1; - tkey32_tfile; tfile; - tdirectory32; dummy_padding2; - RBlob1; rnt_header; + dummy_tkey32_tfile; dummy_tfile; + dummy_tdirectory32; dummy_padding2; + dummy_RBlob1; dummy_rnt_header; RBlob2; page1; RBlob3; linkedlist_envelope; RBlob4; rnt_footer; @@ -414,4 +698,34 @@ MINE = [ tfile_end ] -@assert MINE == REFERENCE_BYTES +@show MINE == REFERENCE_BYTES + +function write_rntuple(file::IO, table; rntuple_name="myntuple") + rnt_write(file, file_preamble) + rnt_write(file, fileheader) + rnt_write(file, dummy_padding1) + + rnt_write(file, tkey32_tfile) + rnt_write(file, tfile) + + rnt_write(file, tdirectory32) + rnt_write(file, dummy_padding2) + + rnt_write(file, RBlob1) + rnt_write(file, dummy_rnt_header) + + rnt_write(file, [RBlob2; page1]) + + rnt_write(file, [RBlob3; linkedlist_envelope]) + + rnt_write(file, [RBlob4; rnt_footer]) + + rnt_write(file, [tkey32_anchor; magic_6bytes; rnt_anchor]) + rnt_write(file, [tkey32_TDirectory; n_keys; tkey32_anchor]) + rnt_write(file, [tkey32_TStreamerInfo; tsreamerinfo_compressed]) + rnt_write(file, tfile_end) +end + +myio = IOBuffer() +write_rntuple(myio, []) +@show MINE == take!(myio) diff --git a/src/RNTuple/bootstrap.jl b/src/RNTuple/bootstrap.jl index 4cb85db8..b2130e88 100644 --- a/src/RNTuple/bootstrap.jl +++ b/src/RNTuple/bootstrap.jl @@ -15,8 +15,6 @@ end function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) local_io = datastream(io, tkey) - @show position(io) - @show dump(tkey) skip(local_io, 6) anchor = ROOT_3a3a_Experimental_3a3a_RNTuple(; fVersionEpoch = readtype(local_io, UInt16), @@ -32,7 +30,6 @@ function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) fChecksum = readtype(local_io, UInt64), ) - @show dump(anchor) header_bytes = decompress_bytes(read_seek_nb(io, anchor.fSeekHeader, anchor.fNBytesHeader), anchor.fLenHeader) header_io = IOBuffer(header_bytes) header = _rntuple_read(header_io, RNTupleEnvelope{RNTupleHeader}) From 71ca8159a00094c3c5e687695097578e2faa1294 Mon Sep 17 00:00:00 2001 From: Moelf Date: Wed, 19 Jun 2024 15:09:51 -0400 Subject: [PATCH 06/14] RNTupleHeader with Envelope --- src/RNTuple/Writing/TFileWriter.jl | 79 +++++++++++++++++++----------- 1 file changed, 51 insertions(+), 28 deletions(-) diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index bcea6155..be73f1da 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -162,9 +162,9 @@ function rnt_write(io::IO, x::AbstractVector{UInt8}; legacy=false) end end -function test_io(obj, expected) +function test_io(obj, expected; kw...) a = IOBuffer() - rnt_write(a, obj) + rnt_write(a, obj; kw...) ours = take!(a) if ours != expected color_diff(ours, expected) @@ -386,7 +386,7 @@ rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", ], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) -# ==================================== side tests ==================================== +# ==================================== side tests begin ==================================== function rnt_write(io::IO, x::UnROOT.FieldRecord) rnt_write(io, x.field_version) rnt_write(io, x.type_version) @@ -430,18 +430,6 @@ dummy_column_record = [ 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] test_io(column_record, dummy_column_record) - -function rnt_write(io::IO, x::UnROOT.RNTupleHeader) - temp_io = IOBuffer() - id_type = 0x0001 - _rnt_write(temp_io, x) - envelope_size = temp_io.size + 18 - payload_ary = take!(temp_io) - id_length = (UInt64(envelope_size & 0xff) << 24) | id_type - write(io, id_length) - write(io, payload_ary) - write(io, xxh3_64(payload_ary)) -end # reference from reading code # function _rntuple_read(io, ::Type{Vector{T}}) where T # pos = position(io) @@ -466,7 +454,6 @@ end # seek(io, end_pos) # return RNTupleFrame(res) # end - function rnt_write(io::IO, x::RNTupleFrame{T}) where T temp_io = IOBuffer() rnt_write(temp_io, x.payload) @@ -500,16 +487,35 @@ dummy_envelope_frame_field_record = [ test_io(envelope_frame_field_record, dummy_envelope_frame_field_record) -function rnt_write(io::IO, x::UnROOT.RNTupleHeader) - rnt_write(io, x.feature_flag) - rnt_write(io, x.name) - rnt_write(io, x.ntuple_description) - rnt_write(io, x.writer_identifier) - rnt_write(io, x.field_records) - rnt_write(io, x.column_records) - rnt_write(io, x.alias_columns) - rnt_write(io, x.extra_type_infos) +function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.feature_flag) + rnt_write(temp_io, x.name) + rnt_write(temp_io, x.ntuple_description) + rnt_write(temp_io, x.writer_identifier) + rnt_write(temp_io, x.field_records) + rnt_write(temp_io, x.column_records) + rnt_write(temp_io, x.alias_columns) + rnt_write(temp_io, x.extra_type_infos) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0001 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + + if envelope + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + checksum = xxh3_64(payload_ary) + write(io, payload_ary) + write(io, checksum) + else + write(io, payload_ary) + end end + dummy_rnt_header_payload = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, @@ -524,14 +530,31 @@ dummy_rnt_header_payload = [ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, ] -test_io(rnt_header, dummy_rnt_header_payload) +test_io(rnt_header, dummy_rnt_header_payload; envelope=false) -# ==================================================================================== -# ==================================== side tests ==================================== +# ==================================== side tests end ==================================== + +# reading struct looks like this: +# struct RNTupleEnvelope{T} + # type_id::UInt16 + # envelope_length::UInt64 + # payload::T + # checksum::UInt64 +# end +# +function rnt_write_envelope(io::IO, x; type_id) + temp_io = IOBuffer() + rnt_write(temp_io, x.payload) + size = temp_io.size + 8 + write(io, Int64(size)) + seekstart(temp_io) + write(io, temp_io) +end dummy_rnt_header = [ 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, ] +test_io(rnt_header, dummy_rnt_header; envelope=true) RBlob2 = [ 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, From 14083e6c410220caf8e618b01b55bb1bbe97e25e Mon Sep 17 00:00:00 2001 From: Moelf Date: Wed, 19 Jun 2024 17:40:02 -0400 Subject: [PATCH 07/14] RNTupleFooter finish out --- src/RNTuple/Writing/TFileWriter.jl | 317 ++++++++++++++++++++++++++--- src/RNTuple/footer.jl | 40 ++-- 2 files changed, 308 insertions(+), 49 deletions(-) diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index be73f1da..6de089de 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -1,6 +1,6 @@ using StaticArrays using UnROOT -using UnROOT: RNTupleFrame +using UnROOT: RNTupleFrame, ClusterSummary, PageDescription using XXHashNative: xxh3_64 const REFERENCE_BYTES = [ @@ -146,6 +146,10 @@ function rnt_write(io::IO, x::AbstractString; legacy=false) end end +function rnt_write(io::IO, x::AbstractVector) + error("Ambiguous type: $(typeof(x)), use Write_RNTupleListFrame or similar wrapper") +end + function rnt_write(io::IO, x; legacy=false) if legacy write(io, bswap(x)) @@ -329,7 +333,6 @@ struct RBlob fName::String fTitle::String end -RBlob1 = RBlob(220, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") function rnt_write(io::IO, x::RBlob) rnt_write(io, x.fNbytes; legacy=true) rnt_write(io, x.fVersion; legacy=true) @@ -344,12 +347,12 @@ function rnt_write(io::IO, x::RBlob) rnt_write(io, x.fTitle; legacy=true) end +RBlob1 = RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") dummy_RBlob1 = [ 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] - test_io(RBlob1, dummy_RBlob1) @@ -463,7 +466,11 @@ function rnt_write(io::IO, x::RNTupleFrame{T}) where T write(io, temp_io) end -function rnt_write(io::IO, ary::AbstractArray) +struct Write_RNTupleListFrame{T<:AbstractArray} + payload::T +end +function rnt_write(io::IO, x::Write_RNTupleListFrame) + ary = x.payload N = length(ary) temp_io = IOBuffer() for x in ary @@ -476,7 +483,7 @@ function rnt_write(io::IO, ary::AbstractArray) write(io, temp_io) end -envelope_frame_field_record = [field_record] +envelope_frame_field_record = Write_RNTupleListFrame([field_record]) dummy_envelope_frame_field_record = [ 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -493,10 +500,10 @@ function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) rnt_write(temp_io, x.name) rnt_write(temp_io, x.ntuple_description) rnt_write(temp_io, x.writer_identifier) - rnt_write(temp_io, x.field_records) - rnt_write(temp_io, x.column_records) - rnt_write(temp_io, x.alias_columns) - rnt_write(temp_io, x.extra_type_infos) + rnt_write(temp_io, Write_RNTupleListFrame(x.field_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.alias_columns)) + rnt_write(temp_io, Write_RNTupleListFrame(x.extra_type_infos)) # add id_length size and checksum size envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) @@ -556,21 +563,156 @@ dummy_rnt_header = [ ] test_io(rnt_header, dummy_rnt_header; envelope=true) -RBlob2 = [ +RBlob2 = RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "") +dummy_RBlob2 = [ 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] +test_io(RBlob2, dummy_RBlob2) + page1 = [ 0xCE, 0xCE, 0xCE, 0xCE, ] -RBlob3 = [ +RBlob3 = RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "") +dummy_RBlob3 = [ 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] -linkedlist_envelope = [ +test_io(RBlob3, dummy_RBlob3) + +function rnt_write(io::IO, x::ClusterSummary) + rnt_write(io, x.first_entry_number) + rnt_write(io, x.number_of_entries) +end + +function rnt_write(io::IO, x::UnROOT.Locator) + rnt_write(io, x.num_bytes) + rnt_write(io, x.offset) +end + +function rnt_write(io::IO, x::PageDescription) + rnt_write(io, x.num_elements) + rnt_write(io, x.locator) +end + +function rnt_write(io::IO, x::UnROOT.RNTuplePageTopList) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, x) + end + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end +function rnt_write(io::IO, x::UnROOT.RNTuplePageOuterList) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, x) + end + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end +function rnt_write(io::IO, x::UnROOT.RNTuplePageInnerList) + ary = x.payload + N = length(ary) + temp_io = IOBuffer() + for x in ary + rnt_write(temp_io, x) + end + offset = zero(UInt64) + compression = zero(UInt32) + write(temp_io, offset, compression) + size = temp_io.size + sizeof(Int64) + sizeof(Int32) + write(io, Int64(-size)) + write(io, Int32(N)) + seekstart(temp_io) + write(io, temp_io) +end +# ================= side tests begin ================= +dummy_cluster_summary = [ + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, +] +cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, 1)]) +test_io(cluster_summary, dummy_cluster_summary) + +# > https://github.com/root-project/root/blob/1a854602e42d4493f56a26e35e19bdf23b7d0933/tree/ntuple/v7/doc/specifications.md?plain=1#L672 +# > The inner list is followed by a 64bit unsigned integer element offset and the 32bit compression settings +dummy_inner_list_frame = [ + 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +innert_list_frame = UnROOT.RNTuplePageInnerList([ + PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), +]) +test_io(innert_list_frame, dummy_inner_list_frame) +# ================= side tests end ================= + +nested_page_locations = +UnROOT.RNTuplePageTopList([ + UnROOT.RNTuplePageOuterList([ + UnROOT.RNTuplePageInnerList([ + PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), + ]), + ]), +]) + +function rnt_write(io::IO, x::UnROOT.PageLink; envelope=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.header_checksum) + rnt_write(temp_io, Write_RNTupleListFrame(x.cluster_summaries)) + rnt_write(temp_io, x.nested_page_locations) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0003 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + + if envelope + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + checksum = xxh3_64(payload_ary) + write(io, payload_ary) + write(io, checksum) + else + write(io, payload_ary) + end +end +pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) + +# ================= side tests begin ================= +dummy_pagelink_noenvelope = [ + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 +] +test_io(pagelink, dummy_pagelink_noenvelope; envelope=false) +# ================= side tests end ================= + + +dummy_pagelink = [ 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, @@ -580,14 +722,97 @@ linkedlist_envelope = [ 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, 0x25, 0x1E, 0x55, 0x4C, ] +test_io(pagelink, dummy_pagelink) - -RBlob4 = [ +RBlob4 = RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "") +dummy_RBlob4 = [ 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] -rnt_footer = [ +test_io(RBlob4, dummy_RBlob4) + +""" +UnROOT.RNTupleFooter: + feature_flag: 0 + header_checksum: 0x3dec59c009c67e28 + extension_header_links: UnROOT.RNTupleSchemaExtension(UnROOT.FieldRecord[], UnROOT.ColumnRecord[], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + column_group_records: UnROOT.ColumnGroupRecord[] + cluster_group_records: UnROOT.ClusterGroupRecord[UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(num_bytes=124, offset=0x0000000000000218, ) +))] + meta_data_links: UnROOT.EnvLink[] +""" + +""" +@SimpleStruct struct EnvLink + uncomp_size::UInt64 + locator::Locator +end +""" +function rnt_write(io::IO, x::UnROOT.EnvLink) + rnt_write(io, x.uncomp_size) + rnt_write(io, x.locator) +end + +""" +@SimpleStruct struct ClusterGroupRecord + minimum_entry_number::Int64 + entry_span::Int64 + num_clusters::Int32 + page_list_link::EnvLink +end +""" +function rnt_write(io::IO, x::UnROOT.ClusterGroupRecord) + rnt_write(io, x.minimum_entry_number) + rnt_write(io, x.entry_span) + rnt_write(io, x.num_clusters) + rnt_write(io, x.page_list_link) +end + +function rnt_write(io::IO, x::UnROOT.RNTupleSchemaExtension) + temp_io = IOBuffer() + rnt_write(temp_io, Write_RNTupleListFrame(x.field_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.alias_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.extra_type_info)) + + size = temp_io.size + sizeof(Int64) + write(io, Int64(size)) + seekstart(temp_io) + write(io, temp_io) +end + +function rnt_write(io::IO, x::UnROOT.RNTupleFooter; envelope=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.feature_flag) + rnt_write(temp_io, x.header_checksum) + rnt_write(temp_io, x.extension_header_links) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_group_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.cluster_group_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.meta_data_links)) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0002 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + + if envelope + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + checksum = xxh3_64(payload_ary) + write(io, payload_ary) + write(io, checksum) + else + write(io, payload_ary) + end +end + +rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))), +], UnROOT.EnvLink[]) +dummy_rnt_footer = [ 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, @@ -600,6 +825,7 @@ rnt_footer = [ 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, 0xFE, 0x3C, ] +test_io(rnt_footer, dummy_rnt_footer) """ UnROOT.TKey32 @@ -615,7 +841,8 @@ UnROOT.TKey32 fName: String "myntuple" fTitle: String "" """ -tkey32_anchor = [ +tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "") +dummy_tkey32_anchor = [ 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, @@ -638,12 +865,28 @@ UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple fLenFooter: UInt64 0x00000000000000ac fChecksum: UInt64 0xdc495fd01479af1b """ -rnt_anchor = [ +function rnt_write(io::IO, x::UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple) + rnt_write(io, x.fVersionEpoch; legacy=true) + rnt_write(io, x.fVersionMajor; legacy=true) + rnt_write(io, x.fVersionMinor; legacy=true) + rnt_write(io, x.fVersionPatch; legacy=true) + rnt_write(io, x.fSeekHeader; legacy=true) + rnt_write(io, x.fNBytesHeader; legacy=true) + rnt_write(io, x.fLenHeader; legacy=true) + rnt_write(io, x.fSeekFooter; legacy=true) + rnt_write(io, x.fNBytesFooter; legacy=true) + rnt_write(io, x.fLenFooter; legacy=true) + rnt_write(io, x.fChecksum; legacy=true) +end + +rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b) +dummy_rnt_anchor = [ 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, ] +test_io(rnt_anchor, dummy_rnt_anchor) tkey32_TDirectory = [ 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, @@ -657,7 +900,7 @@ n_keys = [ 0x00, 0x00, 0x00, 0x01, ] -@assert tkey32_anchor == [ +@assert dummy_tkey32_anchor == [ 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, @@ -665,12 +908,14 @@ n_keys = [ ] -tkey32_TStreamerInfo = [ +tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list") +dummy_tkey32_TStreamerInfo = [ 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, 0x74, 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, 0x6F, 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, 0x74, ] +test_io(tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) tsreamerinfo_compressed = [ 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, @@ -712,12 +957,12 @@ MINE = [ dummy_tkey32_tfile; dummy_tfile; dummy_tdirectory32; dummy_padding2; dummy_RBlob1; dummy_rnt_header; - RBlob2; page1; - RBlob3; linkedlist_envelope; - RBlob4; rnt_footer; - tkey32_anchor; magic_6bytes; rnt_anchor; - tkey32_TDirectory; n_keys; tkey32_anchor; - tkey32_TStreamerInfo; tsreamerinfo_compressed; + dummy_RBlob2; page1; + dummy_RBlob3; dummy_pagelink; + dummy_RBlob4; dummy_rnt_footer; + dummy_tkey32_anchor; magic_6bytes; dummy_rnt_anchor; + tkey32_TDirectory; n_keys; dummy_tkey32_anchor; + dummy_tkey32_TStreamerInfo; tsreamerinfo_compressed; tfile_end ] @@ -735,17 +980,25 @@ function write_rntuple(file::IO, table; rntuple_name="myntuple") rnt_write(file, dummy_padding2) rnt_write(file, RBlob1) - rnt_write(file, dummy_rnt_header) + rnt_write(file, rnt_header) + + rnt_write(file, RBlob2) + rnt_write(file, page1) - rnt_write(file, [RBlob2; page1]) + rnt_write(file, RBlob3) + rnt_write(file, pagelink) - rnt_write(file, [RBlob3; linkedlist_envelope]) + rnt_write(file, RBlob4) + rnt_write(file, rnt_footer) - rnt_write(file, [RBlob4; rnt_footer]) + rnt_write(file, tkey32_anchor) + rnt_write(file, magic_6bytes) + rnt_write(file, rnt_anchor) - rnt_write(file, [tkey32_anchor; magic_6bytes; rnt_anchor]) - rnt_write(file, [tkey32_TDirectory; n_keys; tkey32_anchor]) - rnt_write(file, [tkey32_TStreamerInfo; tsreamerinfo_compressed]) + rnt_write(file, [tkey32_TDirectory; n_keys]) + rnt_write(file, tkey32_anchor) + rnt_write(file, tkey32_TStreamerInfo) + rnt_write(file, tsreamerinfo_compressed) rnt_write(file, tfile_end) end diff --git a/src/RNTuple/footer.jl b/src/RNTuple/footer.jl index e0a9a006..8c825bec 100644 --- a/src/RNTuple/footer.jl +++ b/src/RNTuple/footer.jl @@ -101,26 +101,32 @@ end number_of_entries::Int64 end -struct RNTupleListNoFrame{T} <: AbstractVector{T} - payload::Vector{T} -end -Base.size(r::RNTupleListNoFrame) = size(r.payload) -Base.getindex(r::RNTupleListNoFrame, i) = r.payload[i] -Base.setindex!(r::RNTupleListNoFrame, v, i) = (r.payload[i] = v) -# without the inner Frame for each item -function _rntuple_read(io, ::Type{RNTupleListNoFrame{T}}) where T - pos = position(io) - Size = read(io, Int64) - @assert Size < 0 - NumItems = read(io, Int32) - end_pos = pos - Size - res = T[_rntuple_read(io, T) for _=1:NumItems] - seek(io, end_pos) - return RNTupleListNoFrame(res) +for x in (:RNTuplePageTopList, :RNTuplePageOuterList, :RNTuplePageInnerList) + @eval begin + struct ($x){T} <: AbstractVector{T} + payload::Vector{T} + end + + function _rntuple_read(io, ::Type{$x{T}}) where T + pos = position(io) + Size = read(io, Int64) + @assert Size < 0 + NumItems = read(io, Int32) + end_pos = pos - Size + res = T[_rntuple_read(io, T) for _=1:NumItems] + seek(io, end_pos) + return $x(res) + end + + Base.size(r::$x) = size(r.payload) + Base.getindex(r::$x, i) = r.payload[i] + Base.setindex!(r::$x, v, i) = (r.payload[i] = v) + + end end @SimpleStruct struct PageLink header_checksum::UInt64 cluster_summaries::Vector{ClusterSummary} - nested_page_locations::RNTupleListNoFrame{RNTupleListNoFrame{RNTupleListNoFrame{PageDescription}}} + nested_page_locations::RNTuplePageTopList{RNTuplePageOuterList{RNTuplePageInnerList{PageDescription}}} end From 4cfe0c13d54f1581b2d5f40452832bf3564446df Mon Sep 17 00:00:00 2001 From: Moelf Date: Wed, 19 Jun 2024 19:48:26 -0400 Subject: [PATCH 08/14] track locations that we will need to update later --- src/RNTuple/Writing/TFileWriter.jl | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index 6de089de..fbb76230 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -888,12 +888,14 @@ dummy_rnt_anchor = [ ] test_io(rnt_anchor, dummy_rnt_anchor) -tkey32_TDirectory = [ +tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "") +dummy_tkey32_TDirectory = [ 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] +test_io(tkey32_TDirectory, dummy_tkey32_TDirectory) # 1 key, and it is the RNTuple Anchor n_keys = [ @@ -961,21 +963,23 @@ MINE = [ dummy_RBlob3; dummy_pagelink; dummy_RBlob4; dummy_rnt_footer; dummy_tkey32_anchor; magic_6bytes; dummy_rnt_anchor; - tkey32_TDirectory; n_keys; dummy_tkey32_anchor; + dummy_tkey32_TDirectory; n_keys; dummy_tkey32_anchor; dummy_tkey32_TStreamerInfo; tsreamerinfo_compressed; tfile_end ] -@show MINE == REFERENCE_BYTES - function write_rntuple(file::IO, table; rntuple_name="myntuple") rnt_write(file, file_preamble) + FileHeader32_update = Dict{Symbol, Any}() rnt_write(file, fileheader) rnt_write(file, dummy_padding1) + FileHeader32_update[:fBEGIN] = UInt32(position(file)) + rnt_write(file, tkey32_tfile) rnt_write(file, tfile) + DirectoryHeader32_update = Dict{Symbol, Any}() rnt_write(file, tdirectory32) rnt_write(file, dummy_padding2) @@ -995,13 +999,23 @@ function write_rntuple(file::IO, table; rntuple_name="myntuple") rnt_write(file, magic_6bytes) rnt_write(file, rnt_anchor) - rnt_write(file, [tkey32_TDirectory; n_keys]) + DirectoryHeader32_update[:fSeekKeys] = UInt32(position(file)) + rnt_write(file, tkey32_TDirectory) + rnt_write(file, n_keys) rnt_write(file, tkey32_anchor) + + FileHeader32_update[:fSeekInfo] = UInt32(position(file)) rnt_write(file, tkey32_TStreamerInfo) rnt_write(file, tsreamerinfo_compressed) + FileHeader32_update[:fSeekFree] = UInt32(position(file)) rnt_write(file, tfile_end) + FileHeader32_update[:fEND] = UInt32(position(file)) + + @show FileHeader32_update + @show DirectoryHeader32_update end myio = IOBuffer() write_rntuple(myio, []) +@show MINE == REFERENCE_BYTES @show MINE == take!(myio) From 4dd09bc289c1d320d05b20aac564b8f60939d3cc Mon Sep 17 00:00:00 2001 From: Moelf Date: Thu, 20 Jun 2024 23:26:55 -0400 Subject: [PATCH 09/14] add tests --- src/RNTuple/Writing/TFileWriter.jl | 9 +- src/UnROOT.jl | 2 + src/types.jl | 2 +- test/RNTupleWriting/lowlevel.jl | 431 +++++++++++++++++++++++++++++ test/runtests.jl | 39 +-- 5 files changed, 459 insertions(+), 24 deletions(-) create mode 100644 test/RNTupleWriting/lowlevel.jl diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index fbb76230..438d980d 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -201,7 +201,6 @@ struct FileHeader32 fUUID::SVector{18, UInt8} # Universal Unique ID end """ -fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) function rnt_write(io::IO, x::UnROOT.FileHeader32) rnt_write(io, x.fBEGIN; legacy=true) rnt_write(io, x.fEND; legacy=true) @@ -216,6 +215,8 @@ function rnt_write(io::IO, x::UnROOT.FileHeader32) rnt_write(io, x.fUUID; legacy=true) end +fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) + dummy_FileHeader = [ 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, @@ -246,7 +247,6 @@ struct TKey32 fTitle::String end """ -tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") function rnt_write(io::IO, x::UnROOT.TKey32) rnt_write(io, x.fNbytes; legacy=true) rnt_write(io, x.fVersion; legacy=true) @@ -261,6 +261,7 @@ function rnt_write(io::IO, x::UnROOT.TKey32) rnt_write(io, x.fTitle; legacy=true) end +tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") dummy_tkey32_tfile = [ 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, @@ -274,11 +275,11 @@ struct TFile_write filename::String unknown::String end -tfile = TFile_write("test_ntuple_minimal.root", "") function rnt_write(io::IO, x::TFile_write) rnt_write(io, x.filename; legacy=true) rnt_write(io, x.unknown; legacy=true) end +tfile = TFile_write("test_ntuple_minimal.root", "") dummy_tfile = [ 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, @@ -298,7 +299,6 @@ struct ROOTDirectoryHeader32 fSeekKeys::Int32 end """ -tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) rnt_write(io, x.fVersion; legacy=true) rnt_write(io, x.fDatimeC; legacy=true) @@ -309,6 +309,7 @@ function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) rnt_write(io, x.fSeekParent; legacy=true) rnt_write(io, x.fSeekKeys; legacy=true) end +tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) dummy_tdirectory32 = [ 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, diff --git a/src/UnROOT.jl b/src/UnROOT.jl index 6126c9cf..d4cbd9e4 100644 --- a/src/UnROOT.jl +++ b/src/UnROOT.jl @@ -65,6 +65,8 @@ include("RNTuple/highlevel.jl") include("RNTuple/fieldcolumn_reading.jl") include("RNTuple/displays.jl") +include("RNTuple/Writing/TFileWriter.jl") + _maxthreadid() = @static if VERSION < v"1.9" Threads.nthreads() else diff --git a/src/types.jl b/src/types.jl index e3cb39f1..fc5e08b9 100644 --- a/src/types.jl +++ b/src/types.jl @@ -110,7 +110,7 @@ end iscompressed(t::T) where T<:Union{TKey, TBasketKey} = t.fObjlen != t.fNbytes - t.fKeylen origin(t::T) where T<:Union{TKey, TBasketKey} = iscompressed(t) ? -t.fKeylen : t.fSeekKey -seekstart(io, t::T) where T<:Union{TKey, TBasketKey} = seek(io, t.fSeekKey + t.fKeylen) +Base.seekstart(io, t::T) where T<:Union{TKey, TBasketKey} = seek(io, t.fSeekKey + t.fKeylen) datastream(io, tkey::TKey) = IOBuffer(decompress_datastreambytes(compressed_datastream(io, tkey), tkey)) diff --git a/test/RNTupleWriting/lowlevel.jl b/test/RNTupleWriting/lowlevel.jl new file mode 100644 index 00000000..19cbaf5f --- /dev/null +++ b/test/RNTupleWriting/lowlevel.jl @@ -0,0 +1,431 @@ +using UnROOT: rnt_write, RNTupleFrame, ClusterSummary, PageDescription, Write_RNTupleListFrame, RBlob +using XXHashNative: xxh3_64 + +const REFERENCE_BYTES = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, + 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, + 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, + 0x46, 0x69, 0x6C, 0x65, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, + 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x18, 0x74, + 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, + 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, + 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, + 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, 0x20, 0x76, + 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, + 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, + 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, + 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, 0xCE, 0xCE, 0xCE, 0xCE, 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, + 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, + 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, + 0x25, 0x1E, 0x55, 0x4C, 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, + 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, + 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, + 0xFE, 0x3C, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, + 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, + 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, + 0x3A, 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, + 0x65, 0x00, 0x40, 0x00, 0x00, 0x42, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, + 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, + 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, + 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, + 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, + 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, + 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, + 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, + 0x00, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, + 0x40, 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, + 0x74, 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, + 0x6F, 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, + 0x74, 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, + 0xC2, 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, + 0x15, 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, + 0xDA, 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, + 0x04, 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, + 0xDE, 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, + 0x2A, 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, + 0x37, 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, + 0x64, 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, + 0xD6, 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, + 0xC5, 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, + 0x0C, 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, + 0x8A, 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, + 0x17, 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, + 0x92, 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, + 0x0F, 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, + 0x7C, 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, + 0x58, 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, + 0xF2, 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, + 0xC7, 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, + 0x90, 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, + 0x2E, 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, + 0x15, 0x3D, 0xC1, 0xCA, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, + 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, + 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, + 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, + 0x35, 0x94, 0x00, +] + +function test_io(obj, expected; kw...) + a = IOBuffer() + rnt_write(a, obj; kw...) + ours = take!(a) + @test ours == expected +end + +@testset "RNTuple Writing - Internal" begin + +file_preamble = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, +] + +fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +dummy_FileHeader = [ + 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(fileheader, dummy_FileHeader) + +dummy_padding1 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] + + +tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") +dummy_tkey32_tfile = [ + 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, + 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, + 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, + 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, +] +test_io(tkey32_tfile, dummy_tkey32_tfile) + +tfile = UnROOT.TFile_write("test_ntuple_minimal.root", "") +dummy_tfile = [ + 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, + 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, +] +test_io(tfile, dummy_tfile) + +tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) +dummy_tdirectory32 = [ + 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, + 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, +] +test_io(tdirectory32, dummy_tdirectory32) + +dummy_padding2 = [ + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + + +RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") +dummy_RBlob1 = [ + 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(RBlob1, dummy_RBlob1) + +rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ + UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), +], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + + +# ==================================== side tests begin ==================================== + +field_record = UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", "") +dummy_field_record = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, + 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(field_record, dummy_field_record) + +column_record = UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0) +dummy_column_record = [ + 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(column_record, dummy_column_record) + +envelope_frame_field_record = Write_RNTupleListFrame([field_record]) +dummy_envelope_frame_field_record = [ + 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, + 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, + 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] +test_io(envelope_frame_field_record, dummy_envelope_frame_field_record) + +dummy_rnt_header_payload = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, + 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, + 0x20, 0x76, 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, + 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, +] +test_io(rnt_header, dummy_rnt_header_payload; envelope=false) + +# ==================================== side tests end ==================================== + +dummy_rnt_header = [ + 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, +] +test_io(rnt_header, dummy_rnt_header; envelope=true) + +RBlob2 = RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "") +dummy_RBlob2 = [ + 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(RBlob2, dummy_RBlob2) + +page1 = [ + 0xCE, 0xCE, 0xCE, 0xCE, +] + +RBlob3 = RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "") +dummy_RBlob3 = [ + 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(RBlob3, dummy_RBlob3) + +# ================= side tests begin ================= +dummy_cluster_summary = [ + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, +] +cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, 1)]) +test_io(cluster_summary, dummy_cluster_summary) + +# > https://github.com/root-project/root/blob/1a854602e42d4493f56a26e35e19bdf23b7d0933/tree/ntuple/v7/doc/specifications.md?plain=1#L672 +# > The inner list is followed by a 64bit unsigned integer element offset and the 32bit compression settings +dummy_inner_list_frame = [ + 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + +innert_list_frame = UnROOT.RNTuplePageInnerList([ + PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), +]) +test_io(innert_list_frame, dummy_inner_list_frame) +# ================= side tests end ================= + +nested_page_locations = +UnROOT.RNTuplePageTopList([ + UnROOT.RNTuplePageOuterList([ + UnROOT.RNTuplePageInnerList([ + PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), + ]), + ]), +]) + +pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) + +# ================= side tests begin ================= +dummy_pagelink_noenvelope = [ + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 +] +test_io(pagelink, dummy_pagelink_noenvelope; envelope=false) +# ================= side tests end ================= + + +dummy_pagelink = [ + 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, + 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, + 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, 0x25, 0x1E, 0x55, 0x4C, +] +test_io(pagelink, dummy_pagelink) + +RBlob4 = RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "") +dummy_RBlob4 = [ + 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, + 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, + 0x00, 0x00, +] +test_io(RBlob4, dummy_RBlob4) + +rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))), +], UnROOT.EnvLink[]) +dummy_rnt_footer = [ + 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, + 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, + 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, 0xFE, 0x3C, +] +test_io(rnt_footer, dummy_rnt_footer) + +tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "") +dummy_tkey32_anchor = [ + 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, + 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, + 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, +] +# these 6 bytes are between tkey32_anchor and the actual anchor +magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04] + +rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b) +dummy_rnt_anchor = [ + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, +] +test_io(rnt_anchor, dummy_rnt_anchor) + +tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "") +dummy_tkey32_TDirectory = [ + 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, +] +test_io(tkey32_TDirectory, dummy_tkey32_TDirectory) + +# 1 key, and it is the RNTuple Anchor +n_keys = [ + 0x00, 0x00, 0x00, 0x01, +] + + +tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list") +dummy_tkey32_TStreamerInfo = [ + 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, + 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, 0x74, + 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, 0x6F, + 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, 0x74, +] +test_io(tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) + +tsreamerinfo_compressed = [ + 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, + 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15, + 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA, + 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04, + 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE, + 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A, + 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37, + 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64, + 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6, + 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5, + 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C, + 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A, + 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17, + 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92, + 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F, + 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C, + 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58, + 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2, + 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7, + 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90, + 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E, + 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15, + 0x3D, 0xC1, 0xCA, +] + +tfile_end = [ + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00, +] + + +MINE = [ + file_preamble; + dummy_FileHeader; dummy_padding1; + dummy_tkey32_tfile; dummy_tfile; + dummy_tdirectory32; dummy_padding2; + dummy_RBlob1; dummy_rnt_header; + dummy_RBlob2; page1; + dummy_RBlob3; dummy_pagelink; + dummy_RBlob4; dummy_rnt_footer; + dummy_tkey32_anchor; magic_6bytes; dummy_rnt_anchor; + dummy_tkey32_TDirectory; n_keys; dummy_tkey32_anchor; + dummy_tkey32_TStreamerInfo; tsreamerinfo_compressed; + tfile_end +] + +myio = IOBuffer() +UnROOT.write_rntuple(myio, []) +@test MINE == REFERENCE_BYTES +@test MINE == take!(myio) +end diff --git a/test/runtests.jl b/test/runtests.jl index 0221f962..a8b3cf0f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,28 +5,29 @@ nthreads = UnROOT._maxthreadid() nthreads == 1 && @warn "Running on a single thread. Please re-run the test suite with at least two threads (`julia --threads 2 ...`)" @testset "UnROOT tests" verbose = true begin - include("Aqua.jl") - include("bootstrapping.jl") - include("compressions.jl") - include("jagged.jl") - include("lazy.jl") - include("histograms.jl") - include("views.jl") - include("multithreading.jl") - include("remote.jl") - include("displays.jl") - include("type_stability.jl") - include("utils.jl") - include("misc.jl") + # include("Aqua.jl") + # include("bootstrapping.jl") + # include("compressions.jl") + # include("jagged.jl") + # include("lazy.jl") + # include("histograms.jl") + # include("views.jl") + # include("multithreading.jl") + # include("remote.jl") + # include("displays.jl") + # include("type_stability.jl") + # include("utils.jl") + # include("misc.jl") - include("type_support.jl") - include("custom_bootstrapping.jl") - include("lorentzvectors.jl") - include("NanoAOD.jl") + # include("type_support.jl") + # include("custom_bootstrapping.jl") + # include("lorentzvectors.jl") + # include("NanoAOD.jl") - include("issues.jl") + # include("issues.jl") if VERSION >= v"1.9" - include("rntuple.jl") + # include("rntuple.jl") + include("./RNTupleWriting/lowlevel.jl") end end From aa9a51a5719e23ef70b8f01931aa33d0e8e613c6 Mon Sep 17 00:00:00 2001 From: Moelf Date: Sun, 30 Jun 2024 01:32:58 -0400 Subject: [PATCH 10/14] finish phase 1 phase 1 bump to 0.10.31 (#341) Release for XRootD.jl fix compat --- Project.toml | 2 + src/RNTuple/Writing/Stubs.jl | 97 ++++ src/RNTuple/Writing/TFileWriter.jl | 698 +++++------------------------ src/UnROOT.jl | 1 + test/RNTupleWriting/lowlevel.jl | 171 +++---- test/runtests.jl | 38 +- 6 files changed, 292 insertions(+), 715 deletions(-) create mode 100644 src/RNTuple/Writing/Stubs.jl diff --git a/Project.toml b/Project.toml index c264a955..543955d2 100644 --- a/Project.toml +++ b/Project.toml @@ -5,6 +5,7 @@ version = "0.10.31" [deps] AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" ArraysOfArrays = "65a8f2f4-9b39-5baf-92e2-a9cc46fdf018" BitIntegers = "c3b6d118-76ef-56ca-8cc7-ebb389d030a1" CodecLz4 = "5ba52731-8f18-5e0d-9241-30f10d1ec561" @@ -32,6 +33,7 @@ XXHashNative = "e5d8e439-e7fa-4681-9c12-1c64bda517be" [compat] AbstractTrees = "^0.4" +Accessors = "0.1.36" Aqua = "^0.8" ArraysOfArrays = "^0.6" Arrow = "~2.7.1" diff --git a/src/RNTuple/Writing/Stubs.jl b/src/RNTuple/Writing/Stubs.jl new file mode 100644 index 00000000..8bd7df97 --- /dev/null +++ b/src/RNTuple/Writing/Stubs.jl @@ -0,0 +1,97 @@ +module Stubs +using ..UnROOT + +const file_preamble = [ + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, +] + +const fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) +const dummy_padding1 = [ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, +] + +const tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") +const tfile = UnROOT.TFile_write("test_ntuple_minimal.root", "") +const tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) +const dummy_padding2 = [ + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +] + + +const RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") +const rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ + UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), +], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + + +const RBlob2 = UnROOT.RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "") +const page1 = [ + 0xCE, 0xCE, 0xCE, 0xCE, +] + +const RBlob3 = UnROOT.RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "") +const cluster_summary = UnROOT.Write_RNTupleListFrame([UnROOT.ClusterSummary(0, 1)]) +const nested_page_locations = +UnROOT.RNTuplePageTopList([ + UnROOT.RNTuplePageOuterList([ + UnROOT.RNTuplePageInnerList([ + UnROOT.PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), + ]), + ]), +]) +const pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) + +const RBlob4 = UnROOT.RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "") +const rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))), +], UnROOT.EnvLink[]) +const tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "") +# these 6 bytes are between tkey32_anchor and the actual anchor +const magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04] + +const rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b) +const tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "") +# 1 key, and it is the RNTuple Anchor +const n_keys = [ + 0x00, 0x00, 0x00, 0x01, +] + + +const tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list") + +const tsreamerinfo_compressed = [ + 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, + 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15, + 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA, + 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04, + 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE, + 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A, + 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37, + 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64, + 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6, + 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5, + 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C, + 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A, + 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17, + 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92, + 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F, + 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C, + 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58, + 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2, + 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7, + 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90, + 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E, + 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15, + 0x3D, 0xC1, 0xCA, +] + +const tfile_end = [ + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, + 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, + 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, + 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00, +] +end diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index 438d980d..44f8a10c 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -2,110 +2,8 @@ using StaticArrays using UnROOT using UnROOT: RNTupleFrame, ClusterSummary, PageDescription using XXHashNative: xxh3_64 - -const REFERENCE_BYTES = [ - 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, - 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, - 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, - 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, - 0x46, 0x69, 0x6C, 0x65, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, - 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x18, 0x74, - 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, - 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, - 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, - 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, - 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, - 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, 0x20, 0x76, - 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, - 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, - 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, - 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, - 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, - 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, - 0x00, 0x00, 0xCE, 0xCE, 0xCE, 0xCE, 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, - 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, - 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, - 0x25, 0x1E, 0x55, 0x4C, 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, - 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, - 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, - 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, - 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, - 0xFE, 0x3C, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, - 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, - 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, - 0x3A, 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, - 0x65, 0x00, 0x40, 0x00, 0x00, 0x42, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, - 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, - 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, - 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, - 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, - 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, - 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, - 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, - 0x00, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, - 0x40, 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, - 0x74, 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, - 0x6F, 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, - 0x74, 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, - 0xC2, 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, - 0x15, 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, - 0xDA, 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, - 0x04, 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, - 0xDE, 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, - 0x2A, 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, - 0x37, 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, - 0x64, 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, - 0xD6, 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, - 0xC5, 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, - 0x0C, 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, - 0x8A, 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, - 0x17, 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, - 0x92, 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, - 0x0F, 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, - 0x7C, 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, - 0x58, 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, - 0xF2, 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, - 0xC7, 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, - 0x90, 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, - 0x2E, 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, - 0x15, 0x3D, 0xC1, 0xCA, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, - 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, - 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, - 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, - 0x35, 0x94, 0x00, -] +using Accessors +using Tables: istable, columntable, schema function color_diff(ary1, ary2) if length(ary1) != length(ary2) @@ -175,32 +73,6 @@ function test_io(obj, expected; kw...) end end -""" -struct FilePreamble - identifier::SVector{4, UInt8} # Root file identifier ("root") - fVersion::Int32 # File format version -end -""" -# file_preamble = UnROOT.FilePreamble(b"root", 0xf745) -file_preamble = [ - 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, -] - -""" -struct FileHeader32 - fBEGIN::Int32 # Pointer to first data record - fEND::UInt32 # Pointer to first free word at the EOF - fSeekFree::UInt32 # Pointer to FREE data record - fNbytesFree::Int32 # Number of bytes in FREE data record - nfree::Int32 # Number of free data records - fNbytesName::Int32 # Number of bytes in TNamed at creation time - fUnits::UInt8 # Number of bytes for file pointers - fCompress::Int32 # Compression level and algorithm - fSeekInfo::UInt32 # Pointer to TStreamerInfo record - fNbytesInfo::Int32 # Number of bytes in TStreamerInfo record - fUUID::SVector{18, UInt8} # Universal Unique ID -end -""" function rnt_write(io::IO, x::UnROOT.FileHeader32) rnt_write(io, x.fBEGIN; legacy=true) rnt_write(io, x.fEND; legacy=true) @@ -215,39 +87,8 @@ function rnt_write(io::IO, x::UnROOT.FileHeader32) rnt_write(io, x.fUUID; legacy=true) end -fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) - -dummy_FileHeader = [ - 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, - 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -] - -test_io(fileheader, dummy_FileHeader) - -dummy_padding1 = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, -] - -""" -struct TKey32 - fNbytes::Int32 - fVersion::Int16 - fObjlen::Int32 - fDatime::UInt32 - fKeylen::Int16 - fCycle::Int16 - fSeekKey::Int32 - fSeekPdir::Int32 - fClassName::String - fName::String - fTitle::String -end -""" function rnt_write(io::IO, x::UnROOT.TKey32) + p = position(io) rnt_write(io, x.fNbytes; legacy=true) rnt_write(io, x.fVersion; legacy=true) rnt_write(io, x.fObjlen; legacy=true) @@ -259,18 +100,9 @@ function rnt_write(io::IO, x::UnROOT.TKey32) rnt_write(io, x.fClassName; legacy=true) rnt_write(io, x.fName; legacy=true) rnt_write(io, x.fTitle; legacy=true) + @assert position(io) - p == x.fKeylen end -tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") -dummy_tkey32_tfile = [ - 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, - 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, - 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, - 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, -] - -test_io(tkey32_tfile, dummy_tkey32_tfile) - struct TFile_write filename::String unknown::String @@ -279,26 +111,7 @@ function rnt_write(io::IO, x::TFile_write) rnt_write(io, x.filename; legacy=true) rnt_write(io, x.unknown; legacy=true) end -tfile = TFile_write("test_ntuple_minimal.root", "") -dummy_tfile = [ - 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, - 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, -] -test_io(tfile, dummy_tfile) - -""" -struct ROOTDirectoryHeader32 - fVersion::Int16 - fDatimeC::UInt32 - fDatimeM::UInt32 - fNbytesKeys::Int32 - fNbytesName::Int32 - fSeekDir::Int32 - fSeekParent::Int32 - fSeekKeys::Int32 -end -""" function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) rnt_write(io, x.fVersion; legacy=true) rnt_write(io, x.fDatimeC; legacy=true) @@ -309,17 +122,6 @@ function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) rnt_write(io, x.fSeekParent; legacy=true) rnt_write(io, x.fSeekKeys; legacy=true) end -tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) -dummy_tdirectory32 = [ - 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, - 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, -] -test_io(tdirectory32, dummy_tdirectory32) - -dummy_padding2 = [ - 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -] struct RBlob fNbytes::Int32 @@ -348,49 +150,6 @@ function rnt_write(io::IO, x::RBlob) rnt_write(io, x.fTitle; legacy=true) end -RBlob1 = RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") -dummy_RBlob1 = [ - 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, - 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, - 0x00, 0x00, -] -test_io(RBlob1, dummy_RBlob1) - - -""" -UnROOT.RNTupleHeader - feature_flag: UInt64 0x0000000000000000 - name: String "myntuple" - ntuple_description: String "" - writer_identifier: String "ROOT v6.33.01" - field_records: Array{UnROOT.FieldRecord}((1,)) - 1: UnROOT.FieldRecord - field_version: UInt32 0x00000000 - type_version: UInt32 0x00000000 - parent_field_id: UInt32 0x00000000 - struct_role: UInt16 0x0000 - flags: UInt16 0x0000 - repetition: Int64 0 - field_name: String "one_uint" - type_name: String "std::uint32_t" - type_alias: String "" - field_desc: String "" - column_records: Array{UnROOT.ColumnRecord}((1,)) - 1: UnROOT.ColumnRecord - type: UInt16 0x0014 - nbits: UInt16 0x0020 - field_id: UInt32 0x00000000 - flags: UInt32 0x00000000 - first_ele_idx: Int64 0 - alias_columns: Array{UnROOT.AliasRecord}((0,)) - extra_type_infos: Array{UnROOT.ExtraTypeInfo}((0,)) -""" -rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ - UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), -], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) - - -# ==================================== side tests begin ==================================== function rnt_write(io::IO, x::UnROOT.FieldRecord) rnt_write(io, x.field_version) rnt_write(io, x.type_version) @@ -408,14 +167,6 @@ function rnt_write(io::IO, x::UnROOT.FieldRecord) rnt_write(io, x.type_alias) rnt_write(io, x.field_desc) end -field_record = UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", "") -dummy_field_record = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, - 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, -] -test_io(field_record, dummy_field_record) function rnt_write(io::IO, x::UnROOT.ColumnRecord) rnt_write(io, x.type) @@ -484,17 +235,6 @@ function rnt_write(io::IO, x::Write_RNTupleListFrame) write(io, temp_io) end -envelope_frame_field_record = Write_RNTupleListFrame([field_record]) -dummy_envelope_frame_field_record = [ - 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, - 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, - 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -] - -test_io(envelope_frame_field_record, dummy_envelope_frame_field_record) - function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) temp_io = IOBuffer() rnt_write(temp_io, x.feature_flag) @@ -524,32 +264,6 @@ function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) end end -dummy_rnt_header_payload = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x6D, 0x79, 0x6E, 0x74, - 0x75, 0x70, 0x6C, 0x65, 0x00, 0x00, 0x00, 0x00, 0x0D, 0x00, 0x00, 0x00, 0x52, 0x4F, 0x4F, 0x54, - 0x20, 0x76, 0x36, 0x2E, 0x33, 0x33, 0x2E, 0x30, 0x31, 0xB7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x01, 0x00, 0x00, 0x00, 0x3D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, - 0x00, 0x6F, 0x6E, 0x65, 0x5F, 0x75, 0x69, 0x6E, 0x74, 0x0D, 0x00, 0x00, 0x00, 0x73, 0x74, 0x64, - 0x3A, 0x3A, 0x75, 0x69, 0x6E, 0x74, 0x33, 0x32, 0x5F, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, -] - -test_io(rnt_header, dummy_rnt_header_payload; envelope=false) - -# ==================================== side tests end ==================================== - -# reading struct looks like this: -# struct RNTupleEnvelope{T} - # type_id::UInt16 - # envelope_length::UInt64 - # payload::T - # checksum::UInt64 -# end -# function rnt_write_envelope(io::IO, x; type_id) temp_io = IOBuffer() rnt_write(temp_io, x.payload) @@ -559,31 +273,6 @@ function rnt_write_envelope(io::IO, x; type_id) write(io, temp_io) end -dummy_rnt_header = [ - 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, -] -test_io(rnt_header, dummy_rnt_header; envelope=true) - -RBlob2 = RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "") -dummy_RBlob2 = [ - 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, - 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, - 0x00, 0x00, -] -test_io(RBlob2, dummy_RBlob2) - -page1 = [ - 0xCE, 0xCE, 0xCE, 0xCE, -] - -RBlob3 = RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "") -dummy_RBlob3 = [ - 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, - 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, - 0x00, 0x00, -] -test_io(RBlob3, dummy_RBlob3) - function rnt_write(io::IO, x::ClusterSummary) rnt_write(io, x.first_entry_number) rnt_write(io, x.number_of_entries) @@ -641,37 +330,6 @@ function rnt_write(io::IO, x::UnROOT.RNTuplePageInnerList) seekstart(temp_io) write(io, temp_io) end -# ================= side tests begin ================= -dummy_cluster_summary = [ - 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, -] -cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, 1)]) -test_io(cluster_summary, dummy_cluster_summary) - -# > https://github.com/root-project/root/blob/1a854602e42d4493f56a26e35e19bdf23b7d0933/tree/ntuple/v7/doc/specifications.md?plain=1#L672 -# > The inner list is followed by a 64bit unsigned integer element offset and the 32bit compression settings -dummy_inner_list_frame = [ - 0xD8, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -] - -innert_list_frame = UnROOT.RNTuplePageInnerList([ - PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), -]) -test_io(innert_list_frame, dummy_inner_list_frame) -# ================= side tests end ================= - -nested_page_locations = -UnROOT.RNTuplePageTopList([ - UnROOT.RNTuplePageOuterList([ - UnROOT.RNTuplePageInnerList([ - PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), - ]), - ]), -]) function rnt_write(io::IO, x::UnROOT.PageLink; envelope=true) temp_io = IOBuffer() @@ -696,60 +354,7 @@ function rnt_write(io::IO, x::UnROOT.PageLink; envelope=true) write(io, payload_ary) end end -pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) - -# ================= side tests begin ================= -dummy_pagelink_noenvelope = [ - 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, - 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, - 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00 -] -test_io(pagelink, dummy_pagelink_noenvelope; envelope=false) -# ================= side tests end ================= - - -dummy_pagelink = [ - 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, - 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xC0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, - 0xCC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0xD8, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, 0x25, 0x1E, 0x55, 0x4C, -] -test_io(pagelink, dummy_pagelink) - -RBlob4 = RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "") -dummy_RBlob4 = [ - 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, - 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, - 0x00, 0x00, -] -test_io(RBlob4, dummy_RBlob4) - -""" -UnROOT.RNTupleFooter: - feature_flag: 0 - header_checksum: 0x3dec59c009c67e28 - extension_header_links: UnROOT.RNTupleSchemaExtension(UnROOT.FieldRecord[], UnROOT.ColumnRecord[], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) - column_group_records: UnROOT.ColumnGroupRecord[] - cluster_group_records: UnROOT.ClusterGroupRecord[UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(num_bytes=124, offset=0x0000000000000218, ) -))] - meta_data_links: UnROOT.EnvLink[] -""" -""" -@SimpleStruct struct EnvLink - uncomp_size::UInt64 - locator::Locator -end -""" function rnt_write(io::IO, x::UnROOT.EnvLink) rnt_write(io, x.uncomp_size) rnt_write(io, x.locator) @@ -810,62 +415,6 @@ function rnt_write(io::IO, x::UnROOT.RNTupleFooter; envelope=true) end end -rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ - UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))), -], UnROOT.EnvLink[]) -dummy_rnt_footer = [ - 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, - 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7C, 0x00, 0x00, 0x00, - 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, 0xFE, 0x3C, -] -test_io(rnt_footer, dummy_rnt_footer) - -""" -UnROOT.TKey32 - fNbytes: Int32 134 - fVersion: Int16 4 - fObjlen: Int32 70 - fDatime: UInt32 0x7567176d - fKeylen: Int16 64 - fCycle: Int16 1 - fSeekKey: Int32 866 - fSeekPdir: Int32 100 - fClassName: String "ROOT::Experimental::RNTuple" - fName: String "myntuple" - fTitle: String "" -""" -tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "") -dummy_tkey32_anchor = [ - 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, - 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, - 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, - 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, -] -# these 6 bytes are between tkey32_anchor and the actual anchor -magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04] - -""" -UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple - fVersionEpoch: UInt16 0x0000 - fVersionMajor: UInt16 0x0002 - fVersionMinor: UInt16 0x0000 - fVersionPatch: UInt16 0x0000 - fSeekHeader: UInt64 0x0000000000000116 - fNBytesHeader: UInt64 0x00000000000000ba - fLenHeader: UInt64 0x00000000000000ba - fSeekFooter: UInt64 0x00000000000002b6 - fNBytesFooter: UInt64 0x00000000000000ac - fLenFooter: UInt64 0x00000000000000ac - fChecksum: UInt64 0xdc495fd01479af1b -""" function rnt_write(io::IO, x::UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple) rnt_write(io, x.fVersionEpoch; legacy=true) rnt_write(io, x.fVersionMajor; legacy=true) @@ -880,143 +429,140 @@ function rnt_write(io::IO, x::UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple) rnt_write(io, x.fChecksum; legacy=true) end -rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b) -dummy_rnt_anchor = [ - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, -] -test_io(rnt_anchor, dummy_rnt_anchor) - -tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "") -dummy_tkey32_TDirectory = [ - 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, - 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, - 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, - 0x72, 0x6F, 0x6F, 0x74, 0x00, -] -test_io(tkey32_TDirectory, dummy_tkey32_TDirectory) +mutable struct WriteObservable{O, T} + io::O + position::Int64 + object::T +end -# 1 key, and it is the RNTuple Anchor -n_keys = [ - 0x00, 0x00, 0x00, 0x01, -] +function Base.setindex!(io::WriteObservable, val, key::Symbol) + new_obj = set(io.object, PropertyLens(key), val) + io.object = new_obj + return io +end -@assert dummy_tkey32_anchor == [ - 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, - 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, - 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, - 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, -] +function Base.setindex!(io::WriteObservable, dict::Dict) + for (key, val) in dict + new_obj = set(io.object, PropertyLens(key), val) + io.object = new_obj + end + return io +end +function flush!(o::WriteObservable) + io = o.io + old_pos = position(io) -tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list") -dummy_tkey32_TStreamerInfo = [ - 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, - 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, 0x74, - 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, 0x6F, - 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, 0x74, -] -test_io(tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) - -tsreamerinfo_compressed = [ - 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, - 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15, - 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA, - 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04, - 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE, - 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A, - 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37, - 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64, - 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6, - 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5, - 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C, - 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A, - 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17, - 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92, - 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F, - 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C, - 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58, - 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2, - 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7, - 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90, - 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E, - 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15, - 0x3D, 0xC1, 0xCA, -] - -tfile_end = [ - 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, - 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, - 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, - 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00, -] + seek(io, o.position) + rnt_write(io, o.object) + seek(io, old_pos) + nothing +end -MINE = [ - file_preamble; - dummy_FileHeader; dummy_padding1; - dummy_tkey32_tfile; dummy_tfile; - dummy_tdirectory32; dummy_padding2; - dummy_RBlob1; dummy_rnt_header; - dummy_RBlob2; page1; - dummy_RBlob3; dummy_pagelink; - dummy_RBlob4; dummy_rnt_footer; - dummy_tkey32_anchor; magic_6bytes; dummy_rnt_anchor; - dummy_tkey32_TDirectory; n_keys; dummy_tkey32_anchor; - dummy_tkey32_TStreamerInfo; tsreamerinfo_compressed; - tfile_end -] +function rnt_write_observe(io::IO, x::T) where T + pos = position(io) + rnt_write(io, x) + WriteObservable(io, pos, x) +end -function write_rntuple(file::IO, table; rntuple_name="myntuple") - rnt_write(file, file_preamble) - FileHeader32_update = Dict{Symbol, Any}() - rnt_write(file, fileheader) - rnt_write(file, dummy_padding1) +function split4_encode(src::AbstractVector{UInt8}) + @views [src[1:4:end-3]; src[2:4:end-2]; src[3:4:end-1]; src[4:4:end]] +end - FileHeader32_update[:fBEGIN] = UInt32(position(file)) +function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rntuple_name="myntuple") + if !istable(table) + error("RNTuple writing accepts object compatible with Tables.jl interface, got type $(typeof(table))") + end - rnt_write(file, tkey32_tfile) - rnt_write(file, tfile) + input_schema = schema(table) + input_Ncols = length(input_schema.names) + if input_Ncols != 1 + error("Currently, RNTuple writing only supports a single, UInt32 column, got $input_Ncols columns") + end + input_T = only(input_schema.types) + if input_T != UInt32 + error("Currently, RNTuple writing only supports a single, UInt32 column, got type $input_T") + end + input_col = only(columntable(table)) + input_length = length(input_col) + if input_length > 65535 + error("Input too long: RNTuple writing currently only supports a single page (65535 elements)") + end - DirectoryHeader32_update = Dict{Symbol, Any}() - rnt_write(file, tdirectory32) - rnt_write(file, dummy_padding2) - rnt_write(file, RBlob1) - rnt_write(file, rnt_header) + rntAnchor_update = Dict{Symbol, Any}() - rnt_write(file, RBlob2) - rnt_write(file, page1) + file_preamble_obs = rnt_write_observe(file, Stubs.file_preamble) + fileheader_obs = rnt_write_observe(file, Stubs.fileheader) + dummy_padding1_obs = rnt_write_observe(file, Stubs.dummy_padding1) - rnt_write(file, RBlob3) - rnt_write(file, pagelink) + fileheader_obs[:fBEGIN] = UInt32(position(file)) - rnt_write(file, RBlob4) - rnt_write(file, rnt_footer) + tkey32_tfile_obs = rnt_write_observe(file, Stubs.tkey32_tfile) + tkey32_tfile_obs[:fName] = file_name + tfile_obs = rnt_write_observe(file, Stubs.tfile) - rnt_write(file, tkey32_anchor) - rnt_write(file, magic_6bytes) - rnt_write(file, rnt_anchor) + tdirectory32_obs = rnt_write_observe(file, Stubs.tdirectory32) + dummy_padding2_obs = rnt_write_observe(file, Stubs.dummy_padding2) - DirectoryHeader32_update[:fSeekKeys] = UInt32(position(file)) - rnt_write(file, tkey32_TDirectory) - rnt_write(file, n_keys) - rnt_write(file, tkey32_anchor) + RBlob1_obs = rnt_write_observe(file, Stubs.RBlob1) + rntAnchor_update[:fSeekHeader] = UInt32(position(file)) + rnt_header_obs = rnt_write_observe(file, Stubs.rnt_header) + rntAnchor_update[:fNBytesHeader] = 0xba + rntAnchor_update[:fLenHeader] = 0xba - FileHeader32_update[:fSeekInfo] = UInt32(position(file)) - rnt_write(file, tkey32_TStreamerInfo) - rnt_write(file, tsreamerinfo_compressed) - FileHeader32_update[:fSeekFree] = UInt32(position(file)) - rnt_write(file, tfile_end) - FileHeader32_update[:fEND] = UInt32(position(file)) + RBlob2_obs = rnt_write_observe(file, Stubs.RBlob2) + page1 = reinterpret(UInt8, input_col) + page1_bytes = split4_encode(page1) + page1_position = position(file) + page1_obs = rnt_write_observe(file, page1_bytes) - @show FileHeader32_update - @show DirectoryHeader32_update + RBlob3_obs = rnt_write_observe(file, Stubs.RBlob3) + cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, input_length)]) + nested_page_locations = + UnROOT.RNTuplePageTopList([ + UnROOT.RNTuplePageOuterList([ + UnROOT.RNTuplePageInnerList([ + PageDescription(input_length, UnROOT.Locator(sizeof(input_T) * input_length, page1_position, )), + ]), + ]), + ]) + + pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) + pagelink_position = position(file) + pagelink_obs = rnt_write_observe(file, pagelink) + + RBlob4_obs = rnt_write_observe(file, Stubs.RBlob4) + rntAnchor_update[:fSeekFooter] = UInt32(position(file)) + rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + UnROOT.ClusterGroupRecord(0, input_length, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, pagelink_position, ))), + ], UnROOT.EnvLink[]) + rnt_footer_obs = rnt_write_observe(file, rnt_footer) + rntAnchor_update[:fNBytesFooter] = 0xac + rntAnchor_update[:fLenFooter] = 0xac + + tkey32_anchor_position = position(file) + tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, tkey32_anchor_position, 100, "ROOT::Experimental::RNTuple", "myntuple", "") + tkey32_anchor_obs1 = rnt_write_observe(file, tkey32_anchor) + magic_6bytes_obs = rnt_write_observe(file, Stubs.magic_6bytes) + rnt_anchor_obs = rnt_write_observe(file, Stubs.rnt_anchor) + Base.setindex!(rnt_anchor_obs, rntAnchor_update) + + tdirectory32_obs[:fSeekKeys] = UInt32(position(file)) + tkey32_TDirectory_obs = rnt_write_observe(file, Stubs.tkey32_TDirectory) + n_keys_obs = rnt_write_observe(file, Stubs.n_keys) + tkey32_anchor_obs2 = rnt_write_observe(file, tkey32_anchor) + + fileheader_obs[:fSeekInfo] = UInt32(position(file)) + tkey32_TStreamerInfo_obs = rnt_write_observe(file, Stubs.tkey32_TStreamerInfo) + tsreamerinfo_compressed_obs = rnt_write_observe(file, Stubs.tsreamerinfo_compressed) + fileheader_obs[:fSeekFree] = UInt32(position(file)) + tfile_end_obs = rnt_write_observe(file, Stubs.tfile_end) + fileheader_obs[:fEND] = UInt32(position(file)) + + flush!(tkey32_tfile_obs) + flush!(tdirectory32_obs) + flush!(fileheader_obs) + flush!(rnt_anchor_obs) end - -myio = IOBuffer() -write_rntuple(myio, []) -@show MINE == REFERENCE_BYTES -@show MINE == take!(myio) diff --git a/src/UnROOT.jl b/src/UnROOT.jl index d4cbd9e4..cddbb185 100644 --- a/src/UnROOT.jl +++ b/src/UnROOT.jl @@ -66,6 +66,7 @@ include("RNTuple/fieldcolumn_reading.jl") include("RNTuple/displays.jl") include("RNTuple/Writing/TFileWriter.jl") +include("RNTuple/Writing/Stubs.jl") _maxthreadid() = @static if VERSION < v"1.9" Threads.nthreads() diff --git a/test/RNTupleWriting/lowlevel.jl b/test/RNTupleWriting/lowlevel.jl index 19cbaf5f..536568cb 100644 --- a/test/RNTupleWriting/lowlevel.jl +++ b/test/RNTupleWriting/lowlevel.jl @@ -1,5 +1,6 @@ using UnROOT: rnt_write, RNTupleFrame, ClusterSummary, PageDescription, Write_RNTupleListFrame, RBlob using XXHashNative: xxh3_64 +using Tables: columntable const REFERENCE_BYTES = [ 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, @@ -114,67 +115,45 @@ end @testset "RNTuple Writing - Internal" begin -file_preamble = [ - 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, -] - -fileheader = UnROOT.FileHeader32(100, 0x00000643, 0x00000604, 63, 1, 84, 0x04, 0, 0x00000461, 419, UInt8[0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]) dummy_FileHeader = [ 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x54, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x01, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] -test_io(fileheader, dummy_FileHeader) - -dummy_padding1 = [ - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, -] +test_io(UnROOT.Stubs.fileheader, dummy_FileHeader) - -tkey32_tfile = UnROOT.TKey32(144, 4, 86, 0x7567176d, 58, 1, 100, 0, "TFile", "test_ntuple_minimal.root", "") dummy_tkey32_tfile = [ 0x00, 0x00, 0x00, 0x90, 0x00, 0x04, 0x00, 0x00, 0x00, 0x56, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x3A, 0x00, 0x01, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x05, 0x54, 0x46, 0x69, 0x6C, 0x65, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] -test_io(tkey32_tfile, dummy_tkey32_tfile) +test_io(UnROOT.Stubs.tkey32_tfile, dummy_tkey32_tfile) -tfile = UnROOT.TFile_write("test_ntuple_minimal.root", "") dummy_tfile = [ 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] -test_io(tfile, dummy_tfile) +test_io(UnROOT.Stubs.tfile, dummy_tfile) -tdirectory32 = UnROOT.ROOTDirectoryHeader32(5, 0x7567176d, 0x7567176d, 121, 84, 100, 0, 1000) dummy_tdirectory32 = [ 0x00, 0x05, 0x75, 0x67, 0x17, 0x6D, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x00, 0x00, 0x79, 0x00, 0x00, 0x00, 0x54, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xE8, ] -test_io(tdirectory32, dummy_tdirectory32) +test_io(UnROOT.Stubs.tdirectory32, dummy_tdirectory32) dummy_padding2 = [ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] - -RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, 0x7567176D, 0x0022, 0x0001, 244, 100, "RBlob", "", "") dummy_RBlob1 = [ 0x00, 0x00, 0x00, 0xDC, 0x00, 0x04, 0x00, 0x00, 0x00, 0xBA, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x00, 0xF4, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] -test_io(RBlob1, dummy_RBlob1) - -rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ - UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, "one_uint", "std::uint32_t", "", ""), -], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) - +test_io(UnROOT.Stubs.RBlob1, dummy_RBlob1) # ==================================== side tests begin ==================================== @@ -216,34 +195,28 @@ dummy_rnt_header_payload = [ 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, ] -test_io(rnt_header, dummy_rnt_header_payload; envelope=false) +test_io(UnROOT.Stubs.rnt_header, dummy_rnt_header_payload; envelope=false) # ==================================== side tests end ==================================== dummy_rnt_header = [ 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, ] -test_io(rnt_header, dummy_rnt_header; envelope=true) +test_io(UnROOT.Stubs.rnt_header, dummy_rnt_header; envelope=true) -RBlob2 = RBlob(0x0026, 0x0004, 0x00000004, 0x7567176D, 0x0022, 0x0001, 0x01D0, 100, "RBlob", "", "") dummy_RBlob2 = [ 0x00, 0x00, 0x00, 0x26, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xD0, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] -test_io(RBlob2, dummy_RBlob2) - -page1 = [ - 0xCE, 0xCE, 0xCE, 0xCE, -] +test_io(UnROOT.Stubs.RBlob2, dummy_RBlob2) -RBlob3 = RBlob(0x009E, 0x0004, 0x0000007C, 0x7567176D, 0x0022, 0x0001, 0x01F6, 100, "RBlob", "", "") dummy_RBlob3 = [ 0x00, 0x00, 0x00, 0x9E, 0x00, 0x04, 0x00, 0x00, 0x00, 0x7C, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x01, 0xF6, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] -test_io(RBlob3, dummy_RBlob3) +test_io(UnROOT.Stubs.RBlob3, dummy_RBlob3) # ================= side tests begin ================= dummy_cluster_summary = [ @@ -251,8 +224,7 @@ dummy_cluster_summary = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] -cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, 1)]) -test_io(cluster_summary, dummy_cluster_summary) +test_io(UnROOT.Stubs.cluster_summary, dummy_cluster_summary) # > https://github.com/root-project/root/blob/1a854602e42d4493f56a26e35e19bdf23b7d0933/tree/ntuple/v7/doc/specifications.md?plain=1#L672 # > The inner list is followed by a 64bit unsigned integer element offset and the 32bit compression settings @@ -262,24 +234,11 @@ dummy_inner_list_frame = [ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, ] -innert_list_frame = UnROOT.RNTuplePageInnerList([ +inner_list_frame = UnROOT.RNTuplePageInnerList([ PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), ]) -test_io(innert_list_frame, dummy_inner_list_frame) -# ================= side tests end ================= - -nested_page_locations = -UnROOT.RNTuplePageTopList([ - UnROOT.RNTuplePageOuterList([ - UnROOT.RNTuplePageInnerList([ - PageDescription(0x00000001, UnROOT.Locator(4, 0x00000000000001f2, )), - ]), - ]), -]) +test_io(inner_list_frame, dummy_inner_list_frame) -pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) - -# ================= side tests begin ================= dummy_pagelink_noenvelope = [ 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0xDC, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, @@ -290,7 +249,7 @@ dummy_pagelink_noenvelope = [ 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 ] -test_io(pagelink, dummy_pagelink_noenvelope; envelope=false) +test_io(UnROOT.Stubs.pagelink, dummy_pagelink_noenvelope; envelope=false) # ================= side tests end ================= @@ -304,19 +263,15 @@ dummy_pagelink = [ 0xF2, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, 0xA0, 0xDF, 0x2E, 0x25, 0x1E, 0x55, 0x4C, ] -test_io(pagelink, dummy_pagelink) +test_io(UnROOT.Stubs.pagelink, dummy_pagelink) -RBlob4 = RBlob(0x00CE, 0x0004, 0x000000AC, 0x7567176D, 0x0022, 0x0001, 0x0294, 100, "RBlob", "", "") dummy_RBlob4 = [ 0x00, 0x00, 0x00, 0xCE, 0x00, 0x04, 0x00, 0x00, 0x00, 0xAC, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x22, 0x00, 0x01, 0x00, 0x00, 0x02, 0x94, 0x00, 0x00, 0x00, 0x64, 0x05, 0x52, 0x42, 0x6C, 0x6F, 0x62, 0x00, 0x00, ] -test_io(RBlob4, dummy_RBlob4) +test_io(UnROOT.Stubs.RBlob4, dummy_RBlob4) -rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ - UnROOT.ClusterGroupRecord(0, 1, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, 0x0000000000000218, ))), -], UnROOT.EnvLink[]) dummy_rnt_footer = [ 0x02, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -330,102 +285,78 @@ dummy_rnt_footer = [ 0x18, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD9, 0x13, 0x0B, 0x80, 0xBB, 0xFE, 0x3C, ] -test_io(rnt_footer, dummy_rnt_footer) +test_io(UnROOT.Stubs.rnt_footer, dummy_rnt_footer) -tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, 866, 100, "ROOT::Experimental::RNTuple", "myntuple", "") dummy_tkey32_anchor = [ 0x00, 0x00, 0x00, 0x86, 0x00, 0x04, 0x00, 0x00, 0x00, 0x46, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x03, 0x62, 0x00, 0x00, 0x00, 0x64, 0x1B, 0x52, 0x4F, 0x4F, 0x54, 0x3A, 0x3A, 0x45, 0x78, 0x70, 0x65, 0x72, 0x69, 0x6D, 0x65, 0x6E, 0x74, 0x61, 0x6C, 0x3A, 0x3A, 0x52, 0x4E, 0x54, 0x75, 0x70, 0x6C, 0x65, 0x08, 0x6D, 0x79, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x00, ] -# these 6 bytes are between tkey32_anchor and the actual anchor -magic_6bytes = [0x40, 0x00, 0x00, 0x42, 0x00, 0x04] -rnt_anchor = UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple(0x0000, 0x0002, 0x0000, 0x0000, 0x0000000000000116, 0x00000000000000ba, 0x00000000000000ba, 0x00000000000002b6, 0x00000000000000ac, 0x00000000000000ac, 0xdc495fd01479af1b) dummy_rnt_anchor = [ 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xDC, 0x49, 0x5F, 0xD0, 0x14, 0x79, 0xAF, 0x1B, ] -test_io(rnt_anchor, dummy_rnt_anchor) +test_io(UnROOT.Stubs.rnt_anchor, dummy_rnt_anchor) -tkey32_TDirectory = UnROOT.TKey32(121, 4, 68, 0x7567176d, 53, 1, 1000, 100, "", "test_ntuple_minimal.root", "") dummy_tkey32_TDirectory = [ 0x00, 0x00, 0x00, 0x79, 0x00, 0x04, 0x00, 0x00, 0x00, 0x44, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x03, 0xE8, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, ] -test_io(tkey32_TDirectory, dummy_tkey32_TDirectory) - -# 1 key, and it is the RNTuple Anchor -n_keys = [ - 0x00, 0x00, 0x00, 0x01, -] +test_io(UnROOT.Stubs.tkey32_TDirectory, dummy_tkey32_TDirectory) - -tkey32_TStreamerInfo = UnROOT.TKey32(419, 4, 1266, 0x7567176d, 64, 1, 1121, 100, "TList", "StreamerInfo", "Doubly linked list") dummy_tkey32_TStreamerInfo = [ 0x00, 0x00, 0x01, 0xA3, 0x00, 0x04, 0x00, 0x00, 0x04, 0xF2, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x40, 0x00, 0x01, 0x00, 0x00, 0x04, 0x61, 0x00, 0x00, 0x00, 0x64, 0x05, 0x54, 0x4C, 0x69, 0x73, 0x74, 0x0C, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6D, 0x65, 0x72, 0x49, 0x6E, 0x66, 0x6F, 0x12, 0x44, 0x6F, 0x75, 0x62, 0x6C, 0x79, 0x20, 0x6C, 0x69, 0x6E, 0x6B, 0x65, 0x64, 0x20, 0x6C, 0x69, 0x73, 0x74, ] -test_io(tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) - -tsreamerinfo_compressed = [ - 0x5A, 0x4C, 0x08, 0x5A, 0x01, 0x00, 0xF2, 0x04, 0x00, 0x78, 0x01, 0xBD, 0x92, 0x4D, 0x4E, 0xC2, - 0x40, 0x1C, 0xC5, 0x1F, 0x05, 0x13, 0x91, 0x8F, 0xAD, 0x1A, 0x36, 0x6E, 0xBD, 0x42, 0x57, 0x15, - 0x83, 0x91, 0x44, 0x29, 0x42, 0xC5, 0x68, 0x82, 0x66, 0x80, 0x29, 0x94, 0x8F, 0x99, 0x66, 0xDA, - 0x26, 0xB2, 0x63, 0xE7, 0x69, 0xBC, 0x84, 0x97, 0xD0, 0x53, 0x78, 0x05, 0xFD, 0x77, 0x24, 0x04, - 0x12, 0x89, 0x68, 0x83, 0x2F, 0x99, 0x69, 0x3B, 0xED, 0xBC, 0x5F, 0xFB, 0x5E, 0x2D, 0x64, 0xDE, - 0xB1, 0x83, 0x14, 0x48, 0x46, 0x3C, 0x91, 0x52, 0x16, 0x32, 0x6F, 0x1F, 0x24, 0xA7, 0x19, 0x2A, - 0xCE, 0x26, 0x5C, 0x55, 0x85, 0x2B, 0x41, 0xAB, 0x2F, 0xC8, 0x5A, 0xC0, 0x31, 0x3D, 0xAE, 0x37, - 0xA4, 0x69, 0x2E, 0x35, 0x6C, 0xDB, 0x31, 0xCD, 0xCA, 0xA3, 0xCF, 0x95, 0x37, 0xE1, 0x22, 0x64, - 0x63, 0xD3, 0x6C, 0xD4, 0x9C, 0xC8, 0x1F, 0x73, 0xB4, 0xF3, 0xCF, 0x55, 0x32, 0xCC, 0xD0, 0xD6, - 0x27, 0x6D, 0x68, 0x77, 0x86, 0x27, 0x4A, 0xB1, 0x69, 0x6C, 0x16, 0x21, 0xBD, 0xCA, 0xCD, 0xC5, - 0x70, 0xF2, 0x8F, 0x56, 0xD8, 0x65, 0x16, 0x78, 0x5D, 0x67, 0xEA, 0xF3, 0xF8, 0xD6, 0x1D, 0x0C, - 0x9A, 0x9D, 0xD8, 0x11, 0xA5, 0xC5, 0x6B, 0x00, 0x28, 0xB8, 0x2D, 0xAE, 0x02, 0x4F, 0x8A, 0x8A, - 0x2F, 0xBB, 0x03, 0x5A, 0x40, 0x9E, 0x86, 0x11, 0x9F, 0xAC, 0x53, 0x31, 0x12, 0x81, 0xD7, 0x17, - 0xBC, 0x77, 0x14, 0x0C, 0xA4, 0x0A, 0x2D, 0xA0, 0x33, 0x03, 0x5E, 0xE9, 0xF8, 0x33, 0xE5, 0x92, - 0x0D, 0xA5, 0x8A, 0x8D, 0xB7, 0x4B, 0xF1, 0xC4, 0x3F, 0x50, 0xEA, 0x2C, 0xFC, 0x73, 0x62, 0x0F, - 0xF3, 0xC4, 0x6E, 0x60, 0x58, 0xC0, 0x95, 0xEE, 0xE5, 0x70, 0xB9, 0x97, 0x9C, 0xDB, 0xE4, 0x7C, - 0x74, 0xCE, 0x59, 0x8F, 0xEB, 0xBC, 0x8A, 0x00, 0x76, 0x69, 0xAC, 0x55, 0x61, 0xD1, 0xCA, 0x58, - 0x8A, 0xBE, 0x05, 0xB0, 0xD9, 0x57, 0x29, 0xB7, 0x30, 0xE8, 0xAA, 0xF9, 0x5D, 0xF5, 0xB5, 0xF2, - 0x34, 0xE4, 0x41, 0x12, 0xC8, 0xFD, 0x1C, 0xD2, 0xD2, 0x90, 0xBA, 0x86, 0x1C, 0x2C, 0x7F, 0xC7, - 0x9E, 0x7B, 0xC1, 0x45, 0x12, 0xC2, 0x86, 0x49, 0x9D, 0x49, 0x19, 0x6E, 0x3D, 0xA9, 0x24, 0x90, - 0xCD, 0x92, 0x4A, 0x42, 0x68, 0xCF, 0xBB, 0xB8, 0xD6, 0x5D, 0xD8, 0xBA, 0x8B, 0xFD, 0xE5, 0x2E, - 0xB2, 0xEE, 0xE9, 0x80, 0x77, 0x47, 0x41, 0x34, 0x01, 0xE9, 0x97, 0x7F, 0x14, 0x3E, 0x01, 0x15, - 0x3D, 0xC1, 0xCA, -] - -tfile_end = [ - 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x75, 0x67, 0x17, 0x6D, 0x00, 0x35, - 0x00, 0x01, 0x00, 0x00, 0x06, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, - 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, - 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x43, 0x77, 0x35, 0x94, 0x00, -] +test_io(UnROOT.Stubs.tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) MINE = [ - file_preamble; - dummy_FileHeader; dummy_padding1; + UnROOT.Stubs.file_preamble; + dummy_FileHeader; UnROOT.Stubs.dummy_padding1; dummy_tkey32_tfile; dummy_tfile; - dummy_tdirectory32; dummy_padding2; + dummy_tdirectory32; UnROOT.Stubs.dummy_padding2; dummy_RBlob1; dummy_rnt_header; - dummy_RBlob2; page1; + dummy_RBlob2; UnROOT.Stubs.page1; dummy_RBlob3; dummy_pagelink; dummy_RBlob4; dummy_rnt_footer; - dummy_tkey32_anchor; magic_6bytes; dummy_rnt_anchor; - dummy_tkey32_TDirectory; n_keys; dummy_tkey32_anchor; - dummy_tkey32_TStreamerInfo; tsreamerinfo_compressed; - tfile_end + dummy_tkey32_anchor; UnROOT.Stubs.magic_6bytes; dummy_rnt_anchor; + dummy_tkey32_TDirectory; UnROOT.Stubs.n_keys; dummy_tkey32_anchor; + dummy_tkey32_TStreamerInfo; UnROOT.Stubs.tsreamerinfo_compressed; + UnROOT.Stubs.tfile_end ] +mytable = Dict("a" => UInt32[0xcececece]) myio = IOBuffer() -UnROOT.write_rntuple(myio, []) +UnROOT.write_rntuple(myio, mytable) @test MINE == REFERENCE_BYTES -@test MINE == take!(myio) +mio = take!(myio) +@test MINE == mio + +for _ = 1:100 + newtable = Dict("a" => rand(UInt32, rand(1:1000))) + newio = IOBuffer() + UnROOT.write_rntuple(newio, newtable) + nio = take!(newio) + + if isfile("a.root") + rm("a.root") + end + + open("a.root", "w") do f + write(f, nio) + end + + t = LazyTree("a.root", "myntuple") + @test only(columntable(t)) == only(columntable(newtable)) + +end end diff --git a/test/runtests.jl b/test/runtests.jl index a8b3cf0f..d4be502c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,29 +5,29 @@ nthreads = UnROOT._maxthreadid() nthreads == 1 && @warn "Running on a single thread. Please re-run the test suite with at least two threads (`julia --threads 2 ...`)" @testset "UnROOT tests" verbose = true begin - # include("Aqua.jl") - # include("bootstrapping.jl") - # include("compressions.jl") - # include("jagged.jl") - # include("lazy.jl") - # include("histograms.jl") - # include("views.jl") - # include("multithreading.jl") - # include("remote.jl") - # include("displays.jl") - # include("type_stability.jl") - # include("utils.jl") - # include("misc.jl") + include("Aqua.jl") + include("bootstrapping.jl") + include("compressions.jl") + include("jagged.jl") + include("lazy.jl") + include("histograms.jl") + include("views.jl") + include("multithreading.jl") + include("remote.jl") + include("displays.jl") + include("type_stability.jl") + include("utils.jl") + include("misc.jl") - # include("type_support.jl") - # include("custom_bootstrapping.jl") - # include("lorentzvectors.jl") - # include("NanoAOD.jl") + include("type_support.jl") + include("custom_bootstrapping.jl") + include("lorentzvectors.jl") + include("NanoAOD.jl") - # include("issues.jl") + include("issues.jl") if VERSION >= v"1.9" - # include("rntuple.jl") + include("rntuple.jl") include("./RNTupleWriting/lowlevel.jl") end end From cb5e1bc83b14b91eff55aecc93802ce25c3db470 Mon Sep 17 00:00:00 2001 From: Jerry Ling Date: Sun, 30 Jun 2024 20:55:16 -0500 Subject: [PATCH 11/14] fix RNTuple Anchor Checksum add anchor checksum --- src/RNTuple/Writing/TFileWriter.jl | 26 +++++++++++++++----------- src/RNTuple/bootstrap.jl | 6 ++++++ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index 44f8a10c..7988f9be 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -416,17 +416,21 @@ function rnt_write(io::IO, x::UnROOT.RNTupleFooter; envelope=true) end function rnt_write(io::IO, x::UnROOT.ROOT_3a3a_Experimental_3a3a_RNTuple) - rnt_write(io, x.fVersionEpoch; legacy=true) - rnt_write(io, x.fVersionMajor; legacy=true) - rnt_write(io, x.fVersionMinor; legacy=true) - rnt_write(io, x.fVersionPatch; legacy=true) - rnt_write(io, x.fSeekHeader; legacy=true) - rnt_write(io, x.fNBytesHeader; legacy=true) - rnt_write(io, x.fLenHeader; legacy=true) - rnt_write(io, x.fSeekFooter; legacy=true) - rnt_write(io, x.fNBytesFooter; legacy=true) - rnt_write(io, x.fLenFooter; legacy=true) - rnt_write(io, x.fChecksum; legacy=true) + temp_io = IOBuffer() + rnt_write(temp_io, x.fVersionEpoch; legacy=true) + rnt_write(temp_io, x.fVersionMajor; legacy=true) + rnt_write(temp_io, x.fVersionMinor; legacy=true) + rnt_write(temp_io, x.fVersionPatch; legacy=true) + rnt_write(temp_io, x.fSeekHeader; legacy=true) + rnt_write(temp_io, x.fNBytesHeader; legacy=true) + rnt_write(temp_io, x.fLenHeader; legacy=true) + rnt_write(temp_io, x.fSeekFooter; legacy=true) + rnt_write(temp_io, x.fNBytesFooter; legacy=true) + rnt_write(temp_io, x.fLenFooter; legacy=true) + payload_ary = take!(temp_io) + checksum = xxh3_64(payload_ary) + rnt_write(io, payload_ary) + rnt_write(io, checksum; legacy=true) end mutable struct WriteObservable{O, T} diff --git a/src/RNTuple/bootstrap.jl b/src/RNTuple/bootstrap.jl index b2130e88..9ea9f61c 100644 --- a/src/RNTuple/bootstrap.jl +++ b/src/RNTuple/bootstrap.jl @@ -16,6 +16,9 @@ end function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) local_io = datastream(io, tkey) skip(local_io, 6) + _before_anchor = position(local_io) + anchor_checksum = xxh3_64(read(local_io, 2*4 + 6*8)) + seek(local_io, _before_anchor) anchor = ROOT_3a3a_Experimental_3a3a_RNTuple(; fVersionEpoch = readtype(local_io, UInt16), fVersionMajor = readtype(local_io, UInt16), @@ -30,6 +33,9 @@ function ROOT_3a3a_Experimental_3a3a_RNTuple(io, tkey::TKey, refs) fChecksum = readtype(local_io, UInt64), ) + @assert anchor.fChecksum == anchor_checksum "RNtuple anchor checksum doesn't match" + + header_bytes = decompress_bytes(read_seek_nb(io, anchor.fSeekHeader, anchor.fNBytesHeader), anchor.fLenHeader) header_io = IOBuffer(header_bytes) header = _rntuple_read(header_io, RNTupleEnvelope{RNTupleHeader}) From 202db3726193645510c49c3b547c319918017e1e Mon Sep 17 00:00:00 2001 From: Moelf Date: Sun, 30 Jun 2024 23:08:45 -0400 Subject: [PATCH 12/14] allow column name change --- Project.toml | 3 +- src/RNTuple/Writing/TFileWriter.jl | 83 ++++++++++++++---------------- test/RNTupleWriting/lowlevel.jl | 11 ++-- 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/Project.toml b/Project.toml index 543955d2..3d7c4177 100644 --- a/Project.toml +++ b/Project.toml @@ -74,10 +74,11 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LorentzVectors = "3f54b04b-17fc-5cd4-9758-90c048d965e3" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "SHA", "StaticArrays", "TOML", "Test"] +test = ["Aqua", "Arrow", "DataFrames", "InteractiveUtils", "LorentzVectors", "Pkg", "Random", "SHA", "StaticArrays", "TOML", "Test"] diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index 7988f9be..ec40d797 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -180,35 +180,7 @@ function rnt_write(io::IO, x::UnROOT.ColumnRecord) rnt_write(io, x.first_ele_idx) end end -column_record = UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0) -dummy_column_record = [ - 0x14, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -] -test_io(column_record, dummy_column_record) -# reference from reading code -# function _rntuple_read(io, ::Type{Vector{T}}) where T -# pos = position(io) -# Size = read(io, Int64) -# @assert Size < 0 -# NumItems = read(io, Int32) -# end_pos = pos - Size -# res = T[_rntuple_read(io, RNTupleFrame{T}).payload for _=1:NumItems] -# seek(io, end_pos) -# return res -# end -# -# struct RNTupleFrame{T} - # payload::T -# end -# function _rntuple_read(io, ::Type{RNTupleFrame{T}}) where T - # pos = position(io) - # Size = read(io, Int64) - # end_pos = pos + Size - # @assert Size >= 0 - # res = _rntuple_read(io, T) - # seek(io, end_pos) - # return RNTupleFrame(res) -# end + function rnt_write(io::IO, x::RNTupleFrame{T}) where T temp_io = IOBuffer() rnt_write(temp_io, x.payload) @@ -235,6 +207,29 @@ function rnt_write(io::IO, x::Write_RNTupleListFrame) write(io, temp_io) end +function _checksum(x::UnROOT.RNTupleHeader) + temp_io = IOBuffer() + rnt_write(temp_io, x.feature_flag) + rnt_write(temp_io, x.name) + rnt_write(temp_io, x.ntuple_description) + rnt_write(temp_io, x.writer_identifier) + rnt_write(temp_io, Write_RNTupleListFrame(x.field_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.column_records)) + rnt_write(temp_io, Write_RNTupleListFrame(x.alias_columns)) + rnt_write(temp_io, Write_RNTupleListFrame(x.extra_type_infos)) + + # add id_length size and checksum size + envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) + id_type = 0x0001 + + id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + + payload_ary = take!(temp_io) + prepend!(payload_ary, reinterpret(UInt8, [id_length])) + + return xxh3_64(payload_ary) +end + function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) temp_io = IOBuffer() rnt_write(temp_io, x.feature_flag) @@ -264,15 +259,6 @@ function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) end end -function rnt_write_envelope(io::IO, x; type_id) - temp_io = IOBuffer() - rnt_write(temp_io, x.payload) - size = temp_io.size + 8 - write(io, Int64(size)) - seekstart(temp_io) - write(io, temp_io) -end - function rnt_write(io::IO, x::ClusterSummary) rnt_write(io, x.first_entry_number) rnt_write(io, x.number_of_entries) @@ -436,6 +422,7 @@ end mutable struct WriteObservable{O, T} io::O position::Int64 + len::Int64 object::T end @@ -467,7 +454,8 @@ end function rnt_write_observe(io::IO, x::T) where T pos = position(io) rnt_write(io, x) - WriteObservable(io, pos, x) + len = position(io) - pos + WriteObservable(io, pos, len, x) end function split4_encode(src::AbstractVector{UInt8}) @@ -512,9 +500,13 @@ function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rn RBlob1_obs = rnt_write_observe(file, Stubs.RBlob1) rntAnchor_update[:fSeekHeader] = UInt32(position(file)) - rnt_header_obs = rnt_write_observe(file, Stubs.rnt_header) - rntAnchor_update[:fNBytesHeader] = 0xba - rntAnchor_update[:fLenHeader] = 0xba + rnt_header = UnROOT.RNTupleHeader(zero(UInt64), rntuple_name, "", "ROOT v6.33.01", [ + UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, string(only(input_schema.names)), "std::uint32_t", "", ""), + ], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), zero(UInt32), 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + + rnt_header_obs = rnt_write_observe(file, rnt_header) + rntAnchor_update[:fNBytesHeader] = rnt_header_obs.len + rntAnchor_update[:fLenHeader] = rnt_header_obs.len RBlob2_obs = rnt_write_observe(file, Stubs.RBlob2) page1 = reinterpret(UInt8, input_col) @@ -533,13 +525,14 @@ function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rn ]), ]) - pagelink = UnROOT.PageLink(0x3dec59c009c67e28, cluster_summary.payload, nested_page_locations) + # stub checksum 0x3dec59c009c67e28 + pagelink = UnROOT.PageLink(_checksum(rnt_header_obs.object), cluster_summary.payload, nested_page_locations) pagelink_position = position(file) pagelink_obs = rnt_write_observe(file, pagelink) RBlob4_obs = rnt_write_observe(file, Stubs.RBlob4) rntAnchor_update[:fSeekFooter] = UInt32(position(file)) - rnt_footer = UnROOT.RNTupleFooter(0, 0x3dec59c009c67e28, UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ + rnt_footer = UnROOT.RNTupleFooter(0, _checksum(rnt_header_obs.object), UnROOT.RNTupleSchemaExtension([], [], [], []), [], [ UnROOT.ClusterGroupRecord(0, input_length, 1, UnROOT.EnvLink(0x000000000000007c, UnROOT.Locator(124, pagelink_position, ))), ], UnROOT.EnvLink[]) rnt_footer_obs = rnt_write_observe(file, rnt_footer) @@ -547,7 +540,7 @@ function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rn rntAnchor_update[:fLenFooter] = 0xac tkey32_anchor_position = position(file) - tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, tkey32_anchor_position, 100, "ROOT::Experimental::RNTuple", "myntuple", "") + tkey32_anchor = UnROOT.TKey32(134, 4, 70, 0x7567176d, 64, 1, tkey32_anchor_position, 100, "ROOT::Experimental::RNTuple", rntuple_name, "") tkey32_anchor_obs1 = rnt_write_observe(file, tkey32_anchor) magic_6bytes_obs = rnt_write_observe(file, Stubs.magic_6bytes) rnt_anchor_obs = rnt_write_observe(file, Stubs.rnt_anchor) diff --git a/test/RNTupleWriting/lowlevel.jl b/test/RNTupleWriting/lowlevel.jl index 536568cb..d01f3f03 100644 --- a/test/RNTupleWriting/lowlevel.jl +++ b/test/RNTupleWriting/lowlevel.jl @@ -1,6 +1,7 @@ using UnROOT: rnt_write, RNTupleFrame, ClusterSummary, PageDescription, Write_RNTupleListFrame, RBlob using XXHashNative: xxh3_64 using Tables: columntable +using Random const REFERENCE_BYTES = [ 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x06, 0x43, @@ -334,15 +335,15 @@ MINE = [ UnROOT.Stubs.tfile_end ] -mytable = Dict("a" => UInt32[0xcececece]) +mytable = Dict("one_uint" => UInt32[0xcececece]) myio = IOBuffer() -UnROOT.write_rntuple(myio, mytable) +UnROOT.write_rntuple(myio, mytable; rntuple_name="myntuple") @test MINE == REFERENCE_BYTES mio = take!(myio) @test MINE == mio for _ = 1:100 - newtable = Dict("a" => rand(UInt32, rand(1:1000))) + newtable = Dict(randstring(rand(2:10)) => rand(UInt32, rand(1:1000))) newio = IOBuffer() UnROOT.write_rntuple(newio, newtable) nio = take!(newio) @@ -355,7 +356,9 @@ for _ = 1:100 write(f, nio) end - t = LazyTree("a.root", "myntuple") + rntuple_name = "myntuple" + t = LazyTree("a.root", rntuple_name) + @test sort(names(t)) == sort(collect(keys(newtable))) @test only(columntable(t)) == only(columntable(newtable)) end From 335e9f1356967bc39b46789767e7e69a101f7b4e Mon Sep 17 00:00:00 2001 From: Moelf Date: Sun, 30 Jun 2024 23:13:27 -0400 Subject: [PATCH 13/14] add Random compat --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 3d7c4177..af5d92b5 100644 --- a/Project.toml +++ b/Project.toml @@ -56,6 +56,7 @@ Parameters = "^0.12" Pkg = "^1.0" PrecompileTools = "^1.2.0" PrettyTables = "^2.1" +Random = "^1.0" SHA = "^0.7, ^1.0" SentinelArrays = "^1.3" StaticArrays = "^1" From 23e3b939c8ffd9bdc493726f33f81b39c9e6523b Mon Sep 17 00:00:00 2001 From: Moelf Date: Fri, 13 Sep 2024 15:45:48 +0200 Subject: [PATCH 14/14] fix upload artifact --- .github/workflows/draft-pdf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml index 76310246..a349ebaa 100644 --- a/.github/workflows/draft-pdf.yml +++ b/.github/workflows/draft-pdf.yml @@ -14,7 +14,7 @@ jobs: # This should be the path to the paper within your repo. paper-path: paper/paper.md - name: Upload - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: paper # This is the output path where Pandoc will write the compiled