diff --git a/.gitattributes b/.gitattributes index b5a6eb1b..3bf1119b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,3 +7,5 @@ test/samples/RNTuple/* linguist-detectable=false # JOSS paper paper/** linguist-detectable=false +# GitHub syntax highlighting +pixi.lock linguist-language=YAML linguist-generated=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f0b5138..0ea0ddf6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,7 @@ jobs: fail-fast: false matrix: version: - - '1.7' - - '1.8' + - 'lts' - '1' # Leave this line unchanged. '1' will automatically expand to the latest stable 1.x release of Julia. - 'pre' os: [ubuntu-latest] diff --git a/.github/workflows/cvmfs.yml b/.github/workflows/cvmfs.yml new file mode 100644 index 00000000..bd519dec --- /dev/null +++ b/.github/workflows/cvmfs.yml @@ -0,0 +1,36 @@ +name: Test C++ ROOT read back +on: + pull_request: + branches: + - main + push: + branches: + - main + tags: '*' +jobs: + test: + name: C++ ROOT read back rntuple files ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + continue-on-error: ${{ matrix.allow_failure }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-22.04] + arch: [x64] + allow_failure: [false] + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: ${{ matrix.arch }} + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - name: Generate root file + run: | + julia --project ./test/RNTupleWriting/output_sample.jl test1.root + - uses: cvmfs-contrib/github-action-cvmfs@v4 + - name: Read root file in C++ + run: | + source /cvmfs/sft.cern.ch/lcg/views/dev3/latest/x86_64-ubuntu2204-gcc11-opt/setup.sh + python ./test/RNTupleWriting/test1.py test1.root diff --git a/.gitignore b/.gitignore index 4f5484e8..94ee381e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ *__pycache__* /.benchmarkci /benchmark/*.json +# pixi environments +.pixi +*.egg-info diff --git a/Project.toml b/Project.toml index 37cb96a1..24ffc652 100644 --- a/Project.toml +++ b/Project.toml @@ -43,7 +43,7 @@ CodecXz = "^0.7" CodecZstd = "^0.8" DataFrames = "^1.5" FHist = "^0.10, ^0.11" -HTTP = "^1" +HTTP = "^1.10" InteractiveUtils = "^1.0" IterTools = "^1" LRUCache = "^1.3.0" @@ -59,14 +59,14 @@ PrettyTables = "^2.1" Random = "^1.0" SHA = "^0.7, ^1.0" SentinelArrays = "^1.3" -StaticArrays = "^1" +StaticArrays = "^1.5" StructArrays = "0.6" TOML = "^1.0" Tables = "^1.9" Test = "^1.0" XRootD = "^0.1" XXHashNative = "^1.0.1" -julia = "^1.7" +julia = "^1.10" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" diff --git a/src/RNTuple/Writing/Stubs.jl b/src/RNTuple/Writing/Stubs.jl index 86f204c2..e55d9450 100644 --- a/src/RNTuple/Writing/Stubs.jl +++ b/src/RNTuple/Writing/Stubs.jl @@ -1,7 +1,8 @@ module Stubs using ..UnROOT -const WRITE_TIME = 0x7670F8CD +const WRITE_TIME = 0x768A676E +const WRITE_TIME_ary = reverse(reinterpret(UInt8, [WRITE_TIME])) const file_preamble = [ 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0xF7, 0x45, @@ -23,7 +24,8 @@ const dummy_padding2 = [ ] -const RBlob1 = UnROOT.RBlob(0x00DC, 0x0004, 0x000000BA, WRITE_TIME, 0x0022, 0x0001, 244, 100, "RBlob", "", "") +const RBlob1 = UnROOT.RBlob(; fNbytes = 0x00DC, fVersion = 0x0004, fObjLen = 0x000000BA, fDatime = WRITE_TIME, fKeyLen = 0x0022, +fCycle = 0x0001, fSeekKey = 244, fSeekPdir = 100, fClassName = "RBlob", fName = "", fTitle = "") const rnt_header = UnROOT.RNTupleHeader(zero(UInt64), "myntuple", "", "ROOT v6.33.01", [ UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, -1, -1, "one_uint", "std::uint32_t", "", ""), ], [UnROOT.ColumnRecord(0x14, 0x20, zero(UInt32), 0x00, 0x00, 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) @@ -90,7 +92,7 @@ const tsreamerinfo_compressed = [ ] const tfile_end = [ - 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, 0x76, 0x70, 0xF8, 0xCD, 0x00, 0x35, + 0x00, 0x00, 0x00, 0x3F, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0A, WRITE_TIME_ary..., 0x00, 0x35, 0x00, 0x01, 0x00, 0x00, 0x05, 0xF9, 0x00, 0x00, 0x00, 0x64, 0x00, 0x18, 0x74, 0x65, 0x73, 0x74, 0x5F, 0x6E, 0x74, 0x75, 0x70, 0x6C, 0x65, 0x5F, 0x6D, 0x69, 0x6E, 0x69, 0x6D, 0x61, 0x6C, 0x2E, 0x72, 0x6F, 0x6F, 0x74, 0x00, 0x00, 0x01, 0x00, 0x00, 0x06, 0x38, 0x77, 0x35, 0x94, 0x00, diff --git a/src/RNTuple/Writing/TFileWriter.jl b/src/RNTuple/Writing/TFileWriter.jl index fdb01854..91eb1451 100644 --- a/src/RNTuple/Writing/TFileWriter.jl +++ b/src/RNTuple/Writing/TFileWriter.jl @@ -134,7 +134,7 @@ function rnt_write(io::IO, x::UnROOT.ROOTDirectoryHeader32) rnt_write(io, x.fSeekKeys; legacy=true) end -struct RBlob +Base.@kwdef struct RBlob fNbytes::Int32 fVersion::Int16 fObjLen::Int32 @@ -237,7 +237,7 @@ function _checksum(x::UnROOT.RNTupleHeader) envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) id_type = 0x0001 - id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + id_length = (UInt64(envelope_size) << 16) | id_type payload_ary = take!(temp_io) prepend!(payload_ary, reinterpret(UInt8, [id_length])) @@ -260,7 +260,7 @@ function rnt_write(io::IO, x::UnROOT.RNTupleHeader; envelope=true) envelope_size = temp_io.size + sizeof(Int64) + sizeof(UInt64) id_type = 0x0001 - id_length = (UInt64(envelope_size & 0xff) << 16) | id_type + id_length = (UInt64(envelope_size) << 16) | id_type payload_ary = take!(temp_io) @@ -473,19 +473,45 @@ function rnt_write_observe(io::IO, x::T) where T WriteObservable(io, pos, len, x) end +function add_field_column_record!(field_records, column_records, input_T::Type{<:Real}, NAME; parent_field_id) + fr = UnROOT.FieldRecord(zero(UInt32), zero(UInt32), parent_field_id, zero(UInt16), zero(UInt16), 0, -1, -1, string(NAME), RNTUPLE_WRITE_TYPE_CPPNAME_DICT[input_T], "", "") + cr = UnROOT.ColumnRecord(RNTUPLE_WRITE_TYPE_IDX_DICT[input_T]..., parent_field_id, 0x00, 0x00, 0) + push!(field_records, fr) + push!(column_records, cr) + nothing +end + +function schema_to_field_column_records(table) + input_schema = schema(table) + input_Ts = input_schema.types + input_names = input_schema.names + field_records = UnROOT.FieldRecord[] + column_records = UnROOT.ColumnRecord[] + + for (input_T, input_name) in zip(input_Ts, input_names) + add_field_column_record!(field_records, column_records, input_T, input_name, parent_field_id=length(field_records)) + end + return field_records, column_records +end + +function generate_page_links(column_records, pages_obses, Nitems) + outer_list = RNTuplePageOuterList{RNTuplePageInnerList{PageDescription}}([]) + for (cr, page_obs) in zip(column_records, pages_obses) + inner_list = RNTuplePageInnerList([ + PageDescription(Nitems, Locator(div(cr.nbits * Nitems, 8, RoundUp), page_obs.position)) + ]) + push!(outer_list, inner_list) + end + return RNTuplePageTopList([outer_list]) +end + function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rntuple_name="myntuple") if !istable(table) error("RNTuple writing accepts object compatible with Tables.jl interface, got type $(typeof(table))") end - input_schema = schema(table) - input_Ncols = length(input_schema.names) - if input_Ncols != 1 - error("Currently, RNTuple writing only supports a single, UInt32 column, got $input_Ncols columns") - end - input_T = only(input_schema.types) - input_col = only(columntable(table)) - input_length = length(input_col) + input_cols = columntable(table) + input_length = length(input_cols[begin]) if input_length > 65535 error("Input too long: RNTuple writing currently only supports a single page (65535 elements)") end @@ -505,30 +531,33 @@ function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rn tdirectory32_obs = rnt_write_observe(file, Stubs.tdirectory32) dummy_padding2_obs = rnt_write_observe(file, Stubs.dummy_padding2) - RBlob1_obs = rnt_write_observe(file, Stubs.RBlob1) - rntAnchor_update[:fSeekHeader] = UInt32(position(file)) - rnt_header = UnROOT.RNTupleHeader(zero(UInt64), rntuple_name, "", "ROOT v6.33.01", [ - UnROOT.FieldRecord(zero(UInt32), zero(UInt32), zero(UInt32), zero(UInt16), zero(UInt16), 0, -1, -1, string(only(input_schema.names)), RNTUPLE_WRITE_TYPE_CPPNAME_DICT[input_T], "", ""), - ], [UnROOT.ColumnRecord(RNTUPLE_WRITE_TYPE_IDX_DICT[input_T]..., zero(UInt32), 0x00, 0x00, 0),], UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[]) + RBlob1 = UnROOT.RBlob(; fNbytes = 0x00DC, fVersion = 0x0004, fObjLen = 0x000000BA, fDatime = Stubs.WRITE_TIME, fKeyLen = 34, + fCycle = 0x0001, fSeekKey = position(file), fSeekPdir = 100, fClassName = "RBlob", fName = "", fTitle = "") + RBlob1_update = Dict{Symbol, Any}() + RBlob1_obs = rnt_write_observe(file, RBlob1) + field_records, col_records = schema_to_field_column_records(table) + rnt_header = UnROOT.RNTupleHeader( + zero(UInt64), rntuple_name, "", "ROOT v6.33.01", + field_records, col_records, + UnROOT.AliasRecord[], UnROOT.ExtraTypeInfo[] + ) + rntAnchor_update[:fSeekHeader] = UInt32(position(file)) rnt_header_obs = rnt_write_observe(file, rnt_header) rntAnchor_update[:fNBytesHeader] = rnt_header_obs.len rntAnchor_update[:fLenHeader] = rnt_header_obs.len + RBlob1_update[:fObjLen] = rnt_header_obs.len + RBlob1_update[:fNbytes] = rnt_header_obs.len + 34 + + Base.setindex!(RBlob1_obs, RBlob1_update) RBlob2_obs = rnt_write_observe(file, Stubs.RBlob2) - page1 = rnt_ary_to_page(input_col) - page1_obs = rnt_write_observe(file, page1) + pages = [rnt_ary_to_page(col, cr) for (col, cr) in zip(input_cols, col_records)] + pages_obses = [rnt_write_observe(file, page) for page in pages] RBlob3_obs = rnt_write_observe(file, Stubs.RBlob3) cluster_summary = Write_RNTupleListFrame([ClusterSummary(0, input_length)]) - nested_page_locations = - UnROOT.RNTuplePageTopList([ - UnROOT.RNTuplePageOuterList([ - UnROOT.RNTuplePageInnerList([ - PageDescription(input_length, UnROOT.Locator(sizeof(input_T) * input_length, page1_obs.position, )), - ]), - ]), - ]) + nested_page_locations = generate_page_links(col_records, pages_obses, input_length) pagelink = UnROOT.PageLink(_checksum(rnt_header_obs.object), cluster_summary.payload, nested_page_locations) pagelink_obs = rnt_write_observe(file, pagelink) @@ -565,6 +594,7 @@ function write_rntuple(file::IO, table; file_name="test_ntuple_minimal.root", rn tfile_end_obs = rnt_write_observe(file, Stubs.tfile_end) fileheader_obs[:fEND] = UInt32(position(file)) + flush!(RBlob1_obs) flush!(tkey32_anchor_obs1) flush!(tkey32_anchor_obs2) flush!(tkey32_tfile_obs) diff --git a/src/RNTuple/Writing/constants.jl b/src/RNTuple/Writing/constants.jl index b70a2b2c..953f9afc 100644 --- a/src/RNTuple/Writing/constants.jl +++ b/src/RNTuple/Writing/constants.jl @@ -2,9 +2,9 @@ const RNTUPLE_WRITE_TYPE_IDX_DICT = Dict( Float64 => (0x10, sizeof(UInt64) * 8), Float32 => (0x11, sizeof(UInt32) * 8), Float16 => (0x12, sizeof(UInt16) * 8), - UInt64 => (0x13, sizeof(UInt64) * 8), - UInt32 => (0x14, sizeof(UInt32) * 8), - UInt16 => (0x15, sizeof(UInt16) * 8), + UInt64 => (0x0A, sizeof(UInt64) * 8), + UInt32 => (0x0B, sizeof(UInt32) * 8), + UInt16 => (0x0C, sizeof(UInt16) * 8), Int64 => (0x16, sizeof(Int64) * 8), Int32 => (0x17, sizeof(Int32) * 8), Int16 => (0x18, sizeof(Int16) * 8), @@ -13,8 +13,8 @@ const RNTUPLE_WRITE_TYPE_IDX_DICT = Dict( const RNTUPLE_WRITE_TYPE_CPPNAME_DICT = Dict( Float16 => "std::float16_t", - Float32 => "std::float32_t", - Float64 => "std::float64_t", + Float32 => "float", + Float64 => "double", Int8 => "std::int8_t", Int16 => "std::int16_t", Int32 => "std::int32_t", diff --git a/src/RNTuple/Writing/page_writing.jl b/src/RNTuple/Writing/page_writing.jl index 026a11e7..3075badb 100644 --- a/src/RNTuple/Writing/page_writing.jl +++ b/src/RNTuple/Writing/page_writing.jl @@ -1,48 +1,82 @@ """ - rnt_ary_to_page(ary::AbstractVector) end + rnt_ary_to_page(ary::AbstractVector, cr::ColumnRecord) end Turns an AbstractVector into a page of an RNTuple. The element type must be primitive for this to work. """ -function rnt_ary_to_page(ary::AbstractVector) end +function rnt_ary_to_page(ary::AbstractVector, cr::ColumnRecord) end -function rnt_ary_to_page(ary::AbstractVector{Float64}) - Page_write(split8_encode(reinterpret(UInt8, ary))) +function rnt_ary_to_page(ary::AbstractVector{Float64}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) + if split + Page_write(split8_encode(reinterpret(UInt8, ary))) + else + Page_write(reinterpret(UInt8, ary)) + end end -function rnt_ary_to_page(ary::AbstractVector{Float32}) - Page_write(split4_encode(reinterpret(UInt8, ary))) +function rnt_ary_to_page(ary::AbstractVector{Float32}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) + if split + Page_write(split4_encode(reinterpret(UInt8, ary))) + else + Page_write(reinterpret(UInt8, ary)) + end end -function rnt_ary_to_page(ary::AbstractVector{Float16}) - Page_write(split2_encode(reinterpret(UInt8, ary))) +function rnt_ary_to_page(ary::AbstractVector{Float16}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) + if split + Page_write(split2_encode(reinterpret(UInt8, ary))) + else + Page_write(reinterpret(UInt8, ary)) + end end -function rnt_ary_to_page(ary::AbstractVector{UInt64}) - Page_write(split8_encode(reinterpret(UInt8, ary))) +function rnt_ary_to_page(ary::AbstractVector{UInt64}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) + if split + Page_write(split8_encode(reinterpret(UInt8, ary))) + else + Page_write(reinterpret(UInt8, ary)) + end end -function rnt_ary_to_page(ary::AbstractVector{UInt32}) - Page_write(split4_encode(reinterpret(UInt8, ary))) +function rnt_ary_to_page(ary::AbstractVector{UInt32}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) + if split + Page_write(split4_encode(reinterpret(UInt8, ary))) + else + Page_write(reinterpret(UInt8, ary)) + end end -function rnt_ary_to_page(ary::AbstractVector{UInt16}) - Page_write(split2_encode(reinterpret(UInt8, ary))) +function rnt_ary_to_page(ary::AbstractVector{UInt16}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) + if split + Page_write(split2_encode(reinterpret(UInt8, ary))) + else + Page_write(reinterpret(UInt8, ary)) + end end -function rnt_ary_to_page(ary::AbstractVector{Int64}) +function rnt_ary_to_page(ary::AbstractVector{Int64}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) Page_write(reinterpret(UInt8, ary)) end -function rnt_ary_to_page(ary::AbstractVector{Int32}) +function rnt_ary_to_page(ary::AbstractVector{Int32}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) Page_write(reinterpret(UInt8, ary)) end -function rnt_ary_to_page(ary::AbstractVector{Int16}) +function rnt_ary_to_page(ary::AbstractVector{Int16}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) Page_write(reinterpret(UInt8, ary)) end -function rnt_ary_to_page(ary::AbstractVector{Int8}) +function rnt_ary_to_page(ary::AbstractVector{Int8}, cr::ColumnRecord) + (;split, zigzag, delta) = _detect_encoding(cr.type) Page_write(reinterpret(UInt8, ary)) end diff --git a/src/RNTuple/footer.jl b/src/RNTuple/footer.jl index 1af17fce..deb0e5c6 100644 --- a/src/RNTuple/footer.jl +++ b/src/RNTuple/footer.jl @@ -120,6 +120,8 @@ for x in (:RNTuplePageTopList, :RNTuplePageOuterList, :RNTuplePageInnerList) Base.size(r::$x) = size(r.payload) Base.getindex(r::$x, i) = r.payload[i] Base.setindex!(r::$x, v, i) = (r.payload[i] = v) + Base.push!(r::$x, v) = push!(r.payload, v) + Base.append!(r::$x, v) = append!(r.payload, v) end end diff --git a/src/RNTuple/header.jl b/src/RNTuple/header.jl index dc8653d3..429a462d 100644 --- a/src/RNTuple/header.jl +++ b/src/RNTuple/header.jl @@ -1,4 +1,4 @@ -struct FieldRecord +Base.@kwdef struct FieldRecord field_version::UInt32 type_version::UInt32 parent_field_id::UInt32 diff --git a/test/RNTupleWriting/lowlevel.jl b/test/RNTupleWriting/lowlevel.jl index 14ccc7dc..2667d213 100644 --- a/test/RNTupleWriting/lowlevel.jl +++ b/test/RNTupleWriting/lowlevel.jl @@ -1,10 +1,9 @@ -using UnROOT: rnt_write, RNTupleFrame, ClusterSummary, PageDescription, Write_RNTupleListFrame, RBlob +using UnROOT: rnt_write, RNTupleFrame, ClusterSummary, PageDescription, Write_RNTupleListFrame, RBlob, Stubs using XXHashNative: xxh3_64 using Tables: columntable using Random -const WRITE_TIME = 0x7670F8CD -const WRITE_TIME_ary = reverse(reinterpret(UInt8, [WRITE_TIME])) +const WRITE_TIME_ary = Stubs.WRITE_TIME_ary const REFERENCE_BYTES = read(joinpath(@__DIR__, "../samples/RNTuple/test_ntuple_minimal.root")) @@ -105,9 +104,7 @@ test_io(UnROOT.Stubs.rnt_header, dummy_rnt_header_payload; envelope=false) # ==================================== side tests end ==================================== -dummy_rnt_header = [ - 0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D, -] +dummy_rnt_header = [0x01, 0x00, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_rnt_header_payload..., 0x28, 0x7E, 0xC6, 0x09, 0xC0, 0x59, 0xEC, 0x3D] test_io(UnROOT.Stubs.rnt_header, dummy_rnt_header; envelope=true) dummy_RBlob2 = [ @@ -155,9 +152,7 @@ test_io(UnROOT.Stubs.pagelink, dummy_pagelink_noenvelope; envelope=false) # ================= side tests end ================= -dummy_pagelink = [ - 0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_pagelink_noenvelope..., -] +dummy_pagelink = [0x03, 0x00, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, dummy_pagelink_noenvelope...] dummy_pagelink = [dummy_pagelink; reinterpret(UInt8, [xxh3_64(dummy_pagelink)])] test_io(UnROOT.Stubs.pagelink, dummy_pagelink) @@ -216,34 +211,18 @@ dummy_tkey32_TStreamerInfo = [ test_io(UnROOT.Stubs.tkey32_TStreamerInfo, dummy_tkey32_TStreamerInfo) -MINE = [ - UnROOT.Stubs.file_preamble; - dummy_FileHeader; UnROOT.Stubs.dummy_padding1; - dummy_tkey32_tfile; dummy_tfile; - dummy_tdirectory32; UnROOT.Stubs.dummy_padding2; - dummy_RBlob1; dummy_rnt_header; - dummy_RBlob2; UnROOT.Stubs.page1; reinterpret(UInt8, [xxh3_64(UnROOT.Stubs.page1)]); - dummy_RBlob3; dummy_pagelink; - dummy_RBlob4; dummy_rnt_footer; - dummy_tkey32_anchor; UnROOT.Stubs.magic_6bytes; dummy_rnt_anchor; - dummy_tkey32_TDirectory; UnROOT.Stubs.n_keys; dummy_tkey32_anchor; - dummy_tkey32_TStreamerInfo; UnROOT.Stubs.tsreamerinfo_compressed; - UnROOT.Stubs.tfile_end -] - mytable = Dict("one_uint" => UInt32[0xcececece]) myio = IOBuffer() UnROOT.write_rntuple(myio, mytable; rntuple_name="myntuple") -@test MINE == REFERENCE_BYTES mio = take!(myio) write("/tmp/mine.root", mio) -@test MINE == mio +@test mio == REFERENCE_BYTES end @testset "RNTuple Writing - Single colunm round trips" begin -for _ = 1:50, T in [Float64, Float32, Float16, Int64, Int32, Int16, Int8, UInt64, UInt32, UInt16] - newtable = Dict(randstring(rand(2:10)) => rand(T, rand(1:1000))) +for _ = 1:10, T in [Float64, Float32, Float16, Int64, Int32, Int16, Int8, UInt64, UInt32, UInt16] + newtable = Dict(randstring(rand(2:10)) => rand(T, rand(1:100))) newio = IOBuffer() UnROOT.write_rntuple(newio, newtable) nio = take!(newio) @@ -263,3 +242,27 @@ for _ = 1:50, T in [Float64, Float32, Float16, Int64, Int32, Int16, Int8, UInt64 end end + +@testset "RNTuple Writing - Multiple colunm round trips" begin + Ts = rand([Float64, Float32, Float16, Int64, Int32, Int16, Int8, UInt64, UInt32, UInt16], 15) + Nitems = rand(10:1000) + newtable = Dict(randstring(rand(2:10)) => rand(T, Nitems) for T in Ts) + newio = IOBuffer() + UnROOT.write_rntuple(newio, newtable) + nio = take!(newio) + + if isfile("a.root") + rm("a.root") + end + + open("a.root", "w") do f + write(f, nio) + end + + rntuple_name = "myntuple" + t = LazyTree("a.root", rntuple_name) + @test sort(names(t)) == sort(collect(keys(newtable))) + for i in propertynames(t) + @test all(getproperty(t, i) .== newtable[String(i)]) + end +end diff --git a/test/RNTupleWriting/output_sample.jl b/test/RNTupleWriting/output_sample.jl new file mode 100644 index 00000000..094be4bf --- /dev/null +++ b/test/RNTupleWriting/output_sample.jl @@ -0,0 +1,11 @@ +using UnROOT + +Nitems = 10 +data = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14] +newtable = Dict( + "x1" => Float64.(data), + "x2" => Float32.(data), + "x3" => Int32.(data), + "y1" => UInt16.(data), +) +UnROOT.write_rntuple(open(only(ARGS), "w"), newtable; rntuple_name="myntuple") diff --git a/test/RNTupleWriting/test1.py b/test/RNTupleWriting/test1.py new file mode 100644 index 00000000..400785ae --- /dev/null +++ b/test/RNTupleWriting/test1.py @@ -0,0 +1,9 @@ +import ROOT +import sys + +df = ROOT.RDataFrame("myntuple", sys.argv[1]) +if len(list(df.GetColumnNames())) != 4: + sys.exit(1) + +if list(df.Take['std::int32_t']("x3")) != [5, 6, 7, 8, 9, 10, 11, 12, 13, 14]: + sys.exit(1) diff --git a/test/samples/RNTuple/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root b/test/samples/RNTuple/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root index d2244657..fe901939 100644 Binary files a/test/samples/RNTuple/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root and b/test/samples/RNTuple/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root differ diff --git a/test/samples/RNTuple/rntuple_minimal_2col.C b/test/samples/RNTuple/rntuple_minimal_2col.C new file mode 100644 index 00000000..cd0cbafc --- /dev/null +++ b/test/samples/RNTuple/rntuple_minimal_2col.C @@ -0,0 +1,26 @@ +R__LOAD_LIBRARY(ROOTNTuple) +#include +#include +#include +#include + +using RNTupleModel = ROOT::Experimental::RNTupleModel; +using RNTupleWriter = ROOT::Experimental::RNTupleWriter; +using RNTupleWriteOptions = ROOT::Experimental::RNTupleWriteOptions; + +void rntuple_minimal_2col() { + auto writeOptions = RNTupleWriteOptions(); + writeOptions.SetCompression(0); + + std::string rootFileName1{"test_ntuple_minimal_2col.root"}; + auto model1 = RNTupleModel::Create(); + auto field1 = model1->MakeField("one_uint"); + auto field2 = model1->MakeField("two_uint"); + auto ntuple1 = + RNTupleWriter::Recreate(std::move(model1), "myntuple", rootFileName1, writeOptions); + // 0xcececece + *field1 = 3469659854; + // 0xabababab + *field2 = 2880154539; + ntuple1->Fill(); +} diff --git a/test/samples/RNTuple/rntuple_nanoAOD_importer.C b/test/samples/RNTuple/rntuple_nanoAOD_importer.C new file mode 100644 index 00000000..78eb2bbd --- /dev/null +++ b/test/samples/RNTuple/rntuple_nanoAOD_importer.C @@ -0,0 +1,12 @@ +R__LOAD_LIBRARY(ROOTNTuple) +#include +#include +#include +#include + +void rntuple_nanoAOD_importer() { + auto importer = ROOT::Experimental::RNTupleImporter::Create("./Run2012BC_DoubleMuParked_Muons.root", "Events", "./Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root"); + auto c = importer.get(); + c->SetMaxEntries(1000); + c->Import(); +} diff --git a/test/samples/RNTuple/runall.sh b/test/samples/RNTuple/runall.sh index 9d582dfd..0f082b44 100644 --- a/test/samples/RNTuple/runall.sh +++ b/test/samples/RNTuple/runall.sh @@ -4,3 +4,5 @@ root -q ./rntuple_stl_containers.C root -q ./rntuple_int_multicluster.C root -q ./rntuple_split_3e4.C root -q ./rntuple_minimal.C +rm ./Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root +root -q ./rntuple_nanoAOD_importer.C diff --git a/test/samples/RNTuple/test_ntuple_bit.root b/test/samples/RNTuple/test_ntuple_bit.root index 64c7de26..8fddbeb4 100644 Binary files a/test/samples/RNTuple/test_ntuple_bit.root and b/test/samples/RNTuple/test_ntuple_bit.root differ diff --git a/test/samples/RNTuple/test_ntuple_int_5e4.root b/test/samples/RNTuple/test_ntuple_int_5e4.root index 10f66873..06d97f23 100644 Binary files a/test/samples/RNTuple/test_ntuple_int_5e4.root and b/test/samples/RNTuple/test_ntuple_int_5e4.root differ diff --git a/test/samples/RNTuple/test_ntuple_int_multicluster.root b/test/samples/RNTuple/test_ntuple_int_multicluster.root index 4f0b2aef..88480da3 100644 Binary files a/test/samples/RNTuple/test_ntuple_int_multicluster.root and b/test/samples/RNTuple/test_ntuple_int_multicluster.root differ diff --git a/test/samples/RNTuple/test_ntuple_minimal.root b/test/samples/RNTuple/test_ntuple_minimal.root index fbbbc0ca..7d91b280 100644 Binary files a/test/samples/RNTuple/test_ntuple_minimal.root and b/test/samples/RNTuple/test_ntuple_minimal.root differ diff --git a/test/samples/RNTuple/test_ntuple_minimal_2col.root b/test/samples/RNTuple/test_ntuple_minimal_2col.root new file mode 100644 index 00000000..50f2836d Binary files /dev/null and b/test/samples/RNTuple/test_ntuple_minimal_2col.root differ diff --git a/test/samples/RNTuple/test_ntuple_split_3e4.root b/test/samples/RNTuple/test_ntuple_split_3e4.root index 712436f1..2c6205d4 100644 Binary files a/test/samples/RNTuple/test_ntuple_split_3e4.root and b/test/samples/RNTuple/test_ntuple_split_3e4.root differ diff --git a/test/samples/RNTuple/test_ntuple_stl_containers.root b/test/samples/RNTuple/test_ntuple_stl_containers.root index a4e1f6f3..6f7014ed 100644 Binary files a/test/samples/RNTuple/test_ntuple_stl_containers.root and b/test/samples/RNTuple/test_ntuple_stl_containers.root differ