Support for PODIO version < 0.17 (#4)

* Added read podio version

* Fixes for legacy podio file structure

* Fixed issues with collectionID == -2

* Protection for version < 1.10

examples/FCC/read_fcc_file.jl

@@ -0,0 +1,17 @@
+using Revise
+using EDM4hep
+using EDM4hep.RootIO
+
+cd(@__DIR__)
+
+f = "root://eospublic.cern.ch//eos/experiment/fcc/ee/generation/DelphesEvents/winter2023/IDEA/p8_ee_ZZ_ecm240/events_000189367.root"
+#f = "/Users/mato/cernbox/Data/events_000189367.root"
+#f = "/Users/mato/cernbox/Data/events_078174375.root"
+#f = "../Output_REC_rntuple.root"
+
+reader = RootIO.Reader(f)
+events = RootIO.get(reader, "events")
+
+evt = events[1];
+
+mcps = RootIO.get(reader, evt, "Particle"; register=false);

podio/genDatatypes.jl

@@ -885,18 +885,18 @@ struct ReconstructedParticle <: POD
     mass::Float32                    # [GeV] mass of the reconstructed particle, set independently from four vector. Four momentum state is not kept consistent internally.
     goodnessOfPID::Float32           # overall goodness of the PID on a scale of [0;1]
     covMatrix::SVector{10,Float32}   # cvariance matrix of the reconstructed particle 4vector (10 parameters). Stored as lower triangle matrix of the four momentum (px,py,pz,E), i.e. cov(px,px), cov(py,##
-    #---OneToOneRelations
-    startVertex_idx::ObjectID{Vertex}  # start vertex associated to this particle
-    particleIDUsed_idx::ObjectID{ParticleID}  # particle Id used for the kinematics of this particle
     #---OneToManyRelations
     clusters::Relation{ReconstructedParticle,Cluster,1}  # clusters that have been used for this particle.
     tracks::Relation{ReconstructedParticle,Track,2}  # tracks that have been used for this particle.
     particles::Relation{ReconstructedParticle,ReconstructedParticle,3}  # reconstructed particles that have been combined to this particle.
     particleIDs::Relation{ReconstructedParticle,ParticleID,4}  # particle Ids (not sorted by their likelihood)
+    #---OneToOneRelations
+    startVertex_idx::ObjectID{Vertex}  # start vertex associated to this particle
+    particleIDUsed_idx::ObjectID{ParticleID}  # particle Id used for the kinematics of this particle
 end
 
-function ReconstructedParticle(;type=0, energy=0, momentum=Vector3f(), referencePoint=Vector3f(), charge=0, mass=0, goodnessOfPID=0, covMatrix=zero(SVector{10,Float32}), startVertex=-1, particleIDUsed=-1, clusters=Relation{ReconstructedParticle,Cluster,1}(), tracks=Relation{ReconstructedParticle,Track,2}(), particles=Relation{ReconstructedParticle,ReconstructedParticle,3}(), particleIDs=Relation{ReconstructedParticle,ParticleID,4}())
-    ReconstructedParticle(-1, type, energy, momentum, referencePoint, charge, mass, goodnessOfPID, covMatrix, startVertex, particleIDUsed, clusters, tracks, particles, particleIDs)
+function ReconstructedParticle(;type=0, energy=0, momentum=Vector3f(), referencePoint=Vector3f(), charge=0, mass=0, goodnessOfPID=0, covMatrix=zero(SVector{10,Float32}), clusters=Relation{ReconstructedParticle,Cluster,1}(), tracks=Relation{ReconstructedParticle,Track,2}(), particles=Relation{ReconstructedParticle,ReconstructedParticle,3}(), particleIDs=Relation{ReconstructedParticle,ParticleID,4}(), startVertex=-1, particleIDUsed=-1)
+    ReconstructedParticle(-1, type, energy, momentum, referencePoint, charge, mass, goodnessOfPID, covMatrix, clusters, tracks, particles, particleIDs, startVertex, particleIDUsed)
 end
 
 function Base.getproperty(obj::ReconstructedParticle, sym::Symbol)

podio/generate.jl

@@ -94,18 +94,6 @@ function gen_datatype(io, key, dtype)
             push!(vectormembers, (varname=v,totype=t))
         end
     end
-    relations1to1 = @NamedTuple{varname::String, totype::String}[]
-    if haskey(dtype, "OneToOneRelations")
-        println(io, "    #---OneToOneRelations")
-        for r in dtype["OneToOneRelations"]
-            t, v, c = split_member(r)
-            vt = gen_member(v*"_idx", "ObjectID{$(t)}")
-            println(io, "    $(vt) $(c)")
-            push!(members, v)
-            push!(defvalues, "-1")
-            push!(relations1to1, (varname=v, totype=t))
-        end
-    end
     relations1toN = @NamedTuple{varname::String, totype::String}[]
     if haskey(dtype, "OneToManyRelations")
         println(io, "    #---OneToManyRelations")
@@ -118,6 +106,18 @@ function gen_datatype(io, key, dtype)
             push!(relations1toN, (varname=v, totype=t))
         end
     end
+    relations1to1 = @NamedTuple{varname::String, totype::String}[]
+    if haskey(dtype, "OneToOneRelations")
+        println(io, "    #---OneToOneRelations")
+        for r in dtype["OneToOneRelations"]
+            t, v, c = split_member(r)
+            vt = gen_member(v*"_idx", "ObjectID{$(t)}")
+            println(io, "    $(vt) $(c)")
+            push!(members, v)
+            push!(defvalues, "-1")
+            push!(relations1to1, (varname=v, totype=t))
+        end
+    end
     println(io, "end\n")
 
     # add an extra constructor with keyword parameters

src/Components.jl

@@ -70,18 +70,25 @@ end
 #--------------------------------------------------------------------------------------------------
 struct ObjectID{ED <: POD} <: POD
     index::Int32
-    collectionID::UInt32
+    collectionID::UInt32    # in some cases (reading from files) the collection ID is -2
 end
-
 Base.zero(::Type{ObjectID{ED}}) where ED = ObjectID{ED}(-1,0)
-Base.iszero(x::ObjectID{ED}) where ED = x.index == -1
-Base.show(io::IO, x::ObjectID{ED}) where ED = print(io, "#$(x.index+1)")
+Base.iszero(x::ObjectID{ED}) where ED = x.index < 0
+Base.show(io::IO, x::ObjectID{ED}) where ED = print(io, "#$(iszero(x) ? 0 : x.index+1)")
 Base.convert(::Type{Integer}, i::ObjectID{ED}) where ED = i.index+1
-Base.convert(::Type{ED}, i::ObjectID{ED}) where ED = iszero(i.index+1) ? nothing : @inbounds EDStore_objects(ED, i.collectionID)[i.index+1]
+Base.convert(::Type{ED}, i::ObjectID{ED}) where ED = iszero(i) ? nothing : @inbounds EDStore_objects(ED, i.collectionID)[i.index+1]
 Base.convert(::Type{ObjectID{ED}}, p::ED) where ED = iszero(p.index) ? register(p).index : return p.index
 Base.convert(::Type{ObjectID{ED}}, i::Integer) where ED = ObjectID{ED}(i,0)
 Base.eltype(::Type{ObjectID{ED}}) where ED = ED
 Base.:-(i::ObjectID{ED}) where ED = ObjectID{ED}(-i.index)
+function Base.getproperty(oid::ObjectID{ED}, sym::Symbol) where ED
+    if sym == :object
+        convert(ED, oid)
+    else # fallback to getfield
+        return getfield(oid, sym)
+    end
+end
+Base.propertynames(oid::ObjectID) = tuple(fieldnames(ObjectID)...,:object)
 function register(p::ED) where ED
     collid = collectionID(ED)
     store::Vector{ED} = EDStore_objects(ED, collid)

src/RootIO.jl

@@ -18,6 +18,8 @@ module RootIO
     "long long" => Int64, "unsigned long long" => UInt64,
     "string" => String)
 
+    const newpodio = v"0.16.99"
+
     """
     The Reader struture keeps a reference to the UnROOT LazyTree and caches already built 'layouts' of the EDM4hep types.
     The layouts maps a set of columns in the LazyTree into an object.
@@ -27,6 +29,7 @@ module RootIO
         treename::String
         file::ROOTFile
         isRNTuple::Bool
+        podioversion::VersionNumber
         collectionIDs::Dict{String, UInt32}
         collectionNames::Dict{UInt32, String}
         btypes::Dict{String, Type} 
@@ -46,19 +49,23 @@ module RootIO
             rtuple = reader.file["podio_metadata"]
             if rtuple isa UnROOT.RNTuple
                 reader.isRNTuple = true
-                meta = LazyTree(rtuple, ["events___idTable", "events_collectionNames"])[1]
+                meta = LazyTree(rtuple, ["events___idTable", "events_collectionNames",  "PodioBuildVersion"])[1]
                 reader.collectionIDs = Dict(meta.events_collectionNames .=> meta.events___idTable)
                 reader.collectionNames = Dict(meta.events___idTable .=> meta.events_collectionNames)
+                reader.podioversion = VersionNumber(meta.PodioBuildVersion...)
             else
                 reader.isRNTuple = false
-                meta = LazyTree(reader.file, "podio_metadata", [Regex("events___idTable/(.*)") => s"\1"])[1]
+                meta = LazyTree(reader.file, "podio_metadata", [Regex("events___idTable/|PodioBuildVersion/(.*)") => s"\1"])[1]
                 reader.collectionIDs = Dict(meta.m_names .=> meta.m_collectionIDs)
                 reader.collectionNames = Dict(meta.m_collectionIDs .=> meta.m_names)
+                reader.podioversion = VersionNumber(meta.major, meta.minor, meta.patch)
             end
         else
-            @warn "ROOT file $filename does not have a 'podio_metadate' tree. Is it a PODIO file?"
-            reader.collectionIDs = Dict{UInt32, String}()
-            reader.collectionNames = Dict{String, UInt32}()
+            error("""ROOT file $(reader.filename) does not have a 'podio_metadata' tree. 
+                     Is it a PODIO file? or perhaps is from a very old version of podio?
+                     Stopping here.""")
+            #reader.collectionIDs = Dict{UInt32, String}()
+            #reader.collectionNames = Dict{String, UInt32}()
         end
         # layouts and branch types
         reader.btypes = Dict{String, Type}()
@@ -67,13 +74,15 @@ module RootIO
     end
 
 
-    function buildlayoutTTree(tree::UnROOT.LazyTree, branch::String, T::Type)
+    function buildlayoutTTree(reader::Reader, branch::String, T::Type)
         layout = []
         relations = []
         vmembers = []
         fnames = fieldnames(T)
         ftypes = fieldtypes(T)
-        splitnames = names(tree)
+        splitnames = names(reader.lazytree)
+        n_rels  = 0      # number of one-to-one or one-to-many Relations 
+        n_pvecs = 0      # number of vector member
         for (fn,ft) in zip(fnames, ftypes)
             n = "$(branch)_$(fn)"
             if isempty(fieldnames(ft))          # foundamental type (Int, Float,...)
@@ -83,37 +92,65 @@ module RootIO
                 b = findfirst(x -> x == n * "_begin", splitnames)
                 e = findfirst(x -> x == n * "_end", splitnames)
                 push!(layout, (ft, (b,e,-2)))   # -2 is collectionID of himself
-                push!(relations, ("_$(branch)_$(fn)", eltype(ft)))  # add a tuple with (relation_branchname, target_type)
+                if reader.podioversion >= newpodio
+                    push!(relations, ("_$(branch)_$(fn)", eltype(ft)))  # add a tuple with (relation_branchname, target_type)
+                else
+                    push!(relations, ("$(branch)#$(n_rels)", eltype(ft)))
+                    n_rels += 1
+                end
             elseif ft <: PVector
                 b = findfirst(x -> x == n * "_begin", splitnames)
+                if isnothing(b)
+                    b = findfirst(x -> lowercase(x) == lowercase( n * "_begin"), splitnames)
+                end
                 e = findfirst(x -> x == n * "_end", splitnames)
+                if isnothing(e)
+                    e = findfirst(x -> lowercase(x) == lowercase( n * "_end"), splitnames)
+                end
                 push!(layout, (ft, (b,e,-2)))   # -2 is collectionID of himself
-                push!(vmembers, ("_$(branch)_$(fn)", eltype(ft)))  # add a tuple with (relation_branchname, target_type)
+                if reader.podioversion >= newpodio
+                    push!(vmembers, ("_$(branch)_$(fn)", eltype(ft)))  # add a tuple with (vector_branchname, target_type)
+                else
+                    push!(vmembers, ("$(branch)_$(n_pvecs)", eltype(ft)))  # add a tuple with (vector_branchname, target_type)
+                    n_pvecs += 1
+                end
             elseif ft <: ObjectID{T}            # index of himself
                 push!(layout, (ft, (-1,-2)))
             elseif ft <: ObjectID               # index of another one....
                 na = replace("$(fn)", "_idx" => "")     # remove the added suffix
-                id = findfirst(x -> x == "_$(branch)_$(na)_index", splitnames)
-                if isnothing(id)  # try with case insensitive compare
-                    id = findfirst(x -> lowercase(x) == lowercase("_$(branch)_$(na)_index"), splitnames)
+                id = cid = nothing
+                if reader.podioversion >= newpodio
+                    id = findfirst(x -> x == "_$(branch)_$(na)_index", splitnames)
+                    if isnothing(id)  # try with case insensitive compare
+                        id = findfirst(x -> lowercase(x) == lowercase("_$(branch)_$(na)_index"), splitnames)
+                    end
+                    cid = findfirst(x -> x == "_$(branch)_$(na)_collectionID", splitnames)
+                    if isnothing(cid)  # try with case insensitive compare
+                        cid = findfirst(x -> lowercase(x) == lowercase("_$(branch)_$(na)_collectionID"), splitnames)
+                    end
+                else
+                    id = findfirst(x -> x == "$(branch)#$(n_rels)_index", splitnames)
+                    cid = findfirst(x -> x == "$(branch)#$(n_rels)_collectionID", splitnames)
+                    n_rels += 1
                 end
-                cid = findfirst(x -> x == "_$(branch)_$(na)_collectionID", splitnames)
-                if isnothing(cid)  # try with case insensitive compare
-                    cid = findfirst(x -> lowercase(x) == lowercase("_$(branch)_$(na)_collectionID"), splitnames)
+                if isnothing(id) || isnothing(cid)  # link not found in the data file
+                    @warn "Cannot find branch for one-to-one relation $(branch)::$(na)"
+                    push!(layout, (ft,(0,0)))
+                else
+                    push!(layout, (ft, (id, cid)))
                 end
-                push!(layout, (ft, (id, cid)))
             elseif ft <: SVector                # fixed arrays are translated to SVector
                 s = size(ft)[1]
                 id = findfirst(x-> x == n * "[$(s)]", splitnames)
                 push!(layout, (ft,(id,s))) 
             else
-                push!(layout, buildlayoutTTree(tree, n, ft))
+                push!(layout, buildlayoutTTree(reader, n, ft))
             end
         end
         (T, Tuple(layout), Tuple(relations), Tuple(vmembers))
     end
 
-    function buildlayoutRNTuple(tree::UnROOT.LazyTree, branch::String, T::Type)
+    function buildlayoutRNTuple(reader::Reader, branch::String, T::Type)
         relations = []
         vmembers = []
         fnames = fieldnames(T)
@@ -141,6 +178,15 @@ module RootIO
                 if l[1] <: SVector    # (type,(id, size))
                     ft, (id, s) = l
                     push!(sa, StructArray{ft}(reshape(evt[id], s, len);dims=1))
+                elseif l[1] <: ObjectID
+                    ft, (id, cid) = l
+                    if id == -1                             # self ObjectID
+                        push!(sa, StructArray{ft}((collect(0:len-1),fill(collid, len))))
+                    elseif len > 0 && evt[cid][1] == -2     # Handle the case collid is -2 :-( )
+                        push!(sa, StructArray{ft}((evt[id],zeros(UInt32,len))))
+                    else                                    # general case
+                        push!(sa, StructArray{ft}((evt[id],evt[cid])))
+                    end    
                 else
                     push!(sa, getStructArrayTTree(evt, l, collid, len))
                 end
@@ -268,23 +314,15 @@ module RootIO
             end
         end
     end
-
-    """
-    get(reader::Reader, evt::UnROOT.LazyEvent, bname::String; btype::Type=Any, register=true)
-
-    Gets an object collection by its name, with the possibility to overwrite the mapping Julia type or use the 
-    type known in the ROOT file (C++ class name). The optonal key parameter `register` indicates is the collection
-    needs to be registered to the `EDStore`.
-    """
-    function get(reader::Reader, evt::UnROOT.LazyEvent, bname::String; btype::Type=Any, register=true)
-        btype =  btype === Any ? reader.btypes[bname] : btype    # Allow the user to force the actual type
+
+    function _get(reader::Reader, evt::UnROOT.LazyEvent, bname::String, btype::Type, register::Bool)
         if haskey(reader.layouts, bname)                         # Check whether the the layout has been pre-compiled 
             layout = reader.layouts[bname]
         else
             if reader.isRNTuple
-                layout = buildlayoutRNTuple(reader.lazytree, bname, btype)
+                layout = buildlayoutRNTuple(reader, bname, btype)
             else
-                layout = buildlayoutTTree(reader.lazytree, bname, btype)
+                layout = buildlayoutTTree(reader, bname, btype)
             end
             reader.layouts[bname] = layout
         end
@@ -302,14 +340,33 @@ module RootIO
         if register
             assignEDStore(sa, collid)
             if !isempty(layout[3])  # check if there are relations in this branch
-                relations = Tuple(get(reader, evt, rb, btype=ObjectID{rt}; register=false) for (rb, rt) in layout[3])
+                relations = Tuple(_get(reader, evt, rb, ObjectID{rt}, false) for (rb, rt) in layout[3])
                 assignEDStore_relations(relations, btype, collid)
             end
             if !isempty(layout[4])  # check if there are vector members in this branch
-                vmembers = Tuple(get(reader, evt, rb, btype=rt; register=false) for (rb, rt) in layout[4])
+                vmembers = Tuple(_get(reader, evt, rb, rt, false) for (rb, rt) in layout[4])
                 assignEDStore_vmembers(vmembers, btype, collid)
             end
         end
         sa
     end
+
+    """
+    get(reader::Reader, evt::UnROOT.LazyEvent, bname::String; btype::Type=Any, register=true)
+
+    Gets an object collection by its name, with the possibility to overwrite the mapping Julia type or use the 
+    type known in the ROOT file (C++ class name). The optonal key parameter `register` indicates is the collection
+    needs to be registered to the `EDStore`.
+    """
+    function get(reader::Reader, evt::UnROOT.LazyEvent, bname::String; btype::Type=Any, register=true)
+        btype = btype === Any ? reader.btypes[bname] : btype     # Allow the user to force the actual type
+        if btype == ObjectID
+            register=false                    # Do not register a collection of ObjectIDs
+            sa = _get(reader, evt, bname, ObjectID{EDM4hep.POD}, register)
+            return convert.(eltype(eltype(sa)), sa)
+        else
+            sa = _get(reader, evt, bname, btype, register)
+            return sa
+        end
+    end
 end

test/runtests.jl

@@ -9,4 +9,5 @@ using EDM4hep
     include("testCluster.jl")         # several one-to-many and Vector members
     #---ROOT I/O----------------------
     include("testRootReader.jl")      # TTree and RNTuple reader
+    include("testRootReaderLegacy.jl")# Testing podio verion < 0.17 
 end

test/testRootReaderLegacy.jl

@@ -0,0 +1,31 @@
+using EDM4hep.RootIO
+
+@testset "TTreeReaderLegacy" begin
+    f =  "root://eospublic.cern.ch//eos/experiment/fcc/ee/generation/DelphesEvents/winter2023/IDEA/p8_ee_ZZ_ecm240/events_000189367.root"
+    #f = "/Users/mato/cernbox/Data/events_000189367.root"
+
+    reader = RootIO.Reader(f)
+    events = RootIO.get(reader, "events")
+
+    @test reader.isRNTuple == false
+    @test length(reader.collectionIDs) == 16
+    @test length(reader.btypes) > 54
+    @test reader.btypes["Particle"] == MCParticle
+
+    events = RootIO.get(reader, "events")
+    @test length(events) == 100000
+
+    # Loop over MC particles
+    for evt in events[1:100]
+        recps = RootIO.get(reader, evt, "ReconstructedParticles");
+        tracks = RootIO.get(reader, evt, "EFlowTrack")
+        pids  =  RootIO.get(reader, evt, "ParticleIDs")
+        if VERSION >= v"1.10"   # Seems to be working only for >= 1.10
+            muons = RootIO.get(reader, evt, "Muon#0")
+            if length(muons) == 2
+                @test abs(sum(muons.charge)) <= 2.0f0
+            end
+        end
+    end
+
+end