Merge pull request #79 from legend-exp/lookup-table

Lookup table to generate `FileKey` Database

.codecov.yml

@@ -1 +1,3 @@
 # comment: false
+ignore:
+  - "ext/LegendDataManagementPlotsExt.jl"

.github/workflows/ci.yml

@@ -62,7 +62,7 @@ jobs:
         with:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
-          file: lcov.info
+          files: lcov.info
   docs:
     name: Documentation
     runs-on: ubuntu-latest

Project.toml

@@ -14,6 +14,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MIMEs = "6c6e2e6c-3030-632d-7369-2d6c69616d65"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7"
+OhMyThreads = "67456a42-1dca-4109-a031-0a68de7e3ad5"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
@@ -55,6 +56,7 @@ LinearAlgebra = "<0.0.1, 1"
 MIMEs = "0.1, 1"
 Markdown = "<0.0.1, 1"
 Measurements = "2.2.1"
+OhMyThreads = "0.5, 0.6, 0.7"
 Pkg = "1"
 Plots = "<0.0.1, 1"
 Printf = "<0.0.1, 1"
@@ -66,7 +68,7 @@ RecipesBase = "1"
 SolidStateDetectors = "0.10.2"
 StaticStrings = "0.2"
 Statistics = "<0.0.1, 1"
-StructArrays = "0.5, 0.6"
+StructArrays = "0.6.6, 0.7"
 Tables = "1.1"
 TypedTables = "1.4"
 UUIDs = "<0.0.1, 1"

docs/src/extensions.md

@@ -1,5 +1,65 @@
 # Extensions
 
+## `Plots` extension
+
+LegendDataManagment provides an extension for [Plots](https://github.com/JuliaPlots/Plots.jl). This makes it possible to directly plot LEGEND data via the `plot` function. The extension is automatically loaded when both packages are loaded.
+You can plot a parameter overview as a 2D plot over a set of detectors (requires a `$LEGEND_DATA_CONFIG` environment variable pointing to a legend data-config file):
+
+```julia
+using LegendDataManagement, Plots
+
+l200 = LegendData(:l200)
+
+filekey = FileKey("l200-p03-r000-cal-20230311T235840Z")
+
+pars = l200.par.ppars.ecal(filekey)
+properties = [:e_cusp_ctc, :fwhm, :qbb];
+
+chinfo = channelinfo(l200, filekey; system = :geds, only_processable = true)
+
+plot(chinfo, pars, properties, verbose = true, color = 1, markershape = :o, calculate_mean = true)
+```
+
+The plot recipe takes three arguments:
+- `chinfo`: the channel info with all detectors to be plotted on the x-axis
+- `pars`: a `PropDict` that has the detector IDs as keys and parameters as values
+- `properties`: an array of `Symbols` to access the data that should be plotted
+(if no `properties` are provided, the `PropDict` `pars` is expected to just contain the data to be plotted as values)
+
+There are also keyword arguments:
+- `calculate_mean`: If set to `true`, then the mean values are included in the legend labels. For values with uncertainties, the mean values are calculated as weighted means.
+- `verbose`: some output when the plot is generated, e.g. if values for (some) detectors are missing
+
+A 3D plot is WIP.
+
+In addition, you can plot an event display of the `raw` waveforms:
+``` julia
+using Unitful, LegendDataManagement, Plots
+
+l200 = LegendData(:l200)
+
+ts = 1.6785791257987175e9u"s"
+
+ch = ChannelId(1104000)
+
+plot(l200, ts, ch)
+```
+
+- `plot_tier`: The data tier to be plotted. Default is `DataTier(:raw)`.
+- `plot_waveform`: All waveforms to be plotted from the data. Default is `[:waveform_presummed]` which plots the presummed waveform.
+- `show_unixtime`: If set to `true`, use unix time instead of the datetime in the title. Default is `false`.
+
+If the channel is not given, the recipe automtically searches for the correct event in the data.
+``` julia
+ts = 1.6785791257987175e9u"s"
+
+plot(l200, ts)
+```
+In case of a `cal` event, only the HPGe channel with that event is plotted. In case of a `phy` event, all waveforms of the full HPGe and SiPM systems are plotted. 
+The following additional keywords arguments can be set (the `plot_waveform` kwarg is replaced by the `system` kwarg here):
+- `system`: The system and the waveforms to be plotted for each system. Default is `Dict{Symbol, Vector{Symbol}}([:geds, :spms] .=> [[:waveform_presummed], [:waveform_bit_drop]])`
+- `only_processable`: If set to `true`, only processable channels are plotted. Default is `true`.
+
 ## `LegendHDF5IO` extension
 
 LegendDataManagment provides an extension for [LegendHDF5IO](https://github.com/legend-exp/LegendHDF5IO.jl).

ext/LegendDataManagementLegendHDF5IOExt.jl

@@ -12,6 +12,15 @@ const ChannelOrDetectorIdLike = Union{ChannelIdLike, DetectorIdLike}
 const AbstractDataSelectorLike = Union{AbstractString, Symbol, DataTierLike, DataCategoryLike, DataPeriodLike, DataRunLike, DataPartitionLike, ChannelOrDetectorIdLike}
 const PossibleDataSelectors = [DataTier, DataCategory, DataPeriod, DataRun, DataPartition, ChannelId, DetectorId]
 
+function _is_valid_channel_or_tier(data::LegendData, rsel::Union{AnyValiditySelection, RunCategorySelLike}, det::ChannelOrDetectorIdLike)
+    if LegendDataManagement._can_convert_to(ChannelId, det) ||  LegendDataManagement._can_convert_to(DetectorId, det) ||  LegendDataManagement._can_convert_to(DataTier, det)
+        true  
+    else  
+        @warn "Skipped $det since it is neither a valid `ChannelId`, `DetectorId` nor a `DataTier`"  
+        false  
+    end  
+end
+
 function _get_channelid(data::LegendData, rsel::Union{AnyValiditySelection, RunCategorySelLike}, det::ChannelOrDetectorIdLike)
     if LegendDataManagement._can_convert_to(ChannelId, det)
         ChannelId(det)
@@ -143,6 +152,7 @@ function LegendDataManagement.read_ldata(f::Base.Callable, data::LegendData, rse
     ch_keys = _lh5_data_open(data, rsel[1], rsel[2], "") do h
         keys(h)
     end
+    filter!(x -> _is_valid_channel_or_tier(data, rsel[2], x), ch_keys)
     @debug "Found keys: $ch_keys"
     if length(ch_keys) == 1
         if string(only(ch_keys)) == string(rsel[1])

ext/LegendDataManagementPlotsExt.jl

@@ -8,8 +8,10 @@ using PropDicts
 using Statistics
 using TypedTables
 using Unitful
+using Dates
 using Format
 using Measurements: value, uncertainty, weightedmean
+using LegendDataManagement
 
 @recipe function f(
     chinfo::Table,
@@ -131,8 +133,165 @@ using Measurements: value, uncertainty, weightedmean
         seriescolor := c
         markerstrokecolor := c
         xvalues, value.(yvalues)
-    end  
+    end
+end
+
+@recipe function f(data::LegendData, fk::FileKey, ts::Unitful.Time{<:Real}, ch::ChannelIdLike; plot_tier=DataTier(:raw), plot_waveform=[:waveform_presummed], show_unixtime=false)
+    framestyle := :box
+    margins := (0.5, :mm)
+    yformatter := :plain
+    legend := :bottomright
+    plot_titlefontsize := 12
+    raw = read_ldata(data, plot_tier, fk, ch)
+    idx = findfirst(isequal(ts), raw.timestamp)
+    if isnothing(idx)
+        throw(ArgumentError("Timestamp $ts not found in the data"))
+    end
+    if show_unixtime
+        title := "Event $(ts)"
+    else
+        title := "Event $(Dates.unix2datetime(ustrip(u"s", ts)))"
+    end
+    plot_title := "$(fk.setup)-$(fk.period)-$(fk.run)-$(fk.category)"
+    for (p, p_wvf) in enumerate(plot_waveform)
+        @series begin
+            if p == 1
+                label := "$(channelinfo(data, fk, ch).detector) ($(ch))"
+            else
+                label := :none
+            end
+            color := 1
+            xunit := u"µs"
+            getproperty(raw, p_wvf)[idx]
+        end
+    end
+end
+
+# TODO: check rounding of `Dates.DateTime`
+@recipe function f(data::LegendData, fk::FileKey, ts::Dates.DateTime, ch::ChannelIdLike; plot_tier=DataTier(:raw), plot_waveform=[:waveform_presummed], show_unixtime=false)
+    @series begin
+        plot_tier := plot_tier
+        plot_waveform := plot_waveform
+        show_unixtime := show_unixtime
+        data, fk, Dates.datetime2unix(ts)*u"s", ch
+    end
 end
 
 
+@recipe function f(data::LegendData, ts::Unitful.Time{<:Real}, ch::ChannelIdLike; plot_tier=DataTier(:raw), plot_waveform=[:waveform_presummed], show_unixtime=false)
+    fk = find_filekey(data, ts)
+    @series begin
+        plot_tier := plot_tier
+        plot_waveform := plot_waveform
+        show_unixtime := show_unixtime
+        data, fk, ts, ch
+    end
+end
+
+# TODO: check rounding of `Dates.DateTime`
+@recipe function f(data::LegendData, ts::Dates.DateTime, ch::ChannelIdLike; plot_tier=DataTier(:raw), plot_waveform=[:waveform_presummed], show_unixtime=false)
+    fk = find_filekey(data, ts)
+    @series begin
+        plot_tier := plot_tier
+        plot_waveform := plot_waveform
+        show_unixtime := show_unixtime
+        data, fk, Dates.datetime2unix(ts)*u"s", ch
+    end
+end
+
+
+@recipe function f(data::LegendData, ts::Unitful.Time{<:Real}; plot_tier=DataTier(:raw), system=Dict{Symbol, Vector{Symbol}}([:geds, :spms] .=> [[:waveform_presummed], [:waveform_bit_drop]]), only_processable=true, show_unixtime=false)
+    fk = find_filekey(data, ts)
+    framestyle := :box
+    margins := (1, :mm)
+    yformatter := :plain
+    if fk.category == DataCategory(:cal)
+        @debug "Got $(fk.category) event, looking for raw event"
+        timestamps = read_ldata(:timestamp, data, DataTier(:raw), fk)
+        ch_ts = ""
+        for ch in keys(timestamps)
+            if any(ts .== timestamps[ch].timestamp)
+                ch_ts = string(ch)
+                @debug "Found event $ts in channel $ch"
+                break
+            end
+        end
+        if isempty(ch_ts)
+            throw(ArgumentError("Timestamp $ts not found in the data"))
+        end
+        ch = ChannelId(ch_ts)
+        chinfo_ch = channelinfo(data, fk, ch)
+        if chinfo_ch.system != :geds
+            throw(ArgumentError("Only HPGe cal events are supported"))
+        end
+        if only_processable && !chinfo_ch.processable
+            throw(ArgumentError("Channel $ch is not processable"))
+        end
+        @series begin
+            plot_tier := plot_tier
+            plot_waveform := system[:geds]
+            show_unixtime := show_unixtime
+            data, fk, ts, ch
+        end
+    elseif fk.category == DataCategory(:phy)
+        # load raw file with all channels
+        raw = read_ldata(data, plot_tier, fk)
+        # layout 
+        layout := (length(system), 1)
+        size := (1500, 500 * length(system))
+        margins := (1, :mm)
+        bottom_margin := (2, :mm)
+        legend := :none
+        legendcolumns := 4
+        @debug "Plot systems $(system) with waveforms $(plot_waveform)"
+        for (s, sys) in enumerate(sort(collect(keys(system))))
+            chinfo = channelinfo(data, fk; system=sys, only_processable=only_processable)
+            if !all(hasproperty.(Ref(raw[Symbol(first(chinfo.channel))]), system[sys]))
+                throw(ArgumentError("Property $(plot_waveform[s]) not found in the data"))
+            end
+            plot_title := "$(fk.setup)-$(fk.period)-$(fk.run)-$(fk.category)"
+            if show_unixtime
+                title := "$sys - Event $(ts)"
+            else
+                title := "$sys - Event $(Dates.unix2datetime(ustrip(u"s", ts)))"
+            end
+            for (c, chinfo_ch) in enumerate(chinfo)
+                @debug "Load $(chinfo_ch.detector)"
+                for (p, p_wvf) in enumerate(system[sys])
+                    @series begin
+                        if p == 1
+                            label := "$(chinfo_ch.detector) ($(chinfo_ch.channel))"
+                        else
+                            label := ""
+                        end
+                        color := c
+                        subplot := s
+                        xunit := u"µs"
+                        idx = findfirst(isequal(ts), raw[Symbol(chinfo_ch.channel)].timestamp)
+                        if isnothing(idx)
+                            @warn "Timestamp $ts not found in $(chinfo_ch.detector) ($(chinfo_ch.channel)) data"
+                            u"µs", NoUnits
+                        else
+                            getproperty(raw[Symbol(chinfo_ch.channel)], p_wvf)[idx]
+                        end
+                    end
+                end
+            end
+        end
+    else
+        throw(ArgumentError("Only `DataCategory` cal and phy are supported"))
+    end
+end
+
+# TODO: check rounding of `Dates.DateTime`
+@recipe function f(data::LegendData, ts::Dates.DateTime; plot_tier=DataTier(:raw), system=Dict{Symbol, Vector{Symbol}}([:geds, :spms] .=> [[:waveform_presummed], [:waveform_bit_drop]]), only_processable=true, show_unixtime=false)
+    @series begin
+        plot_tier := plot_tier
+        system := system
+        only_processable := only_processable
+        show_unixtime := show_unixtime
+        data, Dates.datetime2unix(ts)*u"s"
+    end
+end
+
 end # module LegendDataManagementPlotsExt

src/LegendDataManagement.jl

@@ -24,8 +24,9 @@ using Printf: @printf
 
 using IntervalSets: AbstractInterval, ClosedInterval, leftendpoint, rightendpoint
 using LRUCache: LRU
+using OhMyThreads: @tasks, tmapreduce
 using ProgressMeter: @showprogress
-using PropertyFunctions: PropertyFunction, @pf, filterby, props2varsyms, PropSelFunction
+using PropertyFunctions: PropertyFunction, @pf, filterby, sortby, props2varsyms, PropSelFunction
 using StaticStrings: StaticString
 import Tables
 using Tables: columns