From 563152dad18b2c2d4ca6fce91422dd8424b0b1a6 Mon Sep 17 00:00:00 2001 From: Toby Dixon Date: Tue, 28 Jan 2025 01:43:20 +0100 Subject: [PATCH 1/4] support table.eval returning np.ndarray also without numexpr --- src/lgdo/types/table.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/lgdo/types/table.py b/src/lgdo/types/table.py index bf59153..c84b303 100644 --- a/src/lgdo/types/table.py +++ b/src/lgdo/types/table.py @@ -386,6 +386,21 @@ def eval( return Array(out_data.to_numpy()) return VectorOfVectors(out_data) + # modules can still produce numpy array + if isinstance(out_data, np.ndarray): + if out_data.ndim == 0: + return Scalar(out_data.item()) + if out_data.ndim == 1: + return Array(out_data) + if out_data.ndim == 2: + return ArrayOfEqualSizedArrays(nda=out_data) + + msg = ( + f"evaluation resulted in {out_data.ndim}-dimensional data, " + "I don't know which LGDO this corresponds to" + ) + raise RuntimeError(msg) + if np.isscalar(out_data): return Scalar(out_data) From 113878b4cdbf8205331c3a9e454dddea691af313 Mon Sep 17 00:00:00 2001 From: Luigi Pertoldi Date: Tue, 28 Jan 2025 11:02:30 +0100 Subject: [PATCH 2/4] less duplicated code --- src/lgdo/types/table.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/src/lgdo/types/table.py b/src/lgdo/types/table.py index a59d508..3b5a1de 100644 --- a/src/lgdo/types/table.py +++ b/src/lgdo/types/table.py @@ -351,6 +351,20 @@ def eval( msg = f"evaluating {expr!r} with locals={(self_unwrap | parameters)} and {has_ak=}" log.debug(msg) + def _make_lgdo(data): + if data.ndim == 0: + return Scalar(data.item()) + if data.ndim == 1: + return Array(data) + if data.ndim == 2: + return ArrayOfEqualSizedArrays(nda=data) + + msg = ( + f"evaluation resulted in {data.ndim}-dimensional data, " + "I don't know which LGDO this corresponds to" + ) + raise RuntimeError(msg) + # use numexpr if we are only dealing with numpy data types (and no global dictionary) if not has_ak and modules is None: out_data = ne.evaluate( @@ -363,18 +377,7 @@ def eval( # need to convert back to LGDO # np.evaluate should always return a numpy thing? - if out_data.ndim == 0: - return Scalar(out_data.item()) - if out_data.ndim == 1: - return Array(out_data) - if out_data.ndim == 2: - return ArrayOfEqualSizedArrays(nda=out_data) - - msg = ( - f"evaluation resulted in {out_data.ndim}-dimensional data, " - "I don't know which LGDO this corresponds to" - ) - raise RuntimeError(msg) + return _make_lgdo(out_data) # resort to good ol' eval() globs = {"ak": ak, "np": np} @@ -394,18 +397,7 @@ def eval( # modules can still produce numpy array if isinstance(out_data, np.ndarray): - if out_data.ndim == 0: - return Scalar(out_data.item()) - if out_data.ndim == 1: - return Array(out_data) - if out_data.ndim == 2: - return ArrayOfEqualSizedArrays(nda=out_data) - - msg = ( - f"evaluation resulted in {out_data.ndim}-dimensional data, " - "I don't know which LGDO this corresponds to" - ) - raise RuntimeError(msg) + return _make_lgdo(out_data) if np.isscalar(out_data): return Scalar(out_data) From 8244792f3b7e2aaff87cff860e4673c70c3a8857 Mon Sep 17 00:00:00 2001 From: Toby Dixon Date: Tue, 28 Jan 2025 14:32:22 +0100 Subject: [PATCH 3/4] catch numexpr errors and try with eval() and also test the case that a module returns a np.array --- src/lgdo/types/table.py | 27 ++++++++++++++++----------- tests/types/test_table_eval.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/lgdo/types/table.py b/src/lgdo/types/table.py index 3b5a1de..2ffd570 100644 --- a/src/lgdo/types/table.py +++ b/src/lgdo/types/table.py @@ -367,17 +367,22 @@ def _make_lgdo(data): # use numexpr if we are only dealing with numpy data types (and no global dictionary) if not has_ak and modules is None: - out_data = ne.evaluate( - expr, - local_dict=(self_unwrap | parameters), - ) - - msg = f"...the result is {out_data!r}" - log.debug(msg) - - # need to convert back to LGDO - # np.evaluate should always return a numpy thing? - return _make_lgdo(out_data) + try: + out_data = ne.evaluate( + expr, + local_dict=(self_unwrap | parameters), + ) + + msg = f"...the result is {out_data!r}" + log.debug(msg) + + # need to convert back to LGDO + # np.evaluate should always return a numpy thing? + return _make_lgdo(out_data) + + except Exception: + msg = f"Warning {expr} could not be evaluated with numexpr probably due to some not allowed characters, trying with eval()." + log.debug(msg) # resort to good ol' eval() globs = {"ak": ak, "np": np} diff --git a/tests/types/test_table_eval.py b/tests/types/test_table_eval.py index 1c4a180..e221f27 100644 --- a/tests/types/test_table_eval.py +++ b/tests/types/test_table_eval.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dbetto import hist import numpy as np import pytest @@ -85,6 +86,19 @@ def test_eval_dependency(): res = obj.eval("lgdo.Array([1,2,3])", {}, modules={"lgdo": lgdo}) assert res == lgdo.Array([1, 2, 3]) + # test with module returning np.array + assert obj.eval("np.sum(a)", {}, modules={"np": np}).value == np.int64(10) + # check bad type with pytest.raises(RuntimeError): obj.eval("hist.Hist()", modules={"hist": hist}) + + # check impossible numexpr can still run + assert np.allclose( + obj.eval( + "a*args.value", + {"args": dbetto.AttrsDict({"value": 2})}, + modules={"lgdo": lgdo}, + ).view_as("np"), + [2, 4, 6, 8], + ) From c22d61072542c80f1c7f8a717549c79b74779427 Mon Sep 17 00:00:00 2001 From: Toby Dixon Date: Tue, 28 Jan 2025 14:34:29 +0100 Subject: [PATCH 4/4] add dbetto to test dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 466d3d7..e58b8da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ test = [ "pylegendtestdata", "pytest>=6.0", "pytest-cov", + "dbetto", ] [project.scripts]