From 619b4945633aea903a759ca4d64a9a46f15b0580 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 15:29:52 -0400 Subject: [PATCH 01/27] rename utils --- src/rail/{core => utils}/algo_utils.py | 0 src/rail/{core/utils.py => utils/core_utils.py} | 0 src/rail/{core/util_stages.py => utils/stage_utils.py} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename src/rail/{core => utils}/algo_utils.py (100%) rename src/rail/{core/utils.py => utils/core_utils.py} (100%) rename src/rail/{core/util_stages.py => utils/stage_utils.py} (100%) diff --git a/src/rail/core/algo_utils.py b/src/rail/utils/algo_utils.py similarity index 100% rename from src/rail/core/algo_utils.py rename to src/rail/utils/algo_utils.py diff --git a/src/rail/core/utils.py b/src/rail/utils/core_utils.py similarity index 100% rename from src/rail/core/utils.py rename to src/rail/utils/core_utils.py diff --git a/src/rail/core/util_stages.py b/src/rail/utils/stage_utils.py similarity index 100% rename from src/rail/core/util_stages.py rename to src/rail/utils/stage_utils.py From e0cbeeb7bde9a8aa9a9a413e49a9dddf008d656a Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 15:32:40 -0400 Subject: [PATCH 02/27] change import path for the stage_utils --- src/rail/core/__init__.py | 2 +- src/rail/stages/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rail/core/__init__.py b/src/rail/core/__init__.py index 3da2451e..79bb9936 100644 --- a/src/rail/core/__init__.py +++ b/src/rail/core/__init__.py @@ -7,7 +7,7 @@ from .stage import RailPipeline, RailStage # from .utilPhotometry import PhotormetryManipulator, HyperbolicSmoothing, HyperbolicMagnitudes -from .util_stages import ColumnMapper, RowSelector, TableConverter +# from .util_stages import ColumnMapper, RowSelector, TableConverter from .introspection import RailEnv from .point_estimation import PointEstimationMixin diff --git a/src/rail/stages/__init__.py b/src/rail/stages/__init__.py index bad7524c..4b98ee05 100644 --- a/src/rail/stages/__init__.py +++ b/src/rail/stages/__init__.py @@ -31,6 +31,7 @@ from rail.evaluation.point_to_point_evaluator import PointToPointEvaluator from rail.evaluation.single_evaluator import SingleEvaluator +from rail.utils.stage_utils import ColumnMapper, RowSelector, TableConverter def import_and_attach_all(): """Import all the packages in the rail ecosystem and attach them to this module""" From a2426ddccb7f1347ef74629459683e23b4730b00 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 15:37:20 -0400 Subject: [PATCH 03/27] change import path for algo_utils --- tests/estimation/test_algos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/estimation/test_algos.py b/tests/estimation/test_algos.py index ceb171e6..f81627ab 100644 --- a/tests/estimation/test_algos.py +++ b/tests/estimation/test_algos.py @@ -1,7 +1,7 @@ import numpy as np import scipy.special -from rail.core.algo_utils import one_algo +from rail.utils.algo_utils import one_algo from rail.core.stage import RailStage from rail.estimation.algos import random_gauss, train_z From c267cf7f914d0ea7afb89cb9db5e089bd5507416 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 15:43:23 -0400 Subject: [PATCH 04/27] move unit test for utils to tests/utils --- tests/core/test_core.py | 56 ----------------------------------------- 1 file changed, 56 deletions(-) diff --git a/tests/core/test_core.py b/tests/core/test_core.py index cc5fbf7d..1e0cf6b7 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -18,55 +18,12 @@ TableHandle, ) from rail.core.stage import RailStage -from rail.core.utils import RAILDIR, find_rail_file -from rail.core.util_stages import ( - ColumnMapper, - RowSelector, - TableConverter, -) # def test_data_file(): # with pytest.raises(ValueError) as errinfo: # df = DataFile('dummy', 'x') -def test_find_rail_file(): - afile = find_rail_file(os.path.join("examples_data", "testdata", "test_dc2_training_9816.pq")) - assert afile - with pytest.raises(ValueError): - _not_a_file = find_rail_file("not_a_file") - - -def test_util_stages(): - DS = RailStage.data_store - DS.clear() - datapath = os.path.join( - RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.pq" - ) - - data = DS.read_file("data", TableHandle, datapath) - - table_conv = TableConverter.make_stage(name="conv", output_format="numpyDict") - col_map = ColumnMapper.make_stage(name="col_map", columns={}) - row_sel = RowSelector.make_stage(name="row_sel", start=1, stop=15) - - with pytest.raises(KeyError) as _errinfo: - table_conv.get_handle("nope", allow_missing=False) - - _conv_data = table_conv(data) - mapped_data = col_map(data) - _sel_data = row_sel(mapped_data) - - row_sel_2 = RowSelector.make_stage(name="row_sel_2", start=1, stop=15) - row_sel_2.set_data("input", mapped_data.data) - handle = row_sel_2.get_handle("input") - - row_sel_3 = RowSelector.make_stage( - name="row_sel_3", input=handle.path, start=1, stop=15 - ) - row_sel_3.set_data("input", None, do_read=True) - - def do_data_handle(datapath, handle_class): _DS = RailStage.data_store @@ -403,19 +360,6 @@ def test_common_params(): assert par.value == 0.1 assert par.dtype == float - -def test_set_data_nonexistent_file(): - """Create an instance of a child class of RailStage. Exercise the `set_data` - method and pass in a path to a nonexistent file. A `FileNotFound` exception - should be raised. - """ - - col_map = ColumnMapper.make_stage(name="col_map", columns={}) - with pytest.raises(FileNotFoundError) as err: - col_map.set_data("model", None, path="./bad_directory/no_file.py") - assert "Unable to find file" in err.context - - def test_set_data_real_file(): """Create an instance of a child class of RailStage. Exercise the `set_data` method and pass in a path to model. The output of set_data should be `None`. From 4f01afad9315ae79a296a7e290a12b5c61a39c21 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 15:47:02 -0400 Subject: [PATCH 05/27] change import for find_rail_file --- tests/evaluation/test_evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/evaluation/test_evaluation.py b/tests/evaluation/test_evaluation.py index 6f8188ee..2a50abb2 100644 --- a/tests/evaluation/test_evaluation.py +++ b/tests/evaluation/test_evaluation.py @@ -8,7 +8,7 @@ import rail.evaluation.metrics.pointestimates as pe from rail.core.data import QPHandle, TableHandle, QPOrTableHandle from rail.core.stage import RailStage -from rail.core.utils import find_rail_file +from rail.utils.core_utils import find_rail_file from rail.evaluation.evaluator import OldEvaluator from rail.evaluation.dist_to_dist_evaluator import DistToDistEvaluator from rail.evaluation.dist_to_point_evaluator import DistToPointEvaluator From afb228a1cee501484cf535a2ec0e9deafec4f429 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:02:13 -0400 Subject: [PATCH 06/27] move RAILDIR from core.utils to core.common_params and change all the import path --- src/rail/cli/scripts.py | 2 +- src/rail/core/common_params.py | 2 ++ src/rail/utils/algo_utils.py | 2 +- src/rail/utils/core_utils.py | 1 - tests/core/test_pipeline.py | 2 +- tests/estimation/test_classifier.py | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rail/cli/scripts.py b/src/rail/cli/scripts.py index c8d5b92f..6d5d04b2 100644 --- a/src/rail/cli/scripts.py +++ b/src/rail/cli/scripts.py @@ -4,7 +4,7 @@ import rail.stages from rail.core import RailEnv from rail.cli.options import GitMode -from rail.core.utils import RAILDIR +from rail.core.common_params import RAILDIR def render_nb(outdir, clear_output, dry_run, inputs, skip, **_kwargs): diff --git a/src/rail/core/common_params.py b/src/rail/core/common_params.py index 6356bf7b..703777b8 100644 --- a/src/rail/core/common_params.py +++ b/src/rail/core/common_params.py @@ -3,6 +3,8 @@ from ceci.config import StageParameter as Param from ceci.config import StageConfig +RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) + lsst_bands = "ugrizy" lsst_mag_cols = [f"mag_{band}_lsst" for band in lsst_bands] lsst_mag_err_cols = [f"mag_err_{band}_lsst" for band in lsst_bands] diff --git a/src/rail/utils/algo_utils.py b/src/rail/utils/algo_utils.py index bf86e14e..9ddcbfcc 100644 --- a/src/rail/utils/algo_utils.py +++ b/src/rail/utils/algo_utils.py @@ -2,7 +2,7 @@ import os import scipy.special from rail.core.stage import RailStage -from rail.core.utils import RAILDIR +from rail.core.common_params import RAILDIR from rail.core.data import TableHandle diff --git a/src/rail/utils/core_utils.py b/src/rail/utils/core_utils.py index bfe21f04..7a070341 100644 --- a/src/rail/utils/core_utils.py +++ b/src/rail/utils/core_utils.py @@ -4,7 +4,6 @@ import rail import rail.core -RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) def find_rail_file(relpath): diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py index 8df171de..114e7350 100644 --- a/tests/core/test_pipeline.py +++ b/tests/core/test_pipeline.py @@ -4,7 +4,7 @@ import numpy as np from rail.core.stage import RailPipeline, RailStage -from rail.core.utils import RAILDIR +from rail.core.common_params import RAILDIR from rail.core.util_stages import ColumnMapper, TableConverter diff --git a/tests/estimation/test_classifier.py b/tests/estimation/test_classifier.py index 6b795455..df8497fa 100644 --- a/tests/estimation/test_classifier.py +++ b/tests/estimation/test_classifier.py @@ -2,7 +2,7 @@ import numpy as np import pytest -from rail.core.utils import RAILDIR +from rail.core.common_params import RAILDIR from rail.core.stage import RailStage from rail.core.data import QPHandle from rail.estimation.algos.uniform_binning import UniformBinningClassifier From b04aad9d857172807cf182608b654dc4c67ce6a1 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:06:39 -0400 Subject: [PATCH 07/27] fix import for common_params.py --- src/rail/core/common_params.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rail/core/common_params.py b/src/rail/core/common_params.py index 703777b8..e3f5e442 100644 --- a/src/rail/core/common_params.py +++ b/src/rail/core/common_params.py @@ -2,6 +2,7 @@ from ceci.config import StageParameter as Param from ceci.config import StageConfig +import os RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) From 77ae7605ee8917070bdeef4b549e12a7030933e2 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:09:12 -0400 Subject: [PATCH 08/27] fix import for common_params.py --- src/rail/core/common_params.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rail/core/common_params.py b/src/rail/core/common_params.py index e3f5e442..293564cd 100644 --- a/src/rail/core/common_params.py +++ b/src/rail/core/common_params.py @@ -2,6 +2,7 @@ from ceci.config import StageParameter as Param from ceci.config import StageConfig +import rail import os RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) From d12d361a0d34d39effeec2886c3baff8f83dc470 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:14:44 -0400 Subject: [PATCH 09/27] fix import for test_summarizer --- tests/estimation/test_summarizers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/estimation/test_summarizers.py b/tests/estimation/test_summarizers.py index a48d98ec..c8279731 100644 --- a/tests/estimation/test_summarizers.py +++ b/tests/estimation/test_summarizers.py @@ -2,7 +2,7 @@ from rail.core.data import QPHandle from rail.core.stage import RailStage -from rail.core.utils import RAILDIR +from rail.core.common_params import RAILDIR from rail.estimation.algos import naive_stack, point_est_hist, var_inf testdata = os.path.join(RAILDIR, "rail/examples_data/testdata/output_BPZ_lite.hdf5") From c2bef09a57fbfbe59f0df73b46348cea992277f6 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:16:56 -0400 Subject: [PATCH 10/27] change import for tests/core/test_core.py --- tests/core/test_core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 1e0cf6b7..49f07bb8 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -18,6 +18,7 @@ TableHandle, ) from rail.core.stage import RailStage +from rail.core.common_params import RAILDIR # def test_data_file(): From 4f5eaf4cfdc6b686a586f795314db5fd3c9c36e9 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:25:14 -0400 Subject: [PATCH 11/27] fix import path --- tests/core/test_core.py | 55 -------------------------------- tests/core/test_pipeline.py | 2 +- tests/creation/test_degraders.py | 2 +- 3 files changed, 2 insertions(+), 57 deletions(-) diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 49f07bb8..7ddafd39 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -249,40 +249,6 @@ def test_model_handle(): pickle.dump(obj=mh3.data, file=fout, protocol=pickle.HIGHEST_PROTOCOL) os.remove(model_path_copy) - -def test_data_hdf5_iter(): - DS = RailStage.data_store - DS.clear() - - datapath = os.path.join( - RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.hdf5" - ) - - # data = DS.read_file('data', TableHandle, datapath) - th = Hdf5Handle("data", path=datapath) - x = th.iterator(groupname="photometry", chunk_size=1000) - - assert isinstance(x, GeneratorType) - for i, xx in enumerate(x): - assert xx[0] == i * 1000 - assert xx[1] - xx[0] <= 1000 - - _data = DS.read_file("input", TableHandle, datapath) - cm = ColumnMapper.make_stage( - input=datapath, - chunk_size=1000, - hdf5_groupname="photometry", - columns=dict(id="bob"), - ) - x = cm.input_iterator("input") - - assert isinstance(x, GeneratorType) - - for i, xx in enumerate(x): - assert xx[0] == i * 1000 - assert xx[1] - xx[0] <= 1000 - - def test_data_store(): DS = RailStage.data_store DS.clear() @@ -361,24 +327,3 @@ def test_common_params(): assert par.value == 0.1 assert par.dtype == float -def test_set_data_real_file(): - """Create an instance of a child class of RailStage. Exercise the `set_data` - method and pass in a path to model. The output of set_data should be `None`. - """ - DS = RailStage.data_store - DS.clear() - model_path = os.path.join( - RAILDIR, - "rail", - "examples_data", - "estimation_data", - "data", - "CWW_HDFN_prior.pkl", - ) - DS.add_data("model", None, ModelHandle, path=model_path) - - col_map = ColumnMapper.make_stage(name="col_map", columns={}) - - ret_val = col_map.set_data("model", None, path=model_path, do_read=False) - - assert ret_val is None diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py index 114e7350..163fbedb 100644 --- a/tests/core/test_pipeline.py +++ b/tests/core/test_pipeline.py @@ -5,7 +5,7 @@ from rail.core.stage import RailPipeline, RailStage from rail.core.common_params import RAILDIR -from rail.core.util_stages import ColumnMapper, TableConverter +from rail.utils.stage_utils import ColumnMapper, TableConverter def test_pipeline(): diff --git a/tests/creation/test_degraders.py b/tests/creation/test_degraders.py index 1263fae8..224b2f57 100644 --- a/tests/creation/test_degraders.py +++ b/tests/creation/test_degraders.py @@ -6,7 +6,7 @@ import pytest from rail.core.data import DATA_STORE, TableHandle -from rail.core.util_stages import ColumnMapper +from rail.utils.stage_utils import ColumnMapper from rail.creation.degradation.quantityCut import QuantityCut from rail.creation.degradation.addRandom import AddColumnOfRandom From ae973e36950bb8ebb59c0bfda11ea52b1c8493cb Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 16:30:37 -0400 Subject: [PATCH 12/27] add back the utils unit test in a new path --- tests/utils/test_utils.py | 119 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 tests/utils/test_utils.py diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py new file mode 100644 index 00000000..21657478 --- /dev/null +++ b/tests/utils/test_utils.py @@ -0,0 +1,119 @@ +import os +import pickle +from types import GeneratorType +import pytest + +from rail.core.stage import RailStage + +from rail.core.common_params import RAILDIR +from rail.utils.core_utils import find_rail_file +from rail.utils.stage_utils import ( + ColumnMapper, + RowSelector, + TableConverter, +) + + +def test_find_rail_file(): + afile = find_rail_file(os.path.join("examples_data", "testdata", "test_dc2_training_9816.pq")) + assert afile + with pytest.raises(ValueError): + _not_a_file = find_rail_file("not_a_file") + + +def test_util_stages(): + DS = RailStage.data_store + DS.clear() + datapath = os.path.join( + RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.pq" + ) + + data = DS.read_file("data", TableHandle, datapath) + + table_conv = TableConverter.make_stage(name="conv", output_format="numpyDict") + col_map = ColumnMapper.make_stage(name="col_map", columns={}) + row_sel = RowSelector.make_stage(name="row_sel", start=1, stop=15) + + with pytest.raises(KeyError) as _errinfo: + table_conv.get_handle("nope", allow_missing=False) + + _conv_data = table_conv(data) + mapped_data = col_map(data) + _sel_data = row_sel(mapped_data) + + row_sel_2 = RowSelector.make_stage(name="row_sel_2", start=1, stop=15) + row_sel_2.set_data("input", mapped_data.data) + handle = row_sel_2.get_handle("input") + + row_sel_3 = RowSelector.make_stage( + name="row_sel_3", input=handle.path, start=1, stop=15 + ) + row_sel_3.set_data("input", None, do_read=True) + +def test_set_data_nonexistent_file(): + """Create an instance of a child class of RailStage. Exercise the `set_data` + method and pass in a path to a nonexistent file. A `FileNotFound` exception + should be raised. + """ + + col_map = ColumnMapper.make_stage(name="col_map", columns={}) + with pytest.raises(FileNotFoundError) as err: + col_map.set_data("model", None, path="./bad_directory/no_file.py") + assert "Unable to find file" in err.context + + +def test_set_data_real_file(): + """Create an instance of a child class of RailStage. Exercise the `set_data` + method and pass in a path to model. The output of set_data should be `None`. + """ + DS = RailStage.data_store + DS.clear() + model_path = os.path.join( + RAILDIR, + "rail", + "examples_data", + "estimation_data", + "data", + "CWW_HDFN_prior.pkl", + ) + DS.add_data("model", None, ModelHandle, path=model_path) + + col_map = ColumnMapper.make_stage(name="col_map", columns={}) + + ret_val = col_map.set_data("model", None, path=model_path, do_read=False) + + assert ret_val is None + + +def test_data_hdf5_iter(): + DS = RailStage.data_store + DS.clear() + + datapath = os.path.join( + RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.hdf5" + ) + + # data = DS.read_file('data', TableHandle, datapath) + th = Hdf5Handle("data", path=datapath) + x = th.iterator(groupname="photometry", chunk_size=1000) + + assert isinstance(x, GeneratorType) + for i, xx in enumerate(x): + assert xx[0] == i * 1000 + assert xx[1] - xx[0] <= 1000 + + _data = DS.read_file("input", TableHandle, datapath) + cm = ColumnMapper.make_stage( + input=datapath, + chunk_size=1000, + hdf5_groupname="photometry", + columns=dict(id="bob"), + ) + x = cm.input_iterator("input") + + assert isinstance(x, GeneratorType) + + for i, xx in enumerate(x): + assert xx[0] == i * 1000 + assert xx[1] - xx[0] <= 1000 + From c94d077c161f7662f94541a6d055af521fd0b590 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 30 Apr 2024 18:06:32 -0400 Subject: [PATCH 13/27] fix utils unit test --- tests/utils/test_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 21657478..2ab080d3 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -4,6 +4,17 @@ import pytest from rail.core.stage import RailStage +from rail.core.data import ( + DataHandle, + DataStore, + FitsHandle, + Hdf5Handle, + ModelHandle, + PqHandle, + QPHandle, + QPOrTableHandle, + TableHandle, +) from rail.core.common_params import RAILDIR from rail.utils.core_utils import find_rail_file From 0337e1c5d8dc0d00e717f6c0b0d25daf6375b4ed Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Wed, 1 May 2024 11:48:36 -0400 Subject: [PATCH 14/27] keep from rail.core.utils import RAILDIR for the time being --- src/rail/core/utils.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/rail/core/utils.py diff --git a/src/rail/core/utils.py b/src/rail/core/utils.py new file mode 100644 index 00000000..98344443 --- /dev/null +++ b/src/rail/core/utils.py @@ -0,0 +1,5 @@ + +# This is a deprecative path for importing RAILDIR, it is only for the transition period +# to move RAILDIR to common_params + +from rail.core.common_params import RAILDIR From b017180de69e120964d6bfed86a5d34df95c598e Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Wed, 1 May 2024 13:56:20 -0400 Subject: [PATCH 15/27] add aliasing to find_rail_file to core.utils --- src/rail/core/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rail/core/utils.py b/src/rail/core/utils.py index 98344443..051741c1 100644 --- a/src/rail/core/utils.py +++ b/src/rail/core/utils.py @@ -3,3 +3,7 @@ # to move RAILDIR to common_params from rail.core.common_params import RAILDIR + +from rail.utils.core_utils import find_rail_file + +find_rail_file = find_rail_file \ No newline at end of file From 393a96c513480caf8cdb5429a4fe8d4bba6f3774 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Mon, 6 May 2024 18:35:57 -0400 Subject: [PATCH 16/27] move RAILDIR to path_utils --- src/rail/core/common_params.py | 1 - src/rail/core/utils.py | 4 +--- src/rail/utils/path_utils.py | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 src/rail/utils/path_utils.py diff --git a/src/rail/core/common_params.py b/src/rail/core/common_params.py index 293564cd..b3829e73 100644 --- a/src/rail/core/common_params.py +++ b/src/rail/core/common_params.py @@ -5,7 +5,6 @@ import rail import os -RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) lsst_bands = "ugrizy" lsst_mag_cols = [f"mag_{band}_lsst" for band in lsst_bands] diff --git a/src/rail/core/utils.py b/src/rail/core/utils.py index 051741c1..221cf956 100644 --- a/src/rail/core/utils.py +++ b/src/rail/core/utils.py @@ -2,8 +2,6 @@ # This is a deprecative path for importing RAILDIR, it is only for the transition period # to move RAILDIR to common_params -from rail.core.common_params import RAILDIR - -from rail.utils.core_utils import find_rail_file +from rail.utils.core_utils import find_rail_file, RAILDIR find_rail_file = find_rail_file \ No newline at end of file diff --git a/src/rail/utils/path_utils.py b/src/rail/utils/path_utils.py new file mode 100644 index 00000000..147e6118 --- /dev/null +++ b/src/rail/utils/path_utils.py @@ -0,0 +1,19 @@ +""" Utility functions """ + +import os +import rail +import rail.core + +RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) + +def find_rail_file(relpath): + """Find a file somewhere in rail by searching the namespace path + + This lets us avoid issues that the paths can be different depending + on if we have installed things from source or not + """ + for path_ in rail.__path__: + fullpath = os.path.abspath(os.path.join(path_, relpath)) + if os.path.exists(fullpath): + return fullpath + raise ValueError(f"Could not file {relpath} in {rail.__path__}") From 46a000fe15ca19a34a8142c763c92445cd302390 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Mon, 6 May 2024 18:37:36 -0400 Subject: [PATCH 17/27] remove core_utils --- src/rail/utils/core_utils.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 src/rail/utils/core_utils.py diff --git a/src/rail/utils/core_utils.py b/src/rail/utils/core_utils.py deleted file mode 100644 index 7a070341..00000000 --- a/src/rail/utils/core_utils.py +++ /dev/null @@ -1,19 +0,0 @@ -""" Utility functions """ - -import os -import rail -import rail.core - - - -def find_rail_file(relpath): - """Find a file somewhere in rail by searching the namespace path - - This lets us avoid issues that the paths can be different depending - on if we have installed things from source or not - """ - for path_ in rail.__path__: - fullpath = os.path.abspath(os.path.join(path_, relpath)) - if os.path.exists(fullpath): - return fullpath - raise ValueError(f"Could not file {relpath} in {rail.__path__}") From 17050da1ea1868ff80641847a496247b01426e05 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 11:22:58 -0400 Subject: [PATCH 18/27] rename algo_utils to testing_utils --- src/rail/utils/{algo_utils.py => testing_utils.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/rail/utils/{algo_utils.py => testing_utils.py} (100%) diff --git a/src/rail/utils/algo_utils.py b/src/rail/utils/testing_utils.py similarity index 100% rename from src/rail/utils/algo_utils.py rename to src/rail/utils/testing_utils.py From ee9975ce769154541ce87959382596fa50fc53c2 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 11:28:58 -0400 Subject: [PATCH 19/27] rename stage_utils to table_tools --- src/rail/{utils/stage_utils.py => tools/table_tools.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/rail/{utils/stage_utils.py => tools/table_tools.py} (100%) diff --git a/src/rail/utils/stage_utils.py b/src/rail/tools/table_tools.py similarity index 100% rename from src/rail/utils/stage_utils.py rename to src/rail/tools/table_tools.py From 674a590be9e7f07384a7c2425869d30bea3c6c08 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 17:10:20 -0400 Subject: [PATCH 20/27] change import path of RAILDIR --- src/rail/cli/scripts.py | 2 +- src/rail/utils/testing_utils.py | 2 +- tests/core/test_core.py | 2 +- tests/core/test_pipeline.py | 2 +- tests/estimation/test_classifier.py | 2 +- tests/estimation/test_summarizers.py | 2 +- tests/utils/test_utils.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rail/cli/scripts.py b/src/rail/cli/scripts.py index 6d5d04b2..4605cfb7 100644 --- a/src/rail/cli/scripts.py +++ b/src/rail/cli/scripts.py @@ -4,7 +4,7 @@ import rail.stages from rail.core import RailEnv from rail.cli.options import GitMode -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR def render_nb(outdir, clear_output, dry_run, inputs, skip, **_kwargs): diff --git a/src/rail/utils/testing_utils.py b/src/rail/utils/testing_utils.py index 9ddcbfcc..af77b069 100644 --- a/src/rail/utils/testing_utils.py +++ b/src/rail/utils/testing_utils.py @@ -2,7 +2,7 @@ import os import scipy.special from rail.core.stage import RailStage -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.core.data import TableHandle diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 7ddafd39..71b5a9a5 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -18,7 +18,7 @@ TableHandle, ) from rail.core.stage import RailStage -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR # def test_data_file(): diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py index 163fbedb..524c47d8 100644 --- a/tests/core/test_pipeline.py +++ b/tests/core/test_pipeline.py @@ -4,7 +4,7 @@ import numpy as np from rail.core.stage import RailPipeline, RailStage -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.utils.stage_utils import ColumnMapper, TableConverter diff --git a/tests/estimation/test_classifier.py b/tests/estimation/test_classifier.py index df8497fa..a8627069 100644 --- a/tests/estimation/test_classifier.py +++ b/tests/estimation/test_classifier.py @@ -2,7 +2,7 @@ import numpy as np import pytest -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.core.stage import RailStage from rail.core.data import QPHandle from rail.estimation.algos.uniform_binning import UniformBinningClassifier diff --git a/tests/estimation/test_summarizers.py b/tests/estimation/test_summarizers.py index c8279731..0814692c 100644 --- a/tests/estimation/test_summarizers.py +++ b/tests/estimation/test_summarizers.py @@ -2,7 +2,7 @@ from rail.core.data import QPHandle from rail.core.stage import RailStage -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.estimation.algos import naive_stack, point_est_hist, var_inf testdata = os.path.join(RAILDIR, "rail/examples_data/testdata/output_BPZ_lite.hdf5") diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 2ab080d3..392912c1 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -16,7 +16,7 @@ TableHandle, ) -from rail.core.common_params import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.utils.core_utils import find_rail_file from rail.utils.stage_utils import ( ColumnMapper, From 3fda5570196a26d97a5b3dc87ad35be3dc785a5d Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 17:16:13 -0400 Subject: [PATCH 21/27] change find_rail_file import path --- src/rail/core/utils.py | 2 +- tests/evaluation/test_evaluation.py | 2 +- tests/utils/test_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rail/core/utils.py b/src/rail/core/utils.py index 221cf956..ddd049fa 100644 --- a/src/rail/core/utils.py +++ b/src/rail/core/utils.py @@ -2,6 +2,6 @@ # This is a deprecative path for importing RAILDIR, it is only for the transition period # to move RAILDIR to common_params -from rail.utils.core_utils import find_rail_file, RAILDIR +from rail.utils.path_utils import find_rail_file, RAILDIR find_rail_file = find_rail_file \ No newline at end of file diff --git a/tests/evaluation/test_evaluation.py b/tests/evaluation/test_evaluation.py index 2a50abb2..30d22c17 100644 --- a/tests/evaluation/test_evaluation.py +++ b/tests/evaluation/test_evaluation.py @@ -8,7 +8,7 @@ import rail.evaluation.metrics.pointestimates as pe from rail.core.data import QPHandle, TableHandle, QPOrTableHandle from rail.core.stage import RailStage -from rail.utils.core_utils import find_rail_file +from rail.utils.path_utils import find_rail_file from rail.evaluation.evaluator import OldEvaluator from rail.evaluation.dist_to_dist_evaluator import DistToDistEvaluator from rail.evaluation.dist_to_point_evaluator import DistToPointEvaluator diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index 392912c1..ca50dd99 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -17,7 +17,7 @@ ) from rail.utils.path_utils import RAILDIR -from rail.utils.core_utils import find_rail_file +from rail.utils.path_utils import find_rail_file from rail.utils.stage_utils import ( ColumnMapper, RowSelector, From e28d9b403773cf19cd7c7a402df8b1dbf4abd943 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 17:17:30 -0400 Subject: [PATCH 22/27] change one_algo import path --- tests/estimation/test_algos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/estimation/test_algos.py b/tests/estimation/test_algos.py index f81627ab..eba19cb8 100644 --- a/tests/estimation/test_algos.py +++ b/tests/estimation/test_algos.py @@ -1,7 +1,7 @@ import numpy as np import scipy.special -from rail.utils.algo_utils import one_algo +from rail.utils.testing_utils import one_algo from rail.core.stage import RailStage from rail.estimation.algos import random_gauss, train_z From 90dd8d628aa996b99ce711be758069260f503363 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 17:20:24 -0400 Subject: [PATCH 23/27] change stage_utils path --- src/rail/stages/__init__.py | 2 +- tests/core/test_pipeline.py | 2 +- tests/creation/test_degraders.py | 2 +- tests/utils/test_utils.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rail/stages/__init__.py b/src/rail/stages/__init__.py index 4b98ee05..b5c211dc 100644 --- a/src/rail/stages/__init__.py +++ b/src/rail/stages/__init__.py @@ -31,7 +31,7 @@ from rail.evaluation.point_to_point_evaluator import PointToPointEvaluator from rail.evaluation.single_evaluator import SingleEvaluator -from rail.utils.stage_utils import ColumnMapper, RowSelector, TableConverter +from rail.tools.table_tools import ColumnMapper, RowSelector, TableConverter def import_and_attach_all(): """Import all the packages in the rail ecosystem and attach them to this module""" diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py index 524c47d8..bfa9fdc5 100644 --- a/tests/core/test_pipeline.py +++ b/tests/core/test_pipeline.py @@ -5,7 +5,7 @@ from rail.core.stage import RailPipeline, RailStage from rail.utils.path_utils import RAILDIR -from rail.utils.stage_utils import ColumnMapper, TableConverter +from rail.tools.table_tools import ColumnMapper, TableConverter def test_pipeline(): diff --git a/tests/creation/test_degraders.py b/tests/creation/test_degraders.py index 224b2f57..2ef3c5cf 100644 --- a/tests/creation/test_degraders.py +++ b/tests/creation/test_degraders.py @@ -6,7 +6,7 @@ import pytest from rail.core.data import DATA_STORE, TableHandle -from rail.utils.stage_utils import ColumnMapper +from rail.tools.table_tools import ColumnMapper from rail.creation.degradation.quantityCut import QuantityCut from rail.creation.degradation.addRandom import AddColumnOfRandom diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py index ca50dd99..af79d9a6 100644 --- a/tests/utils/test_utils.py +++ b/tests/utils/test_utils.py @@ -18,7 +18,7 @@ from rail.utils.path_utils import RAILDIR from rail.utils.path_utils import find_rail_file -from rail.utils.stage_utils import ( +from rail.tools.table_tools import ( ColumnMapper, RowSelector, TableConverter, From ae52120330962b815badfac25e9ee18473b1ac21 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 20:23:08 -0400 Subject: [PATCH 24/27] change degradation to degraders --- src/rail/creation/{degradation => degraders}/addRandom.py | 0 src/rail/creation/{degradation => degraders}/quantityCut.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/rail/creation/{degradation => degraders}/addRandom.py (100%) rename src/rail/creation/{degradation => degraders}/quantityCut.py (100%) diff --git a/src/rail/creation/degradation/addRandom.py b/src/rail/creation/degraders/addRandom.py similarity index 100% rename from src/rail/creation/degradation/addRandom.py rename to src/rail/creation/degraders/addRandom.py diff --git a/src/rail/creation/degradation/quantityCut.py b/src/rail/creation/degraders/quantityCut.py similarity index 100% rename from src/rail/creation/degradation/quantityCut.py rename to src/rail/creation/degraders/quantityCut.py From 67edade6c94d0eb9ed0474868de5ffdeb95a505a Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Tue, 7 May 2024 20:24:50 -0400 Subject: [PATCH 25/27] change degradation to degraders import path --- src/rail/stages/__init__.py | 6 +++--- tests/creation/test_degraders.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rail/stages/__init__.py b/src/rail/stages/__init__.py index b5c211dc..92b700f9 100644 --- a/src/rail/stages/__init__.py +++ b/src/rail/stages/__init__.py @@ -14,9 +14,9 @@ from rail.creation.degrader import * -#from rail.creation.degradation.spectroscopic_degraders import * -# from rail.creation.degradation.spectroscopic_selections import * -from rail.creation.degradation.quantityCut import * +#from rail.creation.degraders.spectroscopic_degraders import * +# from rail.creation.degraders.spectroscopic_selections import * +from rail.creation.degraders.quantityCut import * from rail.creation.engine import * diff --git a/tests/creation/test_degraders.py b/tests/creation/test_degraders.py index 2ef3c5cf..16644c24 100644 --- a/tests/creation/test_degraders.py +++ b/tests/creation/test_degraders.py @@ -7,8 +7,8 @@ from rail.core.data import DATA_STORE, TableHandle from rail.tools.table_tools import ColumnMapper -from rail.creation.degradation.quantityCut import QuantityCut -from rail.creation.degradation.addRandom import AddColumnOfRandom +from rail.creation.degraders.quantityCut import QuantityCut +from rail.creation.degraders.addRandom import AddColumnOfRandom @pytest.fixture From 93595937489f2eb6e56867dfca69878d8f750d58 Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Wed, 8 May 2024 10:48:32 -0400 Subject: [PATCH 26/27] delete common_params dependency --- src/rail/core/common_params.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rail/core/common_params.py b/src/rail/core/common_params.py index b3829e73..74c5577b 100644 --- a/src/rail/core/common_params.py +++ b/src/rail/core/common_params.py @@ -2,8 +2,8 @@ from ceci.config import StageParameter as Param from ceci.config import StageConfig -import rail -import os + + lsst_bands = "ugrizy" From 99616e329b036f589d15a3e59db69d5cebcb9e4c Mon Sep 17 00:00:00 2001 From: Tianqing Zhang Date: Wed, 8 May 2024 10:49:52 -0400 Subject: [PATCH 27/27] change utils comment --- src/rail/core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rail/core/utils.py b/src/rail/core/utils.py index ddd049fa..fea8aa09 100644 --- a/src/rail/core/utils.py +++ b/src/rail/core/utils.py @@ -1,6 +1,6 @@ # This is a deprecative path for importing RAILDIR, it is only for the transition period -# to move RAILDIR to common_params +# to move RAILDIR and find_rail_file to utils.path_utils from rail.utils.path_utils import find_rail_file, RAILDIR