diff --git a/src/rail/cli/scripts.py b/src/rail/cli/scripts.py index c8d5b92f..4605cfb7 100644 --- a/src/rail/cli/scripts.py +++ b/src/rail/cli/scripts.py @@ -4,7 +4,7 @@ import rail.stages from rail.core import RailEnv from rail.cli.options import GitMode -from rail.core.utils import RAILDIR +from rail.utils.path_utils import RAILDIR def render_nb(outdir, clear_output, dry_run, inputs, skip, **_kwargs): diff --git a/src/rail/core/__init__.py b/src/rail/core/__init__.py index 3da2451e..79bb9936 100644 --- a/src/rail/core/__init__.py +++ b/src/rail/core/__init__.py @@ -7,7 +7,7 @@ from .stage import RailPipeline, RailStage # from .utilPhotometry import PhotormetryManipulator, HyperbolicSmoothing, HyperbolicMagnitudes -from .util_stages import ColumnMapper, RowSelector, TableConverter +# from .util_stages import ColumnMapper, RowSelector, TableConverter from .introspection import RailEnv from .point_estimation import PointEstimationMixin diff --git a/src/rail/core/common_params.py b/src/rail/core/common_params.py index 6356bf7b..74c5577b 100644 --- a/src/rail/core/common_params.py +++ b/src/rail/core/common_params.py @@ -3,6 +3,9 @@ from ceci.config import StageParameter as Param from ceci.config import StageConfig + + + lsst_bands = "ugrizy" lsst_mag_cols = [f"mag_{band}_lsst" for band in lsst_bands] lsst_mag_err_cols = [f"mag_err_{band}_lsst" for band in lsst_bands] diff --git a/src/rail/core/utils.py b/src/rail/core/utils.py index bfe21f04..fea8aa09 100644 --- a/src/rail/core/utils.py +++ b/src/rail/core/utils.py @@ -1,20 +1,7 @@ -""" Utility functions """ -import os -import rail -import rail.core +# This is a deprecative path for importing RAILDIR, it is only for the transition period +# to move RAILDIR and find_rail_file to utils.path_utils -RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) +from rail.utils.path_utils import find_rail_file, RAILDIR - -def find_rail_file(relpath): - """Find a file somewhere in rail by searching the namespace path - - This lets us avoid issues that the paths can be different depending - on if we have installed things from source or not - """ - for path_ in rail.__path__: - fullpath = os.path.abspath(os.path.join(path_, relpath)) - if os.path.exists(fullpath): - return fullpath - raise ValueError(f"Could not file {relpath} in {rail.__path__}") +find_rail_file = find_rail_file \ No newline at end of file diff --git a/src/rail/creation/degradation/addRandom.py b/src/rail/creation/degraders/addRandom.py similarity index 100% rename from src/rail/creation/degradation/addRandom.py rename to src/rail/creation/degraders/addRandom.py diff --git a/src/rail/creation/degradation/quantityCut.py b/src/rail/creation/degraders/quantityCut.py similarity index 100% rename from src/rail/creation/degradation/quantityCut.py rename to src/rail/creation/degraders/quantityCut.py diff --git a/src/rail/stages/__init__.py b/src/rail/stages/__init__.py index bad7524c..92b700f9 100644 --- a/src/rail/stages/__init__.py +++ b/src/rail/stages/__init__.py @@ -14,9 +14,9 @@ from rail.creation.degrader import * -#from rail.creation.degradation.spectroscopic_degraders import * -# from rail.creation.degradation.spectroscopic_selections import * -from rail.creation.degradation.quantityCut import * +#from rail.creation.degraders.spectroscopic_degraders import * +# from rail.creation.degraders.spectroscopic_selections import * +from rail.creation.degraders.quantityCut import * from rail.creation.engine import * @@ -31,6 +31,7 @@ from rail.evaluation.point_to_point_evaluator import PointToPointEvaluator from rail.evaluation.single_evaluator import SingleEvaluator +from rail.tools.table_tools import ColumnMapper, RowSelector, TableConverter def import_and_attach_all(): """Import all the packages in the rail ecosystem and attach them to this module""" diff --git a/src/rail/core/util_stages.py b/src/rail/tools/table_tools.py similarity index 100% rename from src/rail/core/util_stages.py rename to src/rail/tools/table_tools.py diff --git a/src/rail/utils/path_utils.py b/src/rail/utils/path_utils.py new file mode 100644 index 00000000..147e6118 --- /dev/null +++ b/src/rail/utils/path_utils.py @@ -0,0 +1,19 @@ +""" Utility functions """ + +import os +import rail +import rail.core + +RAILDIR = os.path.abspath(os.path.join(os.path.dirname(rail.core.__file__), "..", "..")) + +def find_rail_file(relpath): + """Find a file somewhere in rail by searching the namespace path + + This lets us avoid issues that the paths can be different depending + on if we have installed things from source or not + """ + for path_ in rail.__path__: + fullpath = os.path.abspath(os.path.join(path_, relpath)) + if os.path.exists(fullpath): + return fullpath + raise ValueError(f"Could not file {relpath} in {rail.__path__}") diff --git a/src/rail/core/algo_utils.py b/src/rail/utils/testing_utils.py similarity index 98% rename from src/rail/core/algo_utils.py rename to src/rail/utils/testing_utils.py index bf86e14e..af77b069 100644 --- a/src/rail/core/algo_utils.py +++ b/src/rail/utils/testing_utils.py @@ -2,7 +2,7 @@ import os import scipy.special from rail.core.stage import RailStage -from rail.core.utils import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.core.data import TableHandle diff --git a/tests/core/test_core.py b/tests/core/test_core.py index cc5fbf7d..71b5a9a5 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -18,55 +18,13 @@ TableHandle, ) from rail.core.stage import RailStage -from rail.core.utils import RAILDIR, find_rail_file -from rail.core.util_stages import ( - ColumnMapper, - RowSelector, - TableConverter, -) +from rail.utils.path_utils import RAILDIR # def test_data_file(): # with pytest.raises(ValueError) as errinfo: # df = DataFile('dummy', 'x') -def test_find_rail_file(): - afile = find_rail_file(os.path.join("examples_data", "testdata", "test_dc2_training_9816.pq")) - assert afile - with pytest.raises(ValueError): - _not_a_file = find_rail_file("not_a_file") - - -def test_util_stages(): - DS = RailStage.data_store - DS.clear() - datapath = os.path.join( - RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.pq" - ) - - data = DS.read_file("data", TableHandle, datapath) - - table_conv = TableConverter.make_stage(name="conv", output_format="numpyDict") - col_map = ColumnMapper.make_stage(name="col_map", columns={}) - row_sel = RowSelector.make_stage(name="row_sel", start=1, stop=15) - - with pytest.raises(KeyError) as _errinfo: - table_conv.get_handle("nope", allow_missing=False) - - _conv_data = table_conv(data) - mapped_data = col_map(data) - _sel_data = row_sel(mapped_data) - - row_sel_2 = RowSelector.make_stage(name="row_sel_2", start=1, stop=15) - row_sel_2.set_data("input", mapped_data.data) - handle = row_sel_2.get_handle("input") - - row_sel_3 = RowSelector.make_stage( - name="row_sel_3", input=handle.path, start=1, stop=15 - ) - row_sel_3.set_data("input", None, do_read=True) - - def do_data_handle(datapath, handle_class): _DS = RailStage.data_store @@ -291,40 +249,6 @@ def test_model_handle(): pickle.dump(obj=mh3.data, file=fout, protocol=pickle.HIGHEST_PROTOCOL) os.remove(model_path_copy) - -def test_data_hdf5_iter(): - DS = RailStage.data_store - DS.clear() - - datapath = os.path.join( - RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.hdf5" - ) - - # data = DS.read_file('data', TableHandle, datapath) - th = Hdf5Handle("data", path=datapath) - x = th.iterator(groupname="photometry", chunk_size=1000) - - assert isinstance(x, GeneratorType) - for i, xx in enumerate(x): - assert xx[0] == i * 1000 - assert xx[1] - xx[0] <= 1000 - - _data = DS.read_file("input", TableHandle, datapath) - cm = ColumnMapper.make_stage( - input=datapath, - chunk_size=1000, - hdf5_groupname="photometry", - columns=dict(id="bob"), - ) - x = cm.input_iterator("input") - - assert isinstance(x, GeneratorType) - - for i, xx in enumerate(x): - assert xx[0] == i * 1000 - assert xx[1] - xx[0] <= 1000 - - def test_data_store(): DS = RailStage.data_store DS.clear() @@ -403,37 +327,3 @@ def test_common_params(): assert par.value == 0.1 assert par.dtype == float - -def test_set_data_nonexistent_file(): - """Create an instance of a child class of RailStage. Exercise the `set_data` - method and pass in a path to a nonexistent file. A `FileNotFound` exception - should be raised. - """ - - col_map = ColumnMapper.make_stage(name="col_map", columns={}) - with pytest.raises(FileNotFoundError) as err: - col_map.set_data("model", None, path="./bad_directory/no_file.py") - assert "Unable to find file" in err.context - - -def test_set_data_real_file(): - """Create an instance of a child class of RailStage. Exercise the `set_data` - method and pass in a path to model. The output of set_data should be `None`. - """ - DS = RailStage.data_store - DS.clear() - model_path = os.path.join( - RAILDIR, - "rail", - "examples_data", - "estimation_data", - "data", - "CWW_HDFN_prior.pkl", - ) - DS.add_data("model", None, ModelHandle, path=model_path) - - col_map = ColumnMapper.make_stage(name="col_map", columns={}) - - ret_val = col_map.set_data("model", None, path=model_path, do_read=False) - - assert ret_val is None diff --git a/tests/core/test_pipeline.py b/tests/core/test_pipeline.py index 8df171de..bfa9fdc5 100644 --- a/tests/core/test_pipeline.py +++ b/tests/core/test_pipeline.py @@ -4,8 +4,8 @@ import numpy as np from rail.core.stage import RailPipeline, RailStage -from rail.core.utils import RAILDIR -from rail.core.util_stages import ColumnMapper, TableConverter +from rail.utils.path_utils import RAILDIR +from rail.tools.table_tools import ColumnMapper, TableConverter def test_pipeline(): diff --git a/tests/creation/test_degraders.py b/tests/creation/test_degraders.py index 1263fae8..16644c24 100644 --- a/tests/creation/test_degraders.py +++ b/tests/creation/test_degraders.py @@ -6,9 +6,9 @@ import pytest from rail.core.data import DATA_STORE, TableHandle -from rail.core.util_stages import ColumnMapper -from rail.creation.degradation.quantityCut import QuantityCut -from rail.creation.degradation.addRandom import AddColumnOfRandom +from rail.tools.table_tools import ColumnMapper +from rail.creation.degraders.quantityCut import QuantityCut +from rail.creation.degraders.addRandom import AddColumnOfRandom @pytest.fixture diff --git a/tests/estimation/test_algos.py b/tests/estimation/test_algos.py index ceb171e6..eba19cb8 100644 --- a/tests/estimation/test_algos.py +++ b/tests/estimation/test_algos.py @@ -1,7 +1,7 @@ import numpy as np import scipy.special -from rail.core.algo_utils import one_algo +from rail.utils.testing_utils import one_algo from rail.core.stage import RailStage from rail.estimation.algos import random_gauss, train_z diff --git a/tests/estimation/test_classifier.py b/tests/estimation/test_classifier.py index 6b795455..a8627069 100644 --- a/tests/estimation/test_classifier.py +++ b/tests/estimation/test_classifier.py @@ -2,7 +2,7 @@ import numpy as np import pytest -from rail.core.utils import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.core.stage import RailStage from rail.core.data import QPHandle from rail.estimation.algos.uniform_binning import UniformBinningClassifier diff --git a/tests/estimation/test_summarizers.py b/tests/estimation/test_summarizers.py index a48d98ec..0814692c 100644 --- a/tests/estimation/test_summarizers.py +++ b/tests/estimation/test_summarizers.py @@ -2,7 +2,7 @@ from rail.core.data import QPHandle from rail.core.stage import RailStage -from rail.core.utils import RAILDIR +from rail.utils.path_utils import RAILDIR from rail.estimation.algos import naive_stack, point_est_hist, var_inf testdata = os.path.join(RAILDIR, "rail/examples_data/testdata/output_BPZ_lite.hdf5") diff --git a/tests/evaluation/test_evaluation.py b/tests/evaluation/test_evaluation.py index 6f8188ee..30d22c17 100644 --- a/tests/evaluation/test_evaluation.py +++ b/tests/evaluation/test_evaluation.py @@ -8,7 +8,7 @@ import rail.evaluation.metrics.pointestimates as pe from rail.core.data import QPHandle, TableHandle, QPOrTableHandle from rail.core.stage import RailStage -from rail.core.utils import find_rail_file +from rail.utils.path_utils import find_rail_file from rail.evaluation.evaluator import OldEvaluator from rail.evaluation.dist_to_dist_evaluator import DistToDistEvaluator from rail.evaluation.dist_to_point_evaluator import DistToPointEvaluator diff --git a/tests/utils/test_utils.py b/tests/utils/test_utils.py new file mode 100644 index 00000000..af79d9a6 --- /dev/null +++ b/tests/utils/test_utils.py @@ -0,0 +1,130 @@ +import os +import pickle +from types import GeneratorType +import pytest + +from rail.core.stage import RailStage +from rail.core.data import ( + DataHandle, + DataStore, + FitsHandle, + Hdf5Handle, + ModelHandle, + PqHandle, + QPHandle, + QPOrTableHandle, + TableHandle, +) + +from rail.utils.path_utils import RAILDIR +from rail.utils.path_utils import find_rail_file +from rail.tools.table_tools import ( + ColumnMapper, + RowSelector, + TableConverter, +) + + +def test_find_rail_file(): + afile = find_rail_file(os.path.join("examples_data", "testdata", "test_dc2_training_9816.pq")) + assert afile + with pytest.raises(ValueError): + _not_a_file = find_rail_file("not_a_file") + + +def test_util_stages(): + DS = RailStage.data_store + DS.clear() + datapath = os.path.join( + RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.pq" + ) + + data = DS.read_file("data", TableHandle, datapath) + + table_conv = TableConverter.make_stage(name="conv", output_format="numpyDict") + col_map = ColumnMapper.make_stage(name="col_map", columns={}) + row_sel = RowSelector.make_stage(name="row_sel", start=1, stop=15) + + with pytest.raises(KeyError) as _errinfo: + table_conv.get_handle("nope", allow_missing=False) + + _conv_data = table_conv(data) + mapped_data = col_map(data) + _sel_data = row_sel(mapped_data) + + row_sel_2 = RowSelector.make_stage(name="row_sel_2", start=1, stop=15) + row_sel_2.set_data("input", mapped_data.data) + handle = row_sel_2.get_handle("input") + + row_sel_3 = RowSelector.make_stage( + name="row_sel_3", input=handle.path, start=1, stop=15 + ) + row_sel_3.set_data("input", None, do_read=True) + +def test_set_data_nonexistent_file(): + """Create an instance of a child class of RailStage. Exercise the `set_data` + method and pass in a path to a nonexistent file. A `FileNotFound` exception + should be raised. + """ + + col_map = ColumnMapper.make_stage(name="col_map", columns={}) + with pytest.raises(FileNotFoundError) as err: + col_map.set_data("model", None, path="./bad_directory/no_file.py") + assert "Unable to find file" in err.context + + +def test_set_data_real_file(): + """Create an instance of a child class of RailStage. Exercise the `set_data` + method and pass in a path to model. The output of set_data should be `None`. + """ + DS = RailStage.data_store + DS.clear() + model_path = os.path.join( + RAILDIR, + "rail", + "examples_data", + "estimation_data", + "data", + "CWW_HDFN_prior.pkl", + ) + DS.add_data("model", None, ModelHandle, path=model_path) + + col_map = ColumnMapper.make_stage(name="col_map", columns={}) + + ret_val = col_map.set_data("model", None, path=model_path, do_read=False) + + assert ret_val is None + + +def test_data_hdf5_iter(): + DS = RailStage.data_store + DS.clear() + + datapath = os.path.join( + RAILDIR, "rail", "examples_data", "testdata", "test_dc2_training_9816.hdf5" + ) + + # data = DS.read_file('data', TableHandle, datapath) + th = Hdf5Handle("data", path=datapath) + x = th.iterator(groupname="photometry", chunk_size=1000) + + assert isinstance(x, GeneratorType) + for i, xx in enumerate(x): + assert xx[0] == i * 1000 + assert xx[1] - xx[0] <= 1000 + + _data = DS.read_file("input", TableHandle, datapath) + cm = ColumnMapper.make_stage( + input=datapath, + chunk_size=1000, + hdf5_groupname="photometry", + columns=dict(id="bob"), + ) + x = cm.input_iterator("input") + + assert isinstance(x, GeneratorType) + + for i, xx in enumerate(x): + assert xx[0] == i * 1000 + assert xx[1] - xx[0] <= 1000 +