Merge pull request #55 from LSSTDESC/user/olynn/bpz-cli

Add bpz-demo-data option to get-data command

src/rail/cli/commands.py

@@ -68,6 +68,7 @@ def info(**kwargs):
 
 
 @cli.command()
+@options.bpz_demo_data()
 @options.verbose_download()
 def get_data(verbose, **kwargs):
     """Downloads data from NERSC (if not already found)"""

src/rail/cli/options.py

@@ -6,6 +6,7 @@
 
 __all__ = [
     "clear_output",
+    "bpz_demo_data",
     "dry_run",
     "outdir",
     "from_source",
@@ -54,7 +55,7 @@ def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
     def __call__(self, *args: Any, **kwargs: Any) -> Any:
         return self._partial(*args, **kwargs)
 
-    
+
 class PartialArgument:
     """Wraps click.argument with partial arguments for convenient reuse"""
 
@@ -147,7 +148,6 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any:
     help="Skip files",
 )
 
-
 inputs = PartialArgument(
     "inputs",
     nargs=-1
@@ -159,3 +159,9 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any:
     help="Verbose output when downloading",
     is_flag=True
 )
+
+bpz_demo_data = PartialOption(
+    "--bpz-demo-data",
+    help="Download data that is explicitly only for use in the bpz demo and nowhere else (it is dummy data that will not make sense)",
+    is_flag=True
+)

src/rail/cli/scripts.py

@@ -6,6 +6,7 @@
 import rail.stages
 from rail.core import RailEnv
 from rail.cli.options import GitMode
+from rail.core.utils import RAILDIR
 
 
 def render_nb(outdir, clear_output, dry_run, inputs, skip, **kwargs):
@@ -141,19 +142,35 @@ def info(**kwargs):
         print("======= Printing RAIL stages ==============")
         RailEnv.print_rail_stage_dict()
         print("\n\n")
-
 
-def get_data(verbose, **kwargs):  # pragma: no cover
 
-    data_files = [
-        {
-            "local_path": "src/rail/examples_data/goldenspike_data/data/base_catalog.pq",
-            "remote_path": "https://portal.nersc.gov/cfs/lsst/PZ/base_catalog.pq"
-        }
-    ]
-
-    for data_file in data_files:
-        if verbose:
-            print(f'Check file: {data_file["local_path"]}')
-        if not os.path.exists(data_file["local_path"]):
-            os.system(f'curl -o {data_file["local_path"]} {data_file["remote_path"]} --create-dirs')
+def get_data(verbose, **kwargs):  # pragma: no cover
+    if kwargs.get("bpz_demo_data"):
+        # The bpz demo data is quarantined into its own flag, as it contains some 
+        # non-physical features that would add systematics if run on any real data.
+        # This data should NOT be used for any science with real data!
+        bpz_local_abs_path = os.path.join(
+            RAILDIR, "rail/examples_data/estimation_data/data/nonphysical_dc2_templates.tar"
+        )
+        bpz_remote_path = "https://portal.nersc.gov/cfs/lsst/PZ/nonphysical_dc2_templates.tar"
+        print(f"Check for bpz demo data: {bpz_local_abs_path}")
+        if not os.path.exists(bpz_local_abs_path):
+            os.system(f"curl -o {bpz_local_abs_path} {bpz_remote_path} --create-dirs")
+            print("Downloaded bpz demo data.")
+        else:
+            print("Already have bpz demo data.")
+        print("\n(Note: you can run get-data without the bpz-demo-data flag to download standard data.)")
+
+    else:
+        data_files = [
+            {
+                "local_path": "rail/examples_data/goldenspike_data/data/base_catalog.pq",
+                "remote_path": "https://portal.nersc.gov/cfs/lsst/PZ/base_catalog.pq",
+            }
+        ]
+        for data_file in data_files:
+            local_abs_path = os.path.join(RAILDIR, data_file["local_path"])
+            if verbose:
+                print(f"Check file exists: {local_abs_path} ({os.path.exists(local_abs_path)})")
+            if not os.path.exists(local_abs_path):
+                os.system(f'curl -o {local_abs_path} {data_file["remote_path"]} --create-dirs')