From 970bf947cb0050935107105c59eea390b8ff27a3 Mon Sep 17 00:00:00 2001 From: schwarzam Date: Fri, 14 Jun 2024 13:45:39 -0300 Subject: [PATCH 1/2] Support to query params on http filesystem --- .gitignore | 5 ++++- .../loaders/hipscat/abstract_catalog_loader.py | 5 ++++- .../loaders/hipscat/hipscat_loading_config.py | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 36af9231..1bdc56e8 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,7 @@ _html/ .initialize_new_project.sh # large, unused fits files -point_map.fits \ No newline at end of file +point_map.fits + +# test notebook +dev/test.ipynb \ No newline at end of file diff --git a/src/lsdb/loaders/hipscat/abstract_catalog_loader.py b/src/lsdb/loaders/hipscat/abstract_catalog_loader.py index 171aacb7..5b5d32bf 100644 --- a/src/lsdb/loaders/hipscat/abstract_catalog_loader.py +++ b/src/lsdb/loaders/hipscat/abstract_catalog_loader.py @@ -64,7 +64,10 @@ def _get_paths_from_pixels( self, catalog: HCHealpixDataset, ordered_pixels: List[HealpixPixel] ) -> List[hc.io.FilePointer]: paths = hc.io.paths.pixel_catalog_files( - catalog.catalog_base_dir, ordered_pixels, storage_options=self.storage_options + catalog.catalog_base_dir, + ordered_pixels, + self.config.make_query_url_params(), + self.storage_options, ) return paths diff --git a/src/lsdb/loaders/hipscat/hipscat_loading_config.py b/src/lsdb/loaders/hipscat/hipscat_loading_config.py index 1e97317a..67af3ab2 100644 --- a/src/lsdb/loaders/hipscat/hipscat_loading_config.py +++ b/src/lsdb/loaders/hipscat/hipscat_loading_config.py @@ -47,3 +47,20 @@ def get_dtype_mapper(self) -> Callable | None: elif self.dtype_backend == "numpy_nullable": mapper = _arrow_dtype_mapping().get return mapper + + def make_query_url_params(self) -> dict: + """ + Generates a dictionary of URL parameters with `columns` and `filters` attributes. + """ + url_params = {} + + if self.columns and len(self.columns) > 0: + url_params["columns"] = self.columns + + if "filters" in self.kwargs: + url_params["filters"] = [] + for filtr in self.kwargs["filters"]: + # This is how hipscat expects the filters to add to the url + url_params["filters"].append(f"{filtr[0]}{filtr[1]}{filtr[2]}") + + return url_params From fdcf60da037a19a1de44a3f09fa717ef9f166514 Mon Sep 17 00:00:00 2001 From: Gustavo Schwarz Date: Fri, 14 Jun 2024 15:26:42 -0300 Subject: [PATCH 2/2] Update src/lsdb/loaders/hipscat/abstract_catalog_loader.py Co-authored-by: Melissa DeLucchi <113376043+delucchi-cmu@users.noreply.github.com> --- src/lsdb/loaders/hipscat/abstract_catalog_loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lsdb/loaders/hipscat/abstract_catalog_loader.py b/src/lsdb/loaders/hipscat/abstract_catalog_loader.py index 5b5d32bf..af0c9c2a 100644 --- a/src/lsdb/loaders/hipscat/abstract_catalog_loader.py +++ b/src/lsdb/loaders/hipscat/abstract_catalog_loader.py @@ -67,7 +67,7 @@ def _get_paths_from_pixels( catalog.catalog_base_dir, ordered_pixels, self.config.make_query_url_params(), - self.storage_options, + storage_options=self.storage_options, ) return paths