From cac5c9c951abf2c672075eb950382876b19ac808 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Fri, 8 Nov 2024 14:42:36 -0700 Subject: [PATCH 01/10] Refactor `search_and_download` into multiple functions This is groundwork for developing a more flexible API. Instead of always just downloading anything that matches search results and then reading it, users will be able to individually: * Search for data * Download data * Read data --- src/nsidc/iceflow/api.py | 6 +-- src/nsidc/iceflow/data/fetch.py | 77 +++++++++++++++++++++----------- src/nsidc/iceflow/data/models.py | 10 +++++ 3 files changed, 64 insertions(+), 29 deletions(-) diff --git a/src/nsidc/iceflow/api.py b/src/nsidc/iceflow/api.py index 25e353c..f3cd0e3 100644 --- a/src/nsidc/iceflow/api.py +++ b/src/nsidc/iceflow/api.py @@ -29,8 +29,7 @@ def _df_for_one_dataset( output_itrf: str | None, ) -> IceflowDataFrame: results = search_and_download( - short_name=dataset.short_name, - version=dataset.version, + dataset=dataset, bounding_box=bounding_box, temporal=temporal, output_dir=output_dir, @@ -120,8 +119,7 @@ def create_iceflow_parquet( for dataset in dataset_search_params.datasets: results = search_and_download( - short_name=dataset.short_name, - version=dataset.version, + dataset=dataset, temporal=dataset_search_params.temporal, bounding_box=dataset_search_params.bounding_box, output_dir=output_dir, diff --git a/src/nsidc/iceflow/data/fetch.py b/src/nsidc/iceflow/data/fetch.py index da05eeb..5742426 100644 --- a/src/nsidc/iceflow/data/fetch.py +++ b/src/nsidc/iceflow/data/fetch.py @@ -6,32 +6,25 @@ import earthaccess from loguru import logger -from nsidc.iceflow.data.models import BoundingBox +from nsidc.iceflow.data.models import BoundingBox, Dataset, IceflowSearchResult -def search_and_download( +def _find_iceflow_data( *, - version: str, - short_name: str, + dataset: Dataset, bounding_box: BoundingBox, temporal: tuple[dt.datetime | dt.date, dt.datetime | dt.date], - output_dir: Path, -) -> list[Path]: - """Search and download data. - - Wraps EDL auth and CMR search using `earthaccess`. - - Data matching the given parameters are downloaded to a subfolder of the - given `output_dir` named after the `short_name`. - """ +) -> IceflowSearchResult: earthaccess.login() - ctx_string = f"{short_name=} {version=} with {bounding_box=} {temporal=}" + ctx_string = ( + f"{dataset.short_name=} {dataset.version=} with {bounding_box=} {temporal=}" + ) try: - results = earthaccess.search_data( - short_name=short_name, - version=version, + granules_list = earthaccess.search_data( + short_name=dataset.short_name, + version=dataset.version, bounding_box=( bounding_box.lower_left_lon, bounding_box.lower_left_lat, @@ -42,27 +35,61 @@ def search_and_download( ) except IndexError: # There's no data matching the given parameters. - logger.error(f"Found no results for {ctx_string}") - return [] + granules_list = [] - num_results = len(results) + num_results = len(granules_list) if not num_results: logger.error(f"Found no results for {ctx_string}") - return [] + granules_list = [] + + iceflow_search_result = IceflowSearchResult(dataset=dataset, granules=granules_list) + return iceflow_search_result + +def _download_iceflow_results( + *, iceflow_results: IceflowSearchResult, output_dir: Path +) -> list[Path]: # short_name based subdir for data. - output_subdir = output_dir / short_name + output_subdir = output_dir / iceflow_results.dataset.short_name logger.info( - f"Found {num_results} granules for {ctx_string}." - f" Downloading to {output_subdir}." + f"Downloading {len(iceflow_results.granules)} granules" f" to {output_subdir}." ) output_subdir.mkdir(exist_ok=True) - downloaded_files = earthaccess.download(results, str(output_subdir)) + downloaded_files = earthaccess.download( + iceflow_results.granules, str(output_subdir) + ) downloaded_filepaths = [Path(filepath_str) for filepath_str in downloaded_files] # There may be duplicate filepaths returned by earthaccess because of data # existing both in the cloud and on ECS. downloaded_filepaths = list(set(downloaded_filepaths)) return downloaded_filepaths + + +def search_and_download( + *, + dataset: Dataset, + bounding_box: BoundingBox, + temporal: tuple[dt.datetime | dt.date, dt.datetime | dt.date], + output_dir: Path, +) -> list[Path]: + """Search and download data. + + Wraps EDL auth and CMR search using `earthaccess`. + + Data matching the given parameters are downloaded to a subfolder of the + given `output_dir` named after the `short_name`. + """ + iceflow_results = _find_iceflow_data( + dataset=dataset, + bounding_box=bounding_box, + temporal=temporal, + ) + + downloaded_filepaths = _download_iceflow_results( + iceflow_results=iceflow_results, output_dir=output_dir + ) + + return downloaded_filepaths diff --git a/src/nsidc/iceflow/data/models.py b/src/nsidc/iceflow/data/models.py index 20125f0..ec1630d 100644 --- a/src/nsidc/iceflow/data/models.py +++ b/src/nsidc/iceflow/data/models.py @@ -5,6 +5,7 @@ import pandera as pa import pydantic +from earthaccess.results import DataGranule from pandera.typing import DataFrame, Index, Series from nsidc.iceflow.itrf import ITRF_REGEX @@ -235,3 +236,12 @@ class DatasetSearchParameters(pydantic.BaseModel): datasets: list[Dataset] bounding_box: BoundingBox temporal: tuple[dt.datetime | dt.date, dt.datetime | dt.date] + + +class IceflowSearchResult(pydantic.BaseModel): + dataset: Dataset + granules: list[DataGranule] + + # Pydantic can't infer what `DataGranule` is. This ignores validation for + # this that type + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) From 1c2af83bb4ded452d6353d2dbfa5d1ff51f3aa4f Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Fri, 8 Nov 2024 14:57:58 -0700 Subject: [PATCH 02/10] Add function to find iceflow data based on `DatasetSearchParameters` And improve kwarg/variable naming --- src/nsidc/iceflow/data/fetch.py | 44 +++++++++++++++++++++++++------- src/nsidc/iceflow/data/models.py | 3 +++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/src/nsidc/iceflow/data/fetch.py b/src/nsidc/iceflow/data/fetch.py index 5742426..a01c8e1 100644 --- a/src/nsidc/iceflow/data/fetch.py +++ b/src/nsidc/iceflow/data/fetch.py @@ -6,7 +6,13 @@ import earthaccess from loguru import logger -from nsidc.iceflow.data.models import BoundingBox, Dataset, IceflowSearchResult +from nsidc.iceflow.data.models import ( + BoundingBox, + Dataset, + DatasetSearchParameters, + IceflowSearchResult, + IceflowSearchResults, +) def _find_iceflow_data( @@ -47,18 +53,21 @@ def _find_iceflow_data( return iceflow_search_result -def _download_iceflow_results( - *, iceflow_results: IceflowSearchResult, output_dir: Path +def _download_iceflow_search_result( + *, + iceflow_search_result: IceflowSearchResult, + output_dir: Path, ) -> list[Path]: # short_name based subdir for data. - output_subdir = output_dir / iceflow_results.dataset.short_name + output_subdir = output_dir / iceflow_search_result.dataset.short_name logger.info( - f"Downloading {len(iceflow_results.granules)} granules" f" to {output_subdir}." + f"Downloading {len(iceflow_search_result.granules)} granules" + f" to {output_subdir}." ) output_subdir.mkdir(exist_ok=True) downloaded_files = earthaccess.download( - iceflow_results.granules, str(output_subdir) + iceflow_search_result.granules, str(output_subdir) ) downloaded_filepaths = [Path(filepath_str) for filepath_str in downloaded_files] # There may be duplicate filepaths returned by earthaccess because of data @@ -82,14 +91,31 @@ def search_and_download( Data matching the given parameters are downloaded to a subfolder of the given `output_dir` named after the `short_name`. """ - iceflow_results = _find_iceflow_data( + iceflow_search_result = _find_iceflow_data( dataset=dataset, bounding_box=bounding_box, temporal=temporal, ) - downloaded_filepaths = _download_iceflow_results( - iceflow_results=iceflow_results, output_dir=output_dir + downloaded_filepaths = _download_iceflow_search_result( + iceflow_search_result=iceflow_search_result, + output_dir=output_dir, ) return downloaded_filepaths + + +def find_iceflow_data( + *, + dataset_search_params: DatasetSearchParameters, +) -> IceflowSearchResults: + iceflow_search_results = [] + for dataset in dataset_search_params.datasets: + iceflow_search_result = _find_iceflow_data( + dataset=dataset, + bounding_box=dataset_search_params.bounding_box, + temporal=dataset_search_params.temporal, + ) + iceflow_search_results.append(iceflow_search_result) + + return iceflow_search_results diff --git a/src/nsidc/iceflow/data/models.py b/src/nsidc/iceflow/data/models.py index ec1630d..0ec8208 100644 --- a/src/nsidc/iceflow/data/models.py +++ b/src/nsidc/iceflow/data/models.py @@ -245,3 +245,6 @@ class IceflowSearchResult(pydantic.BaseModel): # Pydantic can't infer what `DataGranule` is. This ignores validation for # this that type model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + + +IceflowSearchResults = list[IceflowSearchResult] From 3efd690132d9fb20c45cd8b8bebf5602dbc7cf22 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Fri, 8 Nov 2024 15:11:36 -0700 Subject: [PATCH 03/10] Search results are downloaded to subdir based on short_name and version --- src/nsidc/iceflow/data/fetch.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nsidc/iceflow/data/fetch.py b/src/nsidc/iceflow/data/fetch.py index a01c8e1..3bfa934 100644 --- a/src/nsidc/iceflow/data/fetch.py +++ b/src/nsidc/iceflow/data/fetch.py @@ -58,8 +58,10 @@ def _download_iceflow_search_result( iceflow_search_result: IceflowSearchResult, output_dir: Path, ) -> list[Path]: - # short_name based subdir for data. - output_subdir = output_dir / iceflow_search_result.dataset.short_name + # short_name and version-based subdir for data. + subdir_name = f"{iceflow_search_result.dataset.short_name}_{iceflow_search_result.dataset.version}" + output_subdir = output_dir / subdir_name + logger.info( f"Downloading {len(iceflow_search_result.granules)} granules" f" to {output_subdir}." @@ -89,7 +91,7 @@ def search_and_download( Wraps EDL auth and CMR search using `earthaccess`. Data matching the given parameters are downloaded to a subfolder of the - given `output_dir` named after the `short_name`. + given `output_dir` named after the `short_name` and `version`. """ iceflow_search_result = _find_iceflow_data( dataset=dataset, From 6b15bef547cc68c7a0e4d53279e0bfb082dbe2df Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Fri, 8 Nov 2024 16:41:06 -0700 Subject: [PATCH 04/10] Search for data across all supported datasets by default --- src/nsidc/iceflow/data/__init__.py | 15 ++++++++------- src/nsidc/iceflow/data/models.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/nsidc/iceflow/data/__init__.py b/src/nsidc/iceflow/data/__init__.py index 3eb4547..7855f29 100644 --- a/src/nsidc/iceflow/data/__init__.py +++ b/src/nsidc/iceflow/data/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from nsidc.iceflow.data.models import ( + ALL_DATASETS, BLATM1BDataset, Dataset, GLAH06Dataset, @@ -8,11 +9,11 @@ ILVIS2Dataset, ) -ALL_DATASETS: list[Dataset] = [ - ILATM1BDataset(version="1"), - ILATM1BDataset(version="2"), - BLATM1BDataset(version="1"), - ILVIS2Dataset(version="1"), - ILVIS2Dataset(version="2"), - GLAH06Dataset(), +__all__ = [ + "ALL_DATASETS", + "BLATM1BDataset", + "Dataset", + "GLAH06Dataset", + "ILATM1BDataset", + "ILVIS2Dataset", ] diff --git a/src/nsidc/iceflow/data/models.py b/src/nsidc/iceflow/data/models.py index 0ec8208..1eb9ffb 100644 --- a/src/nsidc/iceflow/data/models.py +++ b/src/nsidc/iceflow/data/models.py @@ -232,8 +232,18 @@ class BoundingBox(pydantic.BaseModel): upper_right_lat: float +ALL_DATASETS: list[Dataset] = [ + ILATM1BDataset(version="1"), + ILATM1BDataset(version="2"), + BLATM1BDataset(version="1"), + ILVIS2Dataset(version="1"), + ILVIS2Dataset(version="2"), + GLAH06Dataset(), +] + + class DatasetSearchParameters(pydantic.BaseModel): - datasets: list[Dataset] + datasets: list[Dataset] = ALL_DATASETS bounding_box: BoundingBox temporal: tuple[dt.datetime | dt.date, dt.datetime | dt.date] From d0c7ae00e19c4d4a847317450f0e9e8eeb7d2ba1 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Fri, 8 Nov 2024 16:42:37 -0700 Subject: [PATCH 05/10] Update api to use newly created fetch/read funcs --- src/nsidc/iceflow/api.py | 129 ++++++++++++-------------------- src/nsidc/iceflow/data/fetch.py | 15 ++++ src/nsidc/iceflow/data/read.py | 54 ++++++------- 3 files changed, 93 insertions(+), 105 deletions(-) diff --git a/src/nsidc/iceflow/api.py b/src/nsidc/iceflow/api.py index f3cd0e3..e2e2163 100644 --- a/src/nsidc/iceflow/api.py +++ b/src/nsidc/iceflow/api.py @@ -1,56 +1,20 @@ from __future__ import annotations -import datetime as dt import shutil from pathlib import Path import dask.dataframe as dd -import pandas as pd from loguru import logger -from nsidc.iceflow.data.fetch import search_and_download +from nsidc.iceflow.data.fetch import download_iceflow_results, find_iceflow_data from nsidc.iceflow.data.models import ( - BoundingBox, - Dataset, DatasetSearchParameters, IceflowDataFrame, ) -from nsidc.iceflow.data.read import read_data +from nsidc.iceflow.data.read import read_iceflow_datafiles from nsidc.iceflow.itrf.converter import transform_itrf -def _df_for_one_dataset( - *, - dataset: Dataset, - bounding_box: BoundingBox, - temporal: tuple[dt.datetime | dt.date, dt.datetime | dt.date], - output_dir: Path, - # TODO: also add option for target epoch!! - output_itrf: str | None, -) -> IceflowDataFrame: - results = search_and_download( - dataset=dataset, - bounding_box=bounding_box, - temporal=temporal, - output_dir=output_dir, - ) - - all_dfs = [] - for result in results: - data_df = read_data(dataset, result) - all_dfs.append(data_df) - - complete_df = IceflowDataFrame(pd.concat(all_dfs)) - - if output_itrf is not None: - complete_df = transform_itrf( - data=complete_df, - target_itrf=output_itrf, - ) - - return complete_df - - def fetch_iceflow_df( *, dataset_search_params: DatasetSearchParameters, @@ -70,20 +34,24 @@ def fetch_iceflow_df( format. """ - dfs = [] - for dataset in dataset_search_params.datasets: - result = _df_for_one_dataset( - dataset=dataset, - temporal=dataset_search_params.temporal, - bounding_box=dataset_search_params.bounding_box, - output_dir=output_dir, - output_itrf=output_itrf, - ) - dfs.append(result) + iceflow_search_reuslts = find_iceflow_data( + dataset_search_params=dataset_search_params, + ) + + downloaded_files = download_iceflow_results( + iceflow_search_results=iceflow_search_reuslts, + output_dir=output_dir, + ) + + iceflow_df = read_iceflow_datafiles(downloaded_files) - complete_df = IceflowDataFrame(pd.concat(dfs)) + if output_itrf is not None: + iceflow_df = transform_itrf( + data=iceflow_df, + target_itrf=output_itrf, + ) - return complete_df + return iceflow_df def create_iceflow_parquet( @@ -117,41 +85,42 @@ def create_iceflow_parquet( "An iceflow parquet file already exists. Use `overwrite=True` to overwrite." ) - for dataset in dataset_search_params.datasets: - results = search_and_download( - dataset=dataset, - temporal=dataset_search_params.temporal, - bounding_box=dataset_search_params.bounding_box, + iceflow_search_results = find_iceflow_data( + dataset_search_params=dataset_search_params, + ) + + for iceflow_search_result in iceflow_search_results: + downloaded_files = download_iceflow_results( + iceflow_search_results=[iceflow_search_result], output_dir=output_dir, ) - for result in results: - data_df = read_data(dataset, result) - df = IceflowDataFrame(data_df) + iceflow_df = read_iceflow_datafiles(downloaded_files) - df = transform_itrf( - data=df, - target_itrf=target_itrf, - target_epoch=target_epoch, - ) + iceflow_df = transform_itrf( + data=iceflow_df, + target_itrf=target_itrf, + target_epoch=target_epoch, + ) - # Add a string col w/ dataset name and version. - df["dataset"] = [f"{dataset.short_name}v{dataset.version}"] * len( - df.latitude + # Add a string col w/ dataset name and version. + dataset = iceflow_search_result.dataset + iceflow_df["dataset"] = [f"{dataset.short_name}v{dataset.version}"] * len( + iceflow_df.latitude + ) + common_columns = ["latitude", "longitude", "elevation", "dataset"] + common_dask_df = dd.from_pandas(iceflow_df[common_columns]) # type: ignore[attr-defined] + if output_subdir.exists(): + dd.to_parquet( # type: ignore[attr-defined] + df=common_dask_df, + path=output_subdir, + append=True, + ignore_divisions=True, + ) + else: + dd.to_parquet( # type: ignore[attr-defined] + df=common_dask_df, + path=output_subdir, ) - common_columns = ["latitude", "longitude", "elevation", "dataset"] - common_dask_df = dd.from_pandas(df[common_columns]) # type: ignore[attr-defined] - if output_subdir.exists(): - dd.to_parquet( # type: ignore[attr-defined] - df=common_dask_df, - path=output_subdir, - append=True, - ignore_divisions=True, - ) - else: - dd.to_parquet( # type: ignore[attr-defined] - df=common_dask_df, - path=output_subdir, - ) return output_subdir diff --git a/src/nsidc/iceflow/data/fetch.py b/src/nsidc/iceflow/data/fetch.py index 3bfa934..e17eee4 100644 --- a/src/nsidc/iceflow/data/fetch.py +++ b/src/nsidc/iceflow/data/fetch.py @@ -121,3 +121,18 @@ def find_iceflow_data( iceflow_search_results.append(iceflow_search_result) return iceflow_search_results + + +def download_iceflow_results( + iceflow_search_results: IceflowSearchResults, + output_dir: Path, +) -> list[Path]: + all_downloaded_files = [] + for iceflow_search_result in iceflow_search_results: + downloaded_filepaths = _download_iceflow_search_result( + iceflow_search_result=iceflow_search_result, + output_dir=output_dir, + ) + all_downloaded_files.extend(downloaded_filepaths) + + return all_downloaded_files diff --git a/src/nsidc/iceflow/data/read.py b/src/nsidc/iceflow/data/read.py index 8440262..06614e7 100644 --- a/src/nsidc/iceflow/data/read.py +++ b/src/nsidc/iceflow/data/read.py @@ -1,41 +1,45 @@ from __future__ import annotations -import functools from pathlib import Path +import pandas as pd + from nsidc.iceflow.data.atm1b import atm1b_data from nsidc.iceflow.data.glah06 import glah06_data from nsidc.iceflow.data.ilvis2 import ilvis2_data from nsidc.iceflow.data.models import ( ATM1BDataFrame, - ATM1BDataset, - Dataset, GLAH06DataFrame, - GLAH06Dataset, IceflowDataFrame, ILVIS2DataFrame, - ILVIS2Dataset, ) -@functools.singledispatch -def read_data( - dataset: Dataset, _filepath: Path +def read_iceflow_datafile( + filepath: Path, ) -> IceflowDataFrame | ATM1BDataFrame | ILVIS2DataFrame | GLAH06DataFrame: - msg = f"{dataset=} not recognized." - raise RuntimeError(msg) - - -@read_data.register -def _(_dataset: ATM1BDataset, filepath: Path) -> ATM1BDataFrame: - return atm1b_data(filepath) - - -@read_data.register -def _(_dataset: ILVIS2Dataset, filepath: Path) -> ILVIS2DataFrame: - return ilvis2_data(filepath) - - -@read_data.register -def _(_dataset: GLAH06Dataset, filepath: Path) -> GLAH06DataFrame: - return glah06_data(filepath) + # iceflow data are expected to exist in a directory named like + # `{short_name}_{version}` + dataset_subdir = filepath.parent.name + short_name, _version = dataset_subdir.split("_") + + if short_name in ["ILATM1B", "BLATM1B"]: + return atm1b_data(filepath) + elif short_name == "ILVIS2": + return ilvis2_data(filepath) + elif short_name == "GLAH06": + return glah06_data(filepath) + else: + err_msg = f"Unrecognized dataset {short_name=} extracted from {filepath.parent}" + raise RuntimeError(err_msg) + + +def read_iceflow_datafiles(filepaths: list[Path]) -> IceflowDataFrame: + all_dfs = [] + for filepath in filepaths: + df = read_iceflow_datafile(filepath) + all_dfs.append(df) + + complete_df = IceflowDataFrame(pd.concat(all_dfs)) + + return complete_df From 1880fd2953cfb1cf93ddf2beda2bacfa13a1cd11 Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Fri, 8 Nov 2024 16:46:25 -0700 Subject: [PATCH 06/10] Remove OBE `search_and_download` --- src/nsidc/iceflow/data/fetch.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/src/nsidc/iceflow/data/fetch.py b/src/nsidc/iceflow/data/fetch.py index e17eee4..64f2817 100644 --- a/src/nsidc/iceflow/data/fetch.py +++ b/src/nsidc/iceflow/data/fetch.py @@ -79,34 +79,6 @@ def _download_iceflow_search_result( return downloaded_filepaths -def search_and_download( - *, - dataset: Dataset, - bounding_box: BoundingBox, - temporal: tuple[dt.datetime | dt.date, dt.datetime | dt.date], - output_dir: Path, -) -> list[Path]: - """Search and download data. - - Wraps EDL auth and CMR search using `earthaccess`. - - Data matching the given parameters are downloaded to a subfolder of the - given `output_dir` named after the `short_name` and `version`. - """ - iceflow_search_result = _find_iceflow_data( - dataset=dataset, - bounding_box=bounding_box, - temporal=temporal, - ) - - downloaded_filepaths = _download_iceflow_search_result( - iceflow_search_result=iceflow_search_result, - output_dir=output_dir, - ) - - return downloaded_filepaths - - def find_iceflow_data( *, dataset_search_params: DatasetSearchParameters, From ae44677c4602f96666a3abbe39f95832b11e5cee Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Mon, 9 Dec 2024 10:33:16 -0700 Subject: [PATCH 07/10] Improve API around finding, downloading, and accessing data. Also improve the documentation!! --- docs/getting-started.md | 111 +++++++++++++ docs/index.md | 1 + notebooks/iceflow-example.ipynb | 59 ++++++- notebooks/iceflow-with-icepyx.ipynb | 248 +++++++--------------------- src/nsidc/iceflow/__init__.py | 43 ++++- src/nsidc/iceflow/api.py | 57 ++++--- src/nsidc/iceflow/data/fetch.py | 5 +- src/nsidc/iceflow/data/models.py | 4 + tests/integration/test_e2e.py | 50 +++++- 9 files changed, 352 insertions(+), 226 deletions(-) create mode 100644 docs/getting-started.md diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..be47a41 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,111 @@ +# Getting started with iceflow + +## Jupyter Notebooks + +[Jupyter notebooks](https://docs.jupyter.org/en/latest/) provide executable +examples of how to use `iceflow`: + +- [iceflow-example.ipynb](./iceflow-example.ipynb) provides an example of how to + search for, download, and interact with `ILATM1B v1` data for a small area of + interest. This notebook also illustrates how to perform + [ITRF](https://itrf.ign.fr/) transformations to facilitate comparisons across + datasets. To learn more about ITRF transformations, see the + [corrections.ipynb](https://github.com/nsidc/NSIDC-Data-Tutorials/blob/main/notebooks/iceflow/corrections.ipynb) + notebook in the + [NSIDC-Data-Tutorials GitHub repository](https://github.com/nsidc/NSIDC-Data-Tutorials). + +- [iceflow-with-icepyx.ipynb](./iceflow-with-icepyx.ipynb) shows how to search + for, download, and interact with a large amount of data across many datasets + supported by `iceflow`. It also illustrates how to utilize + [icepyx](https://icepyx.readthedocs.io/en/latest/) to find and access ICESat-2 + data. Finally, the notebook provides a simple time-series analysis for + elevation change over an area of interest across `iceflow` supported datasets + and ICESat-2. + +## API overview + +`iceflow` provides a simple API for finding, downloading, and accessing +iceflow-supported datasets. + +### Finding data + +To find `iceflow`-supported data for an area of interest and timeframe, use +[`find_iceflow_data`](nsidc.iceflow.find_iceflow_data): + +``` +import datetime as dt + +from nsidc.iceflow import ( + find_iceflow_data, + DatasetSearchParameters, + BoundingBox, +) + + +search_results = find_iceflow_data( + dataset_search_params=DatasetSearchParameters( + bounding_box=BoundingBox(lower_left_lon=-103.125559, lower_left_lat=-75.180563, upper_right_lon=-102.677327, upper_right_lat=-74.798063), + temporal=(dt.date(2009, 11, 1), dt.date(2009, 12, 31)), + ), +) +``` + +### Downloading data + +Once search results have been found, download data with +[`download_iceflow_results`](nsidc.iceflow.download_iceflow_results): + +``` +from pathlib import Path +from nsidc.iceflow import download_iceflow_results + +downloaded_filepaths = download_iceflow_results( + iceflow_search_result=iceflow_search_result, + output_dir=Path("/path/to/data/dir/"), +) +``` + +### Accessing data + +Iceflow data can be very large, and fitting it into memory can be a challenge! +To facilitate analysis of iceflow data, +[`make_iceflow_parquet`](nsidc.iceflow.make_iceflow_parquet) provides a +mechanism to create a [parquet](https://parquet.apache.org/docs/overview/) +datastore that can be used alongside [dask](https://www.dask.org/): + +``` +import dask.dataframe as dd +from nsidc.iceflow import make_iceflow_parquet + +parquet_path = make_iceflow_parquet( + data_dir=Path("/path/to/data/dir/"), + target_itrf="ITRF2014", +) +df = dd.read_parquet(parquet_path) +``` + +Note that `make_iceflow_parquet` creates a parquet datastore for the data in the +provided `data_dir` with the data transformed into a common +[ITRF](https://itrf.ign.fr/) to facilitate analysis. Only datetime, lat, lon, +and elevation fields are preserved in the parquet datastore. + +To access and analyze the full data record in the source files, use +[`read_iceflow_datafiles`](nsidc.iceflow.read_iceflow_datafiles): + +``` +from nsidc.iceflow import read_iceflow_datafiles + +# Read all of the data in the source files - not just lat/lon/elev. +df = read_iceflow_datafiles(downloaded_files) + +# Optional: transform lat/lon/elev to common ITRF: +from nsidc.iceflow import transform_itrf +df = transform_itrf( + data=df, + target_itrf="ITRF2014", +) +``` + +Note that `read_iceflow_datafiles` reads all of the data from the given +filepaths. This could be a large amount of data, and could cause your program to +crash if physical memory limits are exceeded. diff --git a/docs/index.md b/docs/index.md index 7349773..34e380c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,6 +5,7 @@ :hidden: :caption: content +getting-started.md iceflow-example.ipynb iceflow-with-icepyx.ipynb api/nsidc diff --git a/notebooks/iceflow-example.ipynb b/notebooks/iceflow-example.ipynb index e88c7cf..76b01b9 100644 --- a/notebooks/iceflow-example.ipynb +++ b/notebooks/iceflow-example.ipynb @@ -33,9 +33,14 @@ "metadata": {}, "outputs": [], "source": [ - "from nsidc.iceflow.api import fetch_iceflow_df\n", - "from nsidc.iceflow.data.models import DatasetSearchParameters, BoundingBox, ILATM1BDataset\n", - "\n", + "from nsidc.iceflow import (\n", + " find_iceflow_data,\n", + " download_iceflow_results,\n", + " read_iceflow_datafiles,\n", + " DatasetSearchParameters,\n", + " BoundingBox,\n", + " ILATM1BDataset,\n", + ")\n", "from pathlib import Path\n", "import datetime as dt\n", "import matplotlib.pyplot as plt\n", @@ -80,25 +85,61 @@ "id": "ada043ae-65ed-4f22-ba19-dc9da6b8e0d2", "metadata": {}, "source": [ - "Now we use the `fetch_iceflow_df` function to search for and download data matching our search parameters. The output is a `pandas` DataFrame containing the matching data." + "Next, we use the `find_iceflow_data` function to search for data matching our search parameters. " ] }, { "cell_type": "code", "execution_count": null, - "id": "15c0633c-1812-45d7-a91f-36d627fdb3b5", + "id": "c440ba2d-0c4a-47ff-b41e-a7486709d3e0", "metadata": {}, "outputs": [], "source": [ - "iceflow_df = fetch_iceflow_df(\n", + "search_results = find_iceflow_data(\n", " dataset_search_params=DatasetSearchParameters(\n", " datasets=[atm1b_v1_dataset],\n", " bounding_box=BBOX,\n", " temporal=date_range,\n", " ),\n", - " output_dir=data_path,\n", - " output_itrf=None,\n", ")\n", + "len(search_results)" + ] + }, + { + "cell_type": "markdown", + "id": "752d5a6a-27ec-42cc-9505-923df8b69549", + "metadata": {}, + "source": [ + "Once we have found data matching our area of interest, we can download the results with `download_iceflow_results`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3b3b3bc-2d2e-49d3-bafc-3f9a4269ee57", + "metadata": {}, + "outputs": [], + "source": [ + "downloaded_files = download_iceflow_results(search_results, output_dir=data_path)\n", + "print(downloaded_files)" + ] + }, + { + "cell_type": "markdown", + "id": "1526b5bb-c08a-43d1-84b4-a207d9db924c", + "metadata": {}, + "source": [ + "Finally, we can read the data we just downloaded with `read_iceflow_datafiles`. The output is a `pandas` DataFrame containing the matching data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15c0633c-1812-45d7-a91f-36d627fdb3b5", + "metadata": {}, + "outputs": [], + "source": [ + "iceflow_df = read_iceflow_datafiles(downloaded_files)\n", "\n", "iceflow_df.head()" ] @@ -139,7 +180,7 @@ "metadata": {}, "outputs": [], "source": [ - "from nsidc.iceflow.itrf.converter import transform_itrf\n", + "from nsidc.iceflow import transform_itrf\n", "\n", "itrf2014_df = transform_itrf(data=iceflow_df, target_itrf=\"ITRF2014\")\n", "itrf2014_df.head()" diff --git a/notebooks/iceflow-with-icepyx.ipynb b/notebooks/iceflow-with-icepyx.ipynb index 4b10b90..1ba8d65 100644 --- a/notebooks/iceflow-with-icepyx.ipynb +++ b/notebooks/iceflow-with-icepyx.ipynb @@ -59,9 +59,14 @@ "import pandas as pd\n", "import xarray as xr\n", "\n", - "from nsidc.iceflow.api import fetch_iceflow_df, create_iceflow_parquet\n", - "from nsidc.iceflow.data import ALL_DATASETS\n", - "from nsidc.iceflow.data.models import DatasetSearchParameters, BoundingBox, IceflowDataFrame" + "from nsidc.iceflow import (\n", + " make_iceflow_parquet,\n", + " DatasetSearchParameters,\n", + " BoundingBox,\n", + " find_iceflow_data,\n", + " download_iceflow_results,\n", + " IceflowDataFrame\n", + ")" ] }, { @@ -103,55 +108,72 @@ }, { "cell_type": "markdown", - "id": "5ac0b6bf-41f5-44b7-9703-fce6c4b5e589", + "id": "420bf37c-c5cb-4339-bd26-ebb91ecb3973", "metadata": {}, "source": [ - "Next we use the `create_iceflow_parquet` function from the `iceflow` API. This function:\n", - "\n", - "* Finds data matching our `BBOX`, `DATE_RANGE`, and desired datasets\n", - "* Downloads and reads the data from the datasets' native formats\n", - "* Transforms all of the lat/lon/elev data into a target ITRF\n", - "* Writes out the lat/lon/elev data to a parquet dataset that can be be read by e.g., `dask` for further processing. \n", + "Next we will use the `find_iceflow_data` function from the `iceflow` API to find data matching our area of interest.\n", "\n", - "Writing data to a parquet dataset allows `dask` (which we will use later!) to read the chunks of data it needs to do calculations (e.g., `mean`) without needing to read all of the data into memory at once. This is important because `iceflow` can find many millions of data points for even small areas of interest!\n", - "\n", - "**Note: This next step may take a while, and will download data to your local disk.**" + "By default, `DatasetSearchParameters` will include all `iceflow` supported datasets, unless one or more are specified as a filter with the `datasets` kwarg. There may be warnings raised about there not being search results for specific datasets supported by `iceflow`." ] }, { "cell_type": "code", "execution_count": null, - "id": "713b93dd-d23b-422b-92dc-f9ed364a566c", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "cfda36320ddb49dc860724b8e9aa4812", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "COLLECTING RESULTS | : 0%| | 0/8 [00:00 Path: - """Create a parquet dataset containing the lat/lon/elev data matching the dataset search params. + """Create a parquet dataset containing the lat/lon/elev data in `data_dir`. - This function creates a parquet dataset that can be easily used alongside dask, - containing lat/lon/elev data. + This function creates a parquet dataset that can be easily used alongside + dask, containing lat/lon/elev data. Users who are interested in the full + data record with all fields provided by data in the `data_dir` should use + `read_iceflow_datafiles`. Note: this function writes a single `iceflow.parquet` to the output dir. This code does not currently support updates to the parquet after being @@ -75,27 +79,24 @@ def create_iceflow_parquet( need to move/remove the existing `iceflow.parquet` first (e.g., with the `overwrite=True` kwarg). """ - output_subdir = output_dir / "iceflow.parquet" - if output_subdir.exists(): + parquet_subdir = data_dir / "iceflow.parquet" + if parquet_subdir.exists(): if overwrite: logger.info("Removing existing iceflow.parquet") - shutil.rmtree(output_subdir) + shutil.rmtree(parquet_subdir) else: raise RuntimeError( "An iceflow parquet file already exists. Use `overwrite=True` to overwrite." ) - iceflow_search_results = find_iceflow_data( - dataset_search_params=dataset_search_params, - ) - - for iceflow_search_result in iceflow_search_results: - downloaded_files = download_iceflow_results( - iceflow_search_results=[iceflow_search_result], - output_dir=output_dir, - ) - - iceflow_df = read_iceflow_datafiles(downloaded_files) + all_subdirs = [ + data_dir / ds.subdir_name + for ds in ALL_DATASETS + if (data_dir / ds.subdir_name).is_dir() + ] + for subdir in all_subdirs: + iceflow_filepaths = [path for path in subdir.iterdir() if path.is_file()] + iceflow_df = read_iceflow_datafiles(iceflow_filepaths) iceflow_df = transform_itrf( data=iceflow_df, @@ -104,23 +105,21 @@ def create_iceflow_parquet( ) # Add a string col w/ dataset name and version. - dataset = iceflow_search_result.dataset - iceflow_df["dataset"] = [f"{dataset.short_name}v{dataset.version}"] * len( - iceflow_df.latitude - ) + short_name, version = subdir.name.split("_") + iceflow_df["dataset"] = [f"{short_name}v{version}"] * len(iceflow_df.latitude) common_columns = ["latitude", "longitude", "elevation", "dataset"] common_dask_df = dd.from_pandas(iceflow_df[common_columns]) # type: ignore[attr-defined] - if output_subdir.exists(): + if parquet_subdir.exists(): dd.to_parquet( # type: ignore[attr-defined] df=common_dask_df, - path=output_subdir, + path=parquet_subdir, append=True, ignore_divisions=True, ) else: dd.to_parquet( # type: ignore[attr-defined] df=common_dask_df, - path=output_subdir, + path=parquet_subdir, ) - return output_subdir + return parquet_subdir diff --git a/src/nsidc/iceflow/data/fetch.py b/src/nsidc/iceflow/data/fetch.py index 64f2817..ee64d9b 100644 --- a/src/nsidc/iceflow/data/fetch.py +++ b/src/nsidc/iceflow/data/fetch.py @@ -46,7 +46,7 @@ def _find_iceflow_data( num_results = len(granules_list) if not num_results: - logger.error(f"Found no results for {ctx_string}") + logger.warning(f"Found no results for {ctx_string}") granules_list = [] iceflow_search_result = IceflowSearchResult(dataset=dataset, granules=granules_list) @@ -58,6 +58,9 @@ def _download_iceflow_search_result( iceflow_search_result: IceflowSearchResult, output_dir: Path, ) -> list[Path]: + # No granules found for this search result object. + if not iceflow_search_result.granules: + return [] # short_name and version-based subdir for data. subdir_name = f"{iceflow_search_result.dataset.short_name}_{iceflow_search_result.dataset.version}" output_subdir = output_dir / subdir_name diff --git a/src/nsidc/iceflow/data/models.py b/src/nsidc/iceflow/data/models.py index 1eb9ffb..d101b3d 100644 --- a/src/nsidc/iceflow/data/models.py +++ b/src/nsidc/iceflow/data/models.py @@ -196,6 +196,10 @@ class Dataset(pydantic.BaseModel): short_name: DatasetShortName version: str + @property + def subdir_name(self): + return f"{self.short_name}_{self.version}" + class ATM1BDataset(Dataset): short_name: ATM1BShortName diff --git a/tests/integration/test_e2e.py b/tests/integration/test_e2e.py index 75f09b5..83bd93a 100644 --- a/tests/integration/test_e2e.py +++ b/tests/integration/test_e2e.py @@ -11,11 +11,17 @@ from __future__ import annotations import datetime as dt +from pathlib import Path import dask.dataframe as dd import pandas as pd -from nsidc.iceflow.api import create_iceflow_parquet, fetch_iceflow_df +from nsidc.iceflow import ( + download_iceflow_results, + find_iceflow_data, + make_iceflow_parquet, +) +from nsidc.iceflow.api import fetch_iceflow_df from nsidc.iceflow.data.models import ( BLATM1BDataset, BoundingBox, @@ -147,6 +153,46 @@ def test_glah06(tmp_path): assert (results.ITRF == "ITRF2008").all() +def _create_iceflow_parquet( + *, + dataset_search_params: DatasetSearchParameters, + output_dir: Path, + target_itrf: str, + overwrite: bool = False, + target_epoch: str | None = None, +) -> Path: + """Create a parquet dataset containing the lat/lon/elev data matching the dataset search params. + + This function creates a parquet dataset that can be easily used alongside dask, + containing lat/lon/elev data. + + Note: this function writes a single `iceflow.parquet` to the output + dir. This code does not currently support updates to the parquet after being + written. This is intended to help facilitate analysis of a specific area + over time. If an existing `iceflow.parquet` exists and the user wants to + create a new `iceflow.parquet` for a different area or timespan, they will + need to move/remove the existing `iceflow.parquet` first (e.g., with the + `overwrite=True` kwarg). + """ + iceflow_search_results = find_iceflow_data( + dataset_search_params=dataset_search_params, + ) + + download_iceflow_results( + iceflow_search_results=iceflow_search_results, + output_dir=output_dir, + ) + + parquet_path = make_iceflow_parquet( + data_dir=output_dir, + target_itrf=target_itrf, + overwrite=overwrite, + target_epoch=target_epoch, + ) + + return parquet_path + + def test_create_iceflow_parquet(tmp_path): target_itrf = "ITRF2014" common_bounding_box = BoundingBox( @@ -157,7 +203,7 @@ def test_create_iceflow_parquet(tmp_path): ) # This should finds 4 results for ILATM1B v1 and 3 results for v2. - parquet_path = create_iceflow_parquet( + parquet_path = _create_iceflow_parquet( dataset_search_params=DatasetSearchParameters( datasets=[ILATM1BDataset(version="1"), ILATM1BDataset(version="2")], bounding_box=common_bounding_box, From cd1a323951cc7fc3bee0a6580239d3d18e8717fb Mon Sep 17 00:00:00 2001 From: Trey Stafford Date: Mon, 9 Dec 2024 10:47:16 -0700 Subject: [PATCH 08/10] Update notebooks for docs and fixup notebook rendering task --- docs/iceflow-example.ipynb | 860 ++++---- docs/iceflow-with-icepyx.ipynb | 3647 ++++++++++++++++---------------- tasks/docs.py | 2 +- 3 files changed, 2324 insertions(+), 2185 deletions(-) diff --git a/docs/iceflow-example.ipynb b/docs/iceflow-example.ipynb index 0c00078..9801a02 100644 --- a/docs/iceflow-example.ipynb +++ b/docs/iceflow-example.ipynb @@ -13,7 +13,7 @@ "\n", "Finding, downloading, and reading ILATM1B v1 data with `nsidc-iceflow` is straightforward. ILATM1B data can be very large, so for the purposes of this example we will focus on just a small area near Pine Island Glacier along with a short date range in order to fetch a manageable amount of data.\n", "\n", - "To learn about how to download and manage larger amounts of data across many datasets with `iceflow`, see the [Using iceflow with icepyx to Generate an Elevation Timeseries](./iceflow-with-icepyx.ipynb) notebook." + "To learn about how to download and manage larger amounts of data across many datasets with `nsidc-iceflow`, see the [Using iceflow with icepyx to Generate an Elevation Timeseries](./iceflow-with-icepyx.ipynb) notebook." ] }, { @@ -32,17 +32,22 @@ "id": "e29f9cf9", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:50:55.406077Z", - "iopub.status.busy": "2024-11-08T18:50:55.405805Z", - "iopub.status.idle": "2024-11-08T18:50:56.410323Z", - "shell.execute_reply": "2024-11-08T18:50:56.410049Z" + "iopub.execute_input": "2024-12-09T17:35:39.880850Z", + "iopub.status.busy": "2024-12-09T17:35:39.880618Z", + "iopub.status.idle": "2024-12-09T17:35:40.959362Z", + "shell.execute_reply": "2024-12-09T17:35:40.959042Z" } }, "outputs": [], "source": [ - "from nsidc.iceflow.api import fetch_iceflow_df\n", - "from nsidc.iceflow.data.models import DatasetSearchParameters, BoundingBox, ILATM1BDataset\n", - "\n", + "from nsidc.iceflow import (\n", + " find_iceflow_data,\n", + " download_iceflow_results,\n", + " read_iceflow_datafiles,\n", + " DatasetSearchParameters,\n", + " BoundingBox,\n", + " ILATM1BDataset,\n", + ")\n", "from pathlib import Path\n", "import datetime as dt\n", "import matplotlib.pyplot as plt\n", @@ -68,10 +73,10 @@ "id": "f45d86d9-404a-4720-9ad0-9959114aa629", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:50:56.412183Z", - "iopub.status.busy": "2024-11-08T18:50:56.411964Z", - "iopub.status.idle": "2024-11-08T18:50:56.414396Z", - "shell.execute_reply": "2024-11-08T18:50:56.414176Z" + "iopub.execute_input": "2024-12-09T17:35:40.961329Z", + "iopub.status.busy": "2024-12-09T17:35:40.961091Z", + "iopub.status.idle": "2024-12-09T17:35:40.963836Z", + "shell.execute_reply": "2024-12-09T17:35:40.963457Z" } }, "outputs": [], @@ -94,19 +99,62 @@ "id": "ada043ae-65ed-4f22-ba19-dc9da6b8e0d2", "metadata": {}, "source": [ - "Now we use the `fetch_iceflow_df` function to search for and download data matching our search parameters. The output is a `pandas` DataFrame containing the matching data." + "Next, we use the `find_iceflow_data` function to search for data matching our search parameters. " ] }, { "cell_type": "code", "execution_count": 3, - "id": "15c0633c-1812-45d7-a91f-36d627fdb3b5", + "id": "c440ba2d-0c4a-47ff-b41e-a7486709d3e0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-09T17:35:40.965375Z", + "iopub.status.busy": "2024-12-09T17:35:40.965208Z", + "iopub.status.idle": "2024-12-09T17:35:45.058227Z", + "shell.execute_reply": "2024-12-09T17:35:45.057626Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "search_results = find_iceflow_data(\n", + " dataset_search_params=DatasetSearchParameters(\n", + " datasets=[atm1b_v1_dataset],\n", + " bounding_box=BBOX,\n", + " temporal=date_range,\n", + " ),\n", + ")\n", + "len(search_results)" + ] + }, + { + "cell_type": "markdown", + "id": "752d5a6a-27ec-42cc-9505-923df8b69549", + "metadata": {}, + "source": [ + "Once we have found data matching our area of interest, we can download the results with `download_iceflow_results`" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "e3b3b3bc-2d2e-49d3-bafc-3f9a4269ee57", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:50:56.415884Z", - "iopub.status.busy": "2024-11-08T18:50:56.415795Z", - "iopub.status.idle": "2024-11-08T18:51:04.321836Z", - "shell.execute_reply": "2024-11-08T18:51:04.321263Z" + "iopub.execute_input": "2024-12-09T17:35:45.061192Z", + "iopub.status.busy": "2024-12-09T17:35:45.060953Z", + "iopub.status.idle": "2024-12-09T17:35:46.823554Z", + "shell.execute_reply": "2024-12-09T17:35:46.823171Z" } }, "outputs": [ @@ -114,13 +162,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32m2024-11-08 11:51:00.613\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnsidc.iceflow.data.fetch\u001b[0m:\u001b[36msearch_and_download\u001b[0m:\u001b[36m56\u001b[0m - \u001b[1mFound 1 granules for short_name='ILATM1B' version='1' with bounding_box=BoundingBox(lower_left_lon=-103.125559, lower_left_lat=-75.180563, upper_right_lon=-102.677327, upper_right_lat=-74.798063) temporal=(datetime.date(2009, 11, 1), datetime.date(2009, 12, 31)). Downloading to downloaded-data/ILATM1B.\u001b[0m\n" + "\u001b[32m2024-12-09 10:35:45.061\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mnsidc.iceflow.data.fetch\u001b[0m:\u001b[36m_download_iceflow_search_result\u001b[0m:\u001b[36m68\u001b[0m - \u001b[1mDownloading 1 granules to downloaded-data/ILATM1B_1.\u001b[0m\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d52d51cba07c42e78a079d4e41ff9b6f", + "model_id": "b44e7f88a7ec45bfb9e3191d93f9ef0b", "version_major": 2, "version_minor": 0 }, @@ -134,7 +182,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7d0be797eeca4f7bb73cd5063650cc54", + "model_id": "687a6e75fcfc447c913563b1431b3e8f", "version_major": 2, "version_minor": 0 }, @@ -148,7 +196,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "c12d9ee611ea4ecca0b7561cc616f137", + "model_id": "dd782e76e307437bb90a4afa7bac4827", "version_major": 2, "version_minor": 0 }, @@ -159,6 +207,40 @@ "metadata": {}, "output_type": "display_data" }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[PosixPath('downloaded-data/ILATM1B_1/ILATM1B_20091109_203148.atm4cT3.qi')]\n" + ] + } + ], + "source": [ + "downloaded_files = download_iceflow_results(search_results, output_dir=data_path)\n", + "print(downloaded_files)" + ] + }, + { + "cell_type": "markdown", + "id": "1526b5bb-c08a-43d1-84b4-a207d9db924c", + "metadata": {}, + "source": [ + "Finally, we can read the data we just downloaded with `read_iceflow_datafiles`. The output is a `pandas` DataFrame containing the matching data." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "15c0633c-1812-45d7-a91f-36d627fdb3b5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-12-09T17:35:46.826168Z", + "iopub.status.busy": "2024-12-09T17:35:46.825807Z", + "iopub.status.idle": "2024-12-09T17:35:48.628664Z", + "shell.execute_reply": "2024-12-09T17:35:48.628054Z" + } + }, + "outputs": [ { "data": { "text/html": [ @@ -374,21 +456,13 @@ "2009-11-09 20:31:49.001 NaN ITRF2005 " ] }, - "execution_count": 3, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "iceflow_df = fetch_iceflow_df(\n", - " dataset_search_params=DatasetSearchParameters(\n", - " datasets=[atm1b_v1_dataset],\n", - " bounding_box=BBOX,\n", - " temporal=date_range,\n", - " ),\n", - " output_dir=data_path,\n", - " output_itrf=None,\n", - ")\n", + "iceflow_df = read_iceflow_datafiles(downloaded_files)\n", "\n", "iceflow_df.head()" ] @@ -403,14 +477,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "id": "99e03fbd", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:04.324287Z", - "iopub.status.busy": "2024-11-08T18:51:04.324040Z", - "iopub.status.idle": "2024-11-08T18:51:04.331175Z", - "shell.execute_reply": "2024-11-08T18:51:04.330860Z" + "iopub.execute_input": "2024-12-09T17:35:48.630798Z", + "iopub.status.busy": "2024-12-09T17:35:48.630364Z", + "iopub.status.idle": "2024-12-09T17:35:48.638029Z", + "shell.execute_reply": "2024-12-09T17:35:48.637720Z" } }, "outputs": [ @@ -498,7 +572,7 @@ "2009-11-09 20:31:49.001 -75.159051 -102.289878 136.298996 ITRF2005" ] }, - "execution_count": 4, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -520,14 +594,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "id": "6a0cd1e6", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:04.332826Z", - "iopub.status.busy": "2024-11-08T18:51:04.332715Z", - "iopub.status.idle": "2024-11-08T18:51:05.918076Z", - "shell.execute_reply": "2024-11-08T18:51:05.917806Z" + "iopub.execute_input": "2024-12-09T17:35:48.639480Z", + "iopub.status.busy": "2024-12-09T17:35:48.639330Z", + "iopub.status.idle": "2024-12-09T17:35:50.311249Z", + "shell.execute_reply": "2024-12-09T17:35:50.310872Z" } }, "outputs": [ @@ -615,13 +689,13 @@ "2009-11-09 20:31:49.001 -75.159051 -102.289878 136.291354 ITRF2014" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "from nsidc.iceflow.itrf.converter import transform_itrf\n", + "from nsidc.iceflow import transform_itrf\n", "\n", "itrf2014_df = transform_itrf(data=iceflow_df, target_itrf=\"ITRF2014\")\n", "itrf2014_df.head()" @@ -637,14 +711,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "b64693c0", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:05.919603Z", - "iopub.status.busy": "2024-11-08T18:51:05.919497Z", - "iopub.status.idle": "2024-11-08T18:51:05.925025Z", - "shell.execute_reply": "2024-11-08T18:51:05.924539Z" + "iopub.execute_input": "2024-12-09T17:35:50.313326Z", + "iopub.status.busy": "2024-12-09T17:35:50.313162Z", + "iopub.status.idle": "2024-12-09T17:35:50.319005Z", + "shell.execute_reply": "2024-12-09T17:35:50.318689Z" } }, "outputs": [ @@ -685,14 +759,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "id": "14e431b0", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:05.927011Z", - "iopub.status.busy": "2024-11-08T18:51:05.926759Z", - "iopub.status.idle": "2024-11-08T18:51:07.512910Z", - "shell.execute_reply": "2024-11-08T18:51:07.512499Z" + "iopub.execute_input": "2024-12-09T17:35:50.320426Z", + "iopub.status.busy": "2024-12-09T17:35:50.320327Z", + "iopub.status.idle": "2024-12-09T17:35:52.054693Z", + "shell.execute_reply": "2024-12-09T17:35:52.054037Z" } }, "outputs": [ @@ -780,7 +854,7 @@ "2009-11-09 20:31:49.001 -75.159051 -102.289872 136.291266 ITRF2014" ] }, - "execution_count": 7, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -805,14 +879,14 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "id": "8c562732", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:07.514709Z", - "iopub.status.busy": "2024-11-08T18:51:07.514599Z", - "iopub.status.idle": "2024-11-08T18:51:07.520153Z", - "shell.execute_reply": "2024-11-08T18:51:07.519796Z" + "iopub.execute_input": "2024-12-09T17:35:52.056409Z", + "iopub.status.busy": "2024-12-09T17:35:52.056299Z", + "iopub.status.idle": "2024-12-09T17:35:52.062172Z", + "shell.execute_reply": "2024-12-09T17:35:52.061727Z" } }, "outputs": [ @@ -843,14 +917,14 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "id": "5bafd254", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:07.521900Z", - "iopub.status.busy": "2024-11-08T18:51:07.521794Z", - "iopub.status.idle": "2024-11-08T18:51:07.530282Z", - "shell.execute_reply": "2024-11-08T18:51:07.530003Z" + "iopub.execute_input": "2024-12-09T17:35:52.064144Z", + "iopub.status.busy": "2024-12-09T17:35:52.064007Z", + "iopub.status.idle": "2024-12-09T17:35:52.073202Z", + "shell.execute_reply": "2024-12-09T17:35:52.072754Z" } }, "outputs": [], @@ -871,14 +945,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "id": "0ffce6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:51:07.531931Z", - "iopub.status.busy": "2024-11-08T18:51:07.531821Z", - "iopub.status.idle": "2024-11-08T18:51:07.625993Z", - "shell.execute_reply": "2024-11-08T18:51:07.625573Z" + "iopub.execute_input": "2024-12-09T17:35:52.074587Z", + "iopub.status.busy": "2024-12-09T17:35:52.074475Z", + "iopub.status.idle": "2024-12-09T17:35:52.195075Z", + "shell.execute_reply": "2024-12-09T17:35:52.194349Z" } }, "outputs": [ @@ -952,23 +1026,7 @@ "widgets": { "application/vnd.jupyter.widget-state+json": { "state": { - "095703066ff14ea4ab7e0c6a3af1f43c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "0a328e07873d4855bd03775d797c8730": { + "018ba985ac6b4d5fb8600b76bc7d7b79": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1021,60 +1079,33 @@ "width": null } }, - "0ae4a39b1fbd4fa5bbf8f1478c0812ec": { - "model_module": "@jupyter-widgets/base", + "09c2f13b70c34e65804af9c933a3b14a": { + "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "LayoutModel", + "model_name": "FloatProgressModel", "state": { - "_model_module": "@jupyter-widgets/base", + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "LayoutModel", + "_model_name": "FloatProgressModel", "_view_count": null, - "_view_module": "@jupyter-widgets/base", + "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border_bottom": null, - "border_left": null, - "border_right": null, - "border_top": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6fbaa72f65f0430a95af5c23f008138b", + "max": 1.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_6a25b1935c374f6e9f807de3a599db35", + "tabbable": null, + "tooltip": null, + "value": 1.0 } }, - "21e03aff26894b9aad15e1deaa3546ef": { + "0af9b5b2b1cc4798baafc8e1281910d6": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "model_name": "HTMLStyleModel", @@ -1092,30 +1123,33 @@ "text_color": null } }, - "3367b617a09d4f00a1d7a258ff3f67f0": { + "1d1949510c9a438e8e9a45e21c7c724c": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "HTMLModel", + "model_name": "FloatProgressModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "HTMLModel", + "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", - "_view_name": "HTMLView", + "_view_name": "ProgressView", + "bar_style": "success", "description": "", "description_allow_html": false, - "layout": "IPY_MODEL_9242d1864c5f427d85a36119bc23ce36", - "placeholder": "​", - "style": "IPY_MODEL_cc773c0630824c43b68d4d3b15ebd5fb", + "layout": "IPY_MODEL_2fd00f9534774660a6d091e8d64e71a0", + "max": 1.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_4100ddb10e4247e6a63acaf605d1c38c", "tabbable": null, "tooltip": null, - "value": " 1/1 [00:00<00:00, 156.22it/s]" + "value": 1.0 } }, - "39cd0dcdc92e4024b46048009aeb71b4": { + "2fd00f9534774660a6d091e8d64e71a0": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1168,7 +1202,23 @@ "width": null } }, - "3dee7879b63841f8a6ceb877ea0b35b9": { + "4100ddb10e4247e6a63acaf605d1c38c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "435d337bceb44597bead638d5850e005": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "model_name": "HTMLModel", @@ -1183,15 +1233,15 @@ "_view_name": "HTMLView", "description": "", "description_allow_html": false, - "layout": "IPY_MODEL_4577fadf3b0c40a7a4d29d9f2cc2fb23", + "layout": "IPY_MODEL_6b1e3827cc4847dfabf524d2b83efe71", "placeholder": "​", - "style": "IPY_MODEL_21e03aff26894b9aad15e1deaa3546ef", + "style": "IPY_MODEL_c7b7b31f2c604a1196546e2997bc0c66", "tabbable": null, "tooltip": null, - "value": " 1/1 [00:01<00:00,  1.74s/it]" + "value": "COLLECTING RESULTS | : 100%" } }, - "42ae9f2d66634262824cade5b9e14e84": { + "4c5e9421191442c2a2c6ed2406fe422f": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "model_name": "HTMLStyleModel", @@ -1209,7 +1259,57 @@ "text_color": null } }, - "4514de7881bc486c9cddf90e085c7fdb": { + "541c1134ac454078b4989985e6df8fb9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_018ba985ac6b4d5fb8600b76bc7d7b79", + "max": 1.0, + "min": 0.0, + "orientation": "horizontal", + "style": "IPY_MODEL_7284d68562514cd5bed97a15775788ca", + "tabbable": null, + "tooltip": null, + "value": 1.0 + } + }, + "687a6e75fcfc447c913563b1431b3e8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_75e3c4f701b54e9aa879e8519997bc93", + "IPY_MODEL_09c2f13b70c34e65804af9c933a3b14a", + "IPY_MODEL_97ec1eae1c9c4c88865cd95082a58c88" + ], + "layout": "IPY_MODEL_95b0d07ea55a40c08fb1f7df827bf4fc", + "tabbable": null, + "tooltip": null + } + }, + "6a25b1935c374f6e9f807de3a599db35": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "model_name": "ProgressStyleModel", @@ -1225,7 +1325,7 @@ "description_width": "" } }, - "4577fadf3b0c40a7a4d29d9f2cc2fb23": { + "6b1e3827cc4847dfabf524d2b83efe71": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1278,23 +1378,30 @@ "width": null } }, - "6dccbe75ff4747d381864550ebce72eb": { + "6e859ebf434c461caff7a96afcb64407": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "ProgressStyleModel", + "model_name": "HTMLModel", "state": { + "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "ProgressStyleModel", + "_model_name": "HTMLModel", "_view_count": null, - "_view_module": "@jupyter-widgets/base", + "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_cb66396c16f74903af9d2c98515b3594", + "placeholder": "​", + "style": "IPY_MODEL_e9b28f4d23134ae8ab8f7d45bd8ed8c2", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:00<00:00, 160.86it/s]" } }, - "74999bb22aca465696abfdaaa380fbd7": { + "6fbaa72f65f0430a95af5c23f008138b": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1347,49 +1454,46 @@ "width": null } }, - "7748851a2ae048a4b89a167f28dace99": { + "7284d68562514cd5bed97a15775788ca": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "HTMLStyleModel", + "model_name": "ProgressStyleModel", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "HTMLStyleModel", + "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", "_view_name": "StyleView", - "background": null, - "description_width": "", - "font_size": null, - "text_color": null - } + "bar_color": null, + "description_width": "" + } }, - "7d0be797eeca4f7bb73cd5063650cc54": { + "75e3c4f701b54e9aa879e8519997bc93": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "HBoxModel", + "model_name": "HTMLModel", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "HBoxModel", + "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "2.0.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b7999bd16748473996d328d4f13fe362", - "IPY_MODEL_d337f9a0ef2348d08835c6742a269707", - "IPY_MODEL_3dee7879b63841f8a6ceb877ea0b35b9" - ], - "layout": "IPY_MODEL_74999bb22aca465696abfdaaa380fbd7", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_c94e80905dbd4467a66eb15c5337b874", + "placeholder": "​", + "style": "IPY_MODEL_0af9b5b2b1cc4798baafc8e1281910d6", "tabbable": null, - "tooltip": null + "tooltip": null, + "value": "PROCESSING TASKS | : 100%" } }, - "7ff2be226e374d739011b7e26dc8cab9": { + "8fca8a66fcea4d7ca20111b2d0262d23": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "model_name": "HTMLModel", @@ -1404,15 +1508,15 @@ "_view_name": "HTMLView", "description": "", "description_allow_html": false, - "layout": "IPY_MODEL_0ae4a39b1fbd4fa5bbf8f1478c0812ec", + "layout": "IPY_MODEL_c4f39810508446018b5384d6de373afa", "placeholder": "​", - "style": "IPY_MODEL_42ae9f2d66634262824cade5b9e14e84", + "style": "IPY_MODEL_4c5e9421191442c2a2c6ed2406fe422f", "tabbable": null, "tooltip": null, "value": "QUEUEING TASKS | : 100%" } }, - "874a71279f88456ea2febf8df9292b00": { + "95b0d07ea55a40c08fb1f7df827bf4fc": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1465,7 +1569,77 @@ "width": null } }, - "8f9e30349a6c4445b1efc001d4386ed3": { + "97ec1eae1c9c4c88865cd95082a58c88": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_b8c1e2aaa184409d8a02ad33e9b6fc38", + "placeholder": "​", + "style": "IPY_MODEL_c3c6c220ad6c4168bcdc717f195fa4d1", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:01<00:00,  1.74s/it]" + } + }, + "a714d6206c934b0583c5572d7d84376d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_ee19d40b63aa4ef691061b855ac3b64e", + "placeholder": "​", + "style": "IPY_MODEL_eb2aa718055f4037afea898c891bac39", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:00<00:00, 133.50it/s]" + } + }, + "b44e7f88a7ec45bfb9e3191d93f9ef0b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_8fca8a66fcea4d7ca20111b2d0262d23", + "IPY_MODEL_1d1949510c9a438e8e9a45e21c7c724c", + "IPY_MODEL_a714d6206c934b0583c5572d7d84376d" + ], + "layout": "IPY_MODEL_c918b740c0fb460488a6f657ba79e328", + "tabbable": null, + "tooltip": null + } + }, + "b8c1e2aaa184409d8a02ad33e9b6fc38": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1518,7 +1692,25 @@ "width": null } }, - "9242d1864c5f427d85a36119bc23ce36": { + "c3c6c220ad6c4168bcdc717f195fa4d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "c4f39810508446018b5384d6de373afa": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1571,7 +1763,25 @@ "width": null } }, - "93fbc32b63da46d384de5f39548125bb": { + "c7b7b31f2c604a1196546e2997bc0c66": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "c918b740c0fb460488a6f657ba79e328": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1624,7 +1834,7 @@ "width": null } }, - "9c71b1b4a11f45968df0d0c29019a597": { + "c94e80905dbd4467a66eb15c5337b874": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1677,139 +1887,60 @@ "width": null } }, - "9f1849010d834788aa304a0758ae0a1a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "HTMLView", - "description": "", - "description_allow_html": false, - "layout": "IPY_MODEL_93fbc32b63da46d384de5f39548125bb", - "placeholder": "​", - "style": "IPY_MODEL_7748851a2ae048a4b89a167f28dace99", - "tabbable": null, - "tooltip": null, - "value": "COLLECTING RESULTS | : 100%" - } - }, - "b7999bd16748473996d328d4f13fe362": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "HTMLView", - "description": "", - "description_allow_html": false, - "layout": "IPY_MODEL_874a71279f88456ea2febf8df9292b00", - "placeholder": "​", - "style": "IPY_MODEL_c0f4e46c194d483eb43a4668d9519b12", - "tabbable": null, - "tooltip": null, - "value": "PROCESSING TASKS | : 100%" - } - }, - "c0f4e46c194d483eb43a4668d9519b12": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "HTMLStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "HTMLStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "StyleView", - "background": null, - "description_width": "", - "font_size": null, - "text_color": null - } - }, - "c12d9ee611ea4ecca0b7561cc616f137": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_9f1849010d834788aa304a0758ae0a1a", - "IPY_MODEL_f7d538f191d04c1ab10c2c3ffc430b96", - "IPY_MODEL_e27320ba73684091af234f5b9930eb7e" - ], - "layout": "IPY_MODEL_39cd0dcdc92e4024b46048009aeb71b4", - "tabbable": null, - "tooltip": null - } - }, - "cc773c0630824c43b68d4d3b15ebd5fb": { - "model_module": "@jupyter-widgets/controls", + "cb66396c16f74903af9d2c98515b3594": { + "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", - "model_name": "HTMLStyleModel", + "model_name": "LayoutModel", "state": { - "_model_module": "@jupyter-widgets/controls", + "_model_module": "@jupyter-widgets/base", "_model_module_version": "2.0.0", - "_model_name": "HTMLStyleModel", + "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", - "_view_name": "StyleView", - "background": null, - "description_width": "", - "font_size": null, - "text_color": null - } - }, - "d337f9a0ef2348d08835c6742a269707": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_allow_html": false, - "layout": "IPY_MODEL_8f9e30349a6c4445b1efc001d4386ed3", - "max": 1.0, - "min": 0.0, - "orientation": "horizontal", - "style": "IPY_MODEL_095703066ff14ea4ab7e0c6a3af1f43c", - "tabbable": null, - "tooltip": null, - "value": 1.0 + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null } }, - "d52d51cba07c42e78a079d4e41ff9b6f": { + "dd782e76e307437bb90a4afa7bac4827": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", "model_name": "HBoxModel", @@ -1824,16 +1955,16 @@ "_view_name": "HBoxView", "box_style": "", "children": [ - "IPY_MODEL_7ff2be226e374d739011b7e26dc8cab9", - "IPY_MODEL_dea132fc5f1f4c17b10e73fcf175c8f4", - "IPY_MODEL_3367b617a09d4f00a1d7a258ff3f67f0" + "IPY_MODEL_435d337bceb44597bead638d5850e005", + "IPY_MODEL_541c1134ac454078b4989985e6df8fb9", + "IPY_MODEL_6e859ebf434c461caff7a96afcb64407" ], - "layout": "IPY_MODEL_0a328e07873d4855bd03775d797c8730", + "layout": "IPY_MODEL_e07ad106fbd8464ea3a1e74d56a549cb", "tabbable": null, "tooltip": null } }, - "de71d5f46cca4bf0a65b89fcf588f002": { + "e07ad106fbd8464ea3a1e74d56a549cb": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1886,56 +2017,43 @@ "width": null } }, - "dea132fc5f1f4c17b10e73fcf175c8f4": { + "e9b28f4d23134ae8ab8f7d45bd8ed8c2": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "FloatProgressModel", + "model_name": "HTMLStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "FloatProgressModel", + "_model_name": "HTMLStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", + "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_allow_html": false, - "layout": "IPY_MODEL_f053133ab52647458863991c49f0c68e", - "max": 1.0, - "min": 0.0, - "orientation": "horizontal", - "style": "IPY_MODEL_6dccbe75ff4747d381864550ebce72eb", - "tabbable": null, - "tooltip": null, - "value": 1.0 + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null } }, - "e27320ba73684091af234f5b9930eb7e": { + "eb2aa718055f4037afea898c891bac39": { "model_module": "@jupyter-widgets/controls", "model_module_version": "2.0.0", - "model_name": "HTMLModel", + "model_name": "HTMLStyleModel", "state": { - "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "2.0.0", - "_model_name": "HTMLModel", + "_model_name": "HTMLStyleModel", "_view_count": null, - "_view_module": "@jupyter-widgets/controls", + "_view_module": "@jupyter-widgets/base", "_view_module_version": "2.0.0", - "_view_name": "HTMLView", - "description": "", - "description_allow_html": false, - "layout": "IPY_MODEL_de71d5f46cca4bf0a65b89fcf588f002", - "placeholder": "​", - "style": "IPY_MODEL_f27416801cb049c3a170298199434e85", - "tabbable": null, - "tooltip": null, - "value": " 1/1 [00:00<00:00, 133.70it/s]" + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null } }, - "f053133ab52647458863991c49f0c68e": { + "ee19d40b63aa4ef691061b855ac3b64e": { "model_module": "@jupyter-widgets/base", "model_module_version": "2.0.0", "model_name": "LayoutModel", @@ -1987,50 +2105,6 @@ "visibility": null, "width": null } - }, - "f27416801cb049c3a170298199434e85": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "HTMLStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "HTMLStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "2.0.0", - "_view_name": "StyleView", - "background": null, - "description_width": "", - "font_size": null, - "text_color": null - } - }, - "f7d538f191d04c1ab10c2c3ffc430b96": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "2.0.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "2.0.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "2.0.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_allow_html": false, - "layout": "IPY_MODEL_9c71b1b4a11f45968df0d0c29019a597", - "max": 1.0, - "min": 0.0, - "orientation": "horizontal", - "style": "IPY_MODEL_4514de7881bc486c9cddf90e085c7fdb", - "tabbable": null, - "tooltip": null, - "value": 1.0 - } } }, "version_major": 2, diff --git a/docs/iceflow-with-icepyx.ipynb b/docs/iceflow-with-icepyx.ipynb index ae63d99..b1cf1ae 100644 --- a/docs/iceflow-with-icepyx.ipynb +++ b/docs/iceflow-with-icepyx.ipynb @@ -47,10 +47,10 @@ "id": "2ad99b0d-4e15-466c-b9aa-7bfa75ef5f26", "metadata": { "execution": { - "iopub.execute_input": "2024-11-08T18:18:25.101422Z", - "iopub.status.busy": "2024-11-08T18:18:25.100773Z", - "iopub.status.idle": "2024-11-08T18:18:27.894676Z", - "shell.execute_reply": "2024-11-08T18:18:27.894150Z" + "iopub.execute_input": "2024-12-09T17:35:54.192543Z", + "iopub.status.busy": "2024-12-09T17:35:54.192247Z", + "iopub.status.idle": "2024-12-09T17:35:56.824194Z", + "shell.execute_reply": "2024-12-09T17:35:56.823710Z" } }, "outputs": [ @@ -58,7 +58,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/tmp/ipykernel_138573/2137618702.py:6: FutureWarning: icepyx v1.x is being deprecated; the back-end systems on which it relies\n", + "/tmp/ipykernel_163498/2248149546.py:6: FutureWarning: icepyx v1.x is being deprecated; the back-end systems on which it relies\n", "will be shut down as of late 2024. At that time, upgrade to icepyx v2.x, which uses the\n", "new NASA Harmony back-end, will be required. Please see\n", " for more\n", @@ -599,11 +599,11 @@ "application/vnd.holoviews_exec.v0+json": "", "text/html": [ "
\n", - "
\n", + "
\n", "
\n", "