Skip to content

Commit

Permalink
pre-commit fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
danielfromearth committed Nov 14, 2024
1 parent 27e1ec5 commit 5e11af8
Show file tree
Hide file tree
Showing 7 changed files with 39 additions and 14 deletions.
Empty file modified .github/workflows/wait-for-pypi.py
100644 → 100755
Empty file.
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ ci:
autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger

default_language_version:
python: python3.9
python: python3.10

repos:
- repo: https://github.com/gitleaks/gitleaks
Expand Down
4 changes: 3 additions & 1 deletion batcher/harmony/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
"""A Harmony CLI wrapper around the concatenate-batcher"""

from argparse import ArgumentParser

import harmony_service_lib
Expand All @@ -45,7 +46,8 @@ def main(config: harmony_service_lib.util.Config = None) -> None:
None
"""
parser = ArgumentParser(
prog="Pre-concatenate-batching", description="Run the pre-concatenate-batching service"
prog="Pre-concatenate-batching",
description="Run the pre-concatenate-batching service",
)
harmony_service_lib.setup_cli(parser)
args = parser.parse_args()
Expand Down
12 changes: 9 additions & 3 deletions batcher/harmony/service_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ def invoke(self):
# Message-only support is being depreciated in Harmony, so we should expect to
# only see requests with catalogs when invoked with a newer Harmony instance
# https://github.com/nasa/harmony-service-lib-py/blob/21bcfbda17caf626fb14d2ac4f8673be9726b549/harmony/adapter.py#L71
raise RuntimeError("Invoking Batchee without a STAC catalog is not supported")
raise RuntimeError(
"Invoking Batchee without a STAC catalog is not supported"
)

return self.message, self.process_catalog(self.catalog)

Expand Down Expand Up @@ -108,7 +110,9 @@ def process_catalog(self, catalog: pystac.Catalog) -> list[pystac.Catalog]:
# and each Catalog holds multiple Items (which represent each granule).
catalogs = []
for batch_id, batch_items in grouped.items():
self.logger.info(f"constructing new pystac.Catalog for batch_id==={batch_id}.")
self.logger.info(
f"constructing new pystac.Catalog for batch_id==={batch_id}."
)
# Initialize a new, empty Catalog
batch_catalog = catalog.clone()
batch_catalog.id = str(uuid4())
Expand All @@ -135,7 +139,9 @@ def process_catalog(self, catalog: pystac.Catalog) -> list[pystac.Catalog]:
)
batch_catalog.add_item(output_item)

self.logger.info("STAC catalog creation for batch_id==={batch_id} complete.")
self.logger.info(
"STAC catalog creation for batch_id==={batch_id} complete."
)
catalogs.append(batch_catalog)

self.logger.info("All STAC catalogs are complete.")
Expand Down
6 changes: 5 additions & 1 deletion batcher/harmony/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
"""Misc utility functions"""

from datetime import datetime

from pystac import Asset, Item
Expand Down Expand Up @@ -104,7 +105,10 @@ def _get_output_date_range(input_items: list[Item]) -> dict[str, str]:
start_datetime = min(start_datetime, new_start_datetime)
end_datetime = max(end_datetime, new_end_datetime)

return {"start_datetime": start_datetime.isoformat(), "end_datetime": end_datetime.isoformat()}
return {
"start_datetime": start_datetime.isoformat(),
"end_datetime": end_datetime.isoformat(),
}


def _get_item_date_range(item: Item) -> tuple[datetime, datetime]:
Expand Down
20 changes: 15 additions & 5 deletions batcher/tempo_filename_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
# either express or implied. See the License for the specific language governing permissions and
# limitations under the License.
"""Holds the logic for grouping together data files based on their filenames."""

import logging
import re
from argparse import ArgumentParser
Expand All @@ -44,7 +45,9 @@
)


def get_batch_indices(filenames: list, logger: logging.Logger = default_logger) -> list[int]:
def get_batch_indices(
filenames: list, logger: logging.Logger = default_logger
) -> list[int]:
"""
Returns
-------
Expand All @@ -59,10 +62,14 @@ def get_batch_indices(filenames: list, logger: logging.Logger = default_logger)
matches = tempo_granule_filename_pattern.match(name)
if matches:
match_dict = matches.groupdict()
day_and_scans.append((match_dict["day_in_granule"], match_dict["daily_scan_id"]))
day_and_scans.append(
(match_dict["day_in_granule"], match_dict["daily_scan_id"])
)

# Unique day-scans are determined (while keeping the same order). Each will be its own batch.
unique_day_scans: list[tuple[str, str]] = sorted(set(day_and_scans), key=day_and_scans.index)
unique_day_scans: list[tuple[str, str]] = sorted(
set(day_and_scans), key=day_and_scans.index
)

logger.info(f"unique_day_scans==={unique_day_scans}.")

Expand All @@ -79,7 +86,8 @@ def main() -> list[list[str]]:
"""Main CLI entrypoint"""

parser = ArgumentParser(
prog="batchee", description="Simple CLI wrapper around the granule batcher module."
prog="batchee",
description="Simple CLI wrapper around the granule batcher module.",
)
parser.add_argument(
"file_names",
Expand All @@ -101,7 +109,9 @@ def main() -> list[list[str]]:
input_filenames = args.file_names

batch_indices = get_batch_indices(input_filenames)
unique_category_indices: list[int] = sorted(set(batch_indices), key=batch_indices.index)
unique_category_indices: list[int] = sorted(
set(batch_indices), key=batch_indices.index
)
logging.info(f"batch_indices = {batch_indices}")

# --- Construct a STAC object based on the batch indices ---
Expand Down
9 changes: 6 additions & 3 deletions tests/test_harmony_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ def test_service_invoke(self, temp_output_dir):

# test with both paged catalogs and un-paged catalogs
for in_catalog_name in ["catalog.json", "catalog0.json"]:

in_catalog_path = self.__harmony_path.joinpath("source", in_catalog_name)

test_args = [
Expand Down Expand Up @@ -64,7 +63,9 @@ def test_service_invoke(self, temp_output_dir):

for item_meta in out_catalog["links"]:
if item_meta["rel"] == "item":
item_path = temp_output_dir.joinpath(item_meta["href"]).resolve()
item_path = temp_output_dir.joinpath(
item_meta["href"]
).resolve()

# -- Item Verification --
item = json.loads(item_path.read_text())
Expand All @@ -80,7 +81,9 @@ def test_service_invoke(self, temp_output_dir):
assert data["type"] == "application/x-netcdf4"
assert data["roles"] == ["data"]

batched_files[batch_index].append(Path(urlsplit(data["href"]).path).stem)
batched_files[batch_index].append(
Path(urlsplit(data["href"]).path).stem
)

# -- batch file list verification --
files_dict = {
Expand Down

0 comments on commit 5e11af8

Please sign in to comment.