Skip to content

Commit

Permalink
Support batch mode
Browse files Browse the repository at this point in the history
  • Loading branch information
matrss committed Aug 19, 2024
1 parent c200591 commit 643f002
Showing 1 changed file with 25 additions and 6 deletions.
31 changes: 25 additions & 6 deletions src/datalad_cds/download_cds.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ class DownloadCDS(Interface):
doc="""target path to download to.""",
constraints=EnsureStr(),
),
batch=Parameter(
args=("--batch",),
action="store_true",
doc="""By default a single request can be supplied for download.
With the batch option, the requests to download are instead read
from stdin, one request and associated filename per line.""",
),
lazy=Parameter(
args=("--lazy",),
action="store_true",
Expand All @@ -68,28 +75,40 @@ class DownloadCDS(Interface):
@datasetmethod(name="download_cds")
@eval_results
def __call__(
spec: Union[str, dict],
path: str,
spec: Optional[Union[str, dict]] = None,
path: Optional[str] = None,
*,
dataset: Optional[str] = None,
message: Optional[str] = None,
save: bool = True,
batch: bool = False,
lazy: bool = False,
save: bool = True,
) -> Iterable[dict]:
ds = require_dataset(dataset, check_installed=True)
ensure_special_remote_exists_and_is_enabled(ds.repo, "cds")
if batch:
try:
next_line = input()
spec, path = next_line.split(" ", 1)
self._handle_one(ds, spec, path, batch, lazy, safe)
except EOFError:
pass
else:
self._handle_one(ds, spec, path, batch, lazy, safe)

def _handle_one(ds, spec, path, batch, lazy, safe):
if isinstance(spec, dict):
parsed_spec = datalad_cds.spec.Spec.from_dict(spec)
elif isinstance(spec, str):
parsed_spec = datalad_cds.spec.Spec.from_json(spec)
else:
raise TypeError("spec could not be parsed")
ds = require_dataset(dataset, check_installed=True)
ensure_special_remote_exists_and_is_enabled(ds.repo, "cds")
pathobj = ds.pathobj / path
url = parsed_spec.to_url()
options = []
if lazy:
options.append("--fast")
ds.repo.add_url_to_file(pathobj, url, options=options)
ds.repo.add_url_to_file(pathobj, url, batch=batch, options=options)
if save:
msg = (
message
Expand Down

0 comments on commit 643f002

Please sign in to comment.