From b0bf065a54dd02f8c071ac8ef13c1bb3a94e0f33 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Thu, 7 Mar 2024 09:59:13 -0500 Subject: [PATCH 01/53] Convert DOI URLs in `related_publications` to related resources --- dandi/metadata/util.py | 25 +++++++++++++++++++ .../tests/data/metadata/metadata2asset_3.json | 7 ++++++ .../data/metadata/metadata2asset_simple1.json | 3 ++- dandi/tests/test_metadata.py | 6 ++++- 4 files changed, 39 insertions(+), 2 deletions(-) diff --git a/dandi/metadata/util.py b/dandi/metadata/util.py index 8849a9164..d3f8f95b4 100644 --- a/dandi/metadata/util.py +++ b/dandi/metadata/util.py @@ -11,6 +11,7 @@ from dandischema import models import requests import tenacity +from yarl import URL from .. import __version__ from ..utils import ensure_datetime @@ -583,6 +584,29 @@ def extract_digest(metadata: dict) -> dict[models.DigestType, str] | None: return None +def extract_related_resource(metadata: dict) -> list[models.Resource] | None: + pubs = metadata.get("related_publications") + if not isinstance(pubs, (list, tuple)): + return None + related = [] + for v in pubs: + if not isinstance(v, str): + continue + try: + u = URL(v) + except ValueError: + continue + if u.scheme not in ("http", "https") or u.host != "doi.org": + continue + related.append( + models.Resource( + identifier=v, + relation=models.RelationType.IsDescribedBy, + ) + ) + return related + + FIELD_EXTRACTORS: dict[str, Callable[[dict], Any]] = { "wasDerivedFrom": extract_wasDerivedFrom, "wasAttributedTo": extract_wasAttributedTo, @@ -595,6 +619,7 @@ def extract_digest(metadata: dict) -> dict[models.DigestType, str] | None: "anatomy": extract_anatomy, "digest": extract_digest, "species": extract_species, + "relatedResource": extract_related_resource, } diff --git a/dandi/tests/data/metadata/metadata2asset_3.json b/dandi/tests/data/metadata/metadata2asset_3.json index f3e844133..44b809a69 100644 --- a/dandi/tests/data/metadata/metadata2asset_3.json +++ b/dandi/tests/data/metadata/metadata2asset_3.json @@ -92,5 +92,12 @@ "name": "Cyperus bulbosus" } } + ], + "relatedResource": [ + { + "schemaKey": "Resource", + "identifier": "https://doi.org/10.48324/dandi.000027/0.210831.2033", + "relation": "dcite:IsDescribedBy" + } ] } diff --git a/dandi/tests/data/metadata/metadata2asset_simple1.json b/dandi/tests/data/metadata/metadata2asset_simple1.json index 04babc1a1..931779db5 100644 --- a/dandi/tests/data/metadata/metadata2asset_simple1.json +++ b/dandi/tests/data/metadata/metadata2asset_simple1.json @@ -42,5 +42,6 @@ "schemaKey": "Participant", "identifier": "sub-01" } - ] + ], + "relatedResource": [] } diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 07c19b24a..1cd082358 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -323,7 +323,9 @@ def test_timedelta2duration(td: timedelta, duration: str) -> None: "institution": "University College", "keywords": ["test", "sample", "example", "test-case"], "lab": "Retriever Laboratory", - "related_publications": "A Brief History of Test Cases", + "related_publications": [ + "https://doi.org/10.48324/dandi.000027/0.210831.2033" + ], "session_description": "Some test data", "session_id": "XYZ789", "session_start_time": "2020-08-31T15:58:28-04:00", @@ -860,6 +862,7 @@ def test_nwb2asset(simple2_nwb: Path) -> None: variableMeasured=[], measurementTechnique=[], approach=[], + relatedResource=[], ) @@ -939,4 +942,5 @@ def test_nwb2asset_remote_asset(nwb_dandiset: SampleDandiset) -> None: variableMeasured=[], measurementTechnique=[], approach=[], + relatedResource=[], ) From ca4c1702e46dae5ca4db2886e2700fe1f408bb31 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 19 Mar 2024 11:05:37 -0400 Subject: [PATCH 02/53] Do not allow dandischema 0.10.1 - new schema not yet supported by dandi-archive With current release of dandi-schema our testing starts to fail, but dandi-archive seems needing CI fixing first to also make upgrade possible. So as a quick resolution to avoid ill effects on users etc, I think we should restrict version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0e0ebcd8b..98950615d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ install_requires = bidsschematools ~= 0.7.0 click >= 7.1 click-didyoumean - dandischema >= 0.9.0, < 0.11 + dandischema >= 0.9.0, < 0.10.1 etelemetry >= 0.2.2 fasteners fscacher >= 0.3.0 From c4674026f34473283becbe53d3ae13c8d7b5697e Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Tue, 19 Mar 2024 18:46:24 +0000 Subject: [PATCH 03/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d8c02ec9..895684b74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,28 @@ +# 0.61.0 (Tue Mar 19 2024) + +#### ๐Ÿš€ Enhancement + +- Add arguments for API query parameters when fetching all Dandisets; support creating embargoed Dandisets [#1414](https://github.com/dandi/dandi-cli/pull/1414) ([@jwodder](https://github.com/jwodder)) + +#### ๐Ÿ› Bug Fix + +- Do not allow dandischema 0.10.1 - new schema not yet supported by dandi-archive [#1419](https://github.com/dandi/dandi-cli/pull/1419) ([@yarikoptic](https://github.com/yarikoptic)) + +#### ๐Ÿ  Internal + +- Clean up URL parsing in `extract_species()` [#1416](https://github.com/dandi/dandi-cli/pull/1416) ([@jwodder](https://github.com/jwodder) [@yarikoptic](https://github.com/yarikoptic)) + +#### ๐Ÿ”ฉ Dependency Updates + +- Use `yarl` to clean up some code [#1415](https://github.com/dandi/dandi-cli/pull/1415) ([@jwodder](https://github.com/jwodder)) + +#### Authors: 2 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # 0.60.0 (Thu Feb 29 2024) #### ๐Ÿš€ Enhancement From a40077feb98fd4eb0eaa2d24668a56ede12eef0c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 21 Mar 2024 11:30:15 -0400 Subject: [PATCH 04/53] Revert "Do not allow dandischema 0.10.1 - new schema not yet supported by dandi-archive" This reverts commit ca4c1702e46dae5ca4db2886e2700fe1f408bb31. dandi-archive supports new schema now, we should be fine to proceed. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 98950615d..0e0ebcd8b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ install_requires = bidsschematools ~= 0.7.0 click >= 7.1 click-didyoumean - dandischema >= 0.9.0, < 0.10.1 + dandischema >= 0.9.0, < 0.11 etelemetry >= 0.2.2 fasteners fscacher >= 0.3.0 From 23e702eadae4914e0b25582deac689af29ab872a Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Thu, 21 Mar 2024 16:09:01 +0000 Subject: [PATCH 05/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 895684b74..3a59b3776 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 0.61.1 (Thu Mar 21 2024) + +#### ๐Ÿ› Bug Fix + +- Revert "Do not allow dandischema 0.10.1 - new schema not yet supporteed by dandi-archive" [#1420](https://github.com/dandi/dandi-cli/pull/1420) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 1 + +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # 0.61.0 (Tue Mar 19 2024) #### ๐Ÿš€ Enhancement From 8b76c82fe98b719af21abb1fcb10e2040b43b2c5 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Fri, 22 Mar 2024 11:09:40 -0400 Subject: [PATCH 06/53] Add `manage.py createcachetable` to test fixture dandi-archive requires this command to be run as part of system initialization in order to initialize the database to serve as a Django cache backend. it must be run in order for any endpoint with caching enabled to work properly. --- dandi/tests/fixtures.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index a18123e15..2b9a40e89 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -402,6 +402,19 @@ def docker_compose_setup() -> Iterator[dict[str, str]]: env=env, check=True, ) + run( + [ + "docker-compose", + "run", + "--rm", + "django", + "./manage.py", + "createcachetable", + ], + cwd=str(LOCAL_DOCKER_DIR), + env=env, + check=True, + ) run( [ "docker-compose", From 21ef19d90511ba8a8500642c39c6fe34dcffd3dd Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Fri, 22 Mar 2024 16:44:35 +0000 Subject: [PATCH 07/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a59b3776..6085949fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 0.61.2 (Fri Mar 22 2024) + +#### ๐Ÿงช Tests + +- Add missing command to `dandi-archive` docker compose fixture [#1421](https://github.com/dandi/dandi-cli/pull/1421) ([@mvandenburgh](https://github.com/mvandenburgh)) + +#### Authors: 1 + +- Mike VanDenburgh ([@mvandenburgh](https://github.com/mvandenburgh)) + +--- + # 0.61.1 (Thu Mar 21 2024) #### ๐Ÿ› Bug Fix From 408f795fa38233895a5049cf1853cfa7c221d1cb Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Fri, 22 Mar 2024 13:00:35 -0400 Subject: [PATCH 08/53] Xfail flaky ontobee tests, unless running daily tests --- .github/workflows/test.yml | 4 ++++ dandi/conftest.py | 6 ++++++ dandi/tests/test_metadata.py | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b20415b68..afe44cf94 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -89,6 +89,10 @@ jobs: echo HOME=/tmp/nfsmount/home >> "$GITHUB_ENV" echo DANDI_DEVEL_INSTRUMENT_REQUESTS_SUPERLEN=1 >> "$GITHUB_ENV" + - name: Use scheduled test configuration + if: github.event_name == 'schedule' + run: echo PYTEST_ADDOPTS=--schedule >> "$GITHUB_ENV" + - name: Run all tests if: matrix.mode != 'dandi-api' run: | diff --git a/dandi/conftest.py b/dandi/conftest.py index 61a3fc760..a59f14b3d 100644 --- a/dandi/conftest.py +++ b/dandi/conftest.py @@ -12,6 +12,12 @@ def pytest_addoption(parser: Parser) -> None: default=False, help="Only run tests of the new Django Dandi API", ) + parser.addoption( + "--scheduled", + action="store_true", + default=False, + help="Use configuration for a scheduled daily test run", + ) def pytest_collection_modifyitems(items: list[Item], config: Config) -> None: diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 07c19b24a..4b6d64e6b 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -29,6 +29,7 @@ from dateutil.tz import tzutc from pydantic import ByteSize import pytest +import requests from semantic_version import Version from .fixtures import SampleDandiset @@ -234,6 +235,12 @@ def test_timedelta2duration(td: timedelta, duration: str) -> None: assert timedelta2duration(td) == duration +@pytest.mark.xfail( + condition="not config.getoption('--scheduled')", + reason="Flaky ontobee site", + strict=False, + raises=requests.RequestException, +) @mark.skipif_no_network @pytest.mark.parametrize( "filename, metadata", @@ -459,6 +466,12 @@ def test_time_extract_gest() -> None: ) +@pytest.mark.xfail( + condition="not config.getoption('--scheduled')", + reason="Flaky ontobee site", + strict=False, + raises=requests.RequestException, +) @mark.skipif_no_network @pytest.mark.obolibrary @pytest.mark.parametrize( @@ -489,6 +502,12 @@ def test_parseobourl(url, value): assert parse_purlobourl(url) == value +@pytest.mark.xfail( + condition="not config.getoption('--scheduled')", + reason="Flaky ontobee site", + strict=False, + raises=requests.RequestException, +) @pytest.mark.obolibrary @mark.skipif_no_network def test_species(): From 6e33659d9068d1c0c954fa44c7bad041d329d17f Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 25 Mar 2024 10:55:54 -0400 Subject: [PATCH 09/53] Put xfail marker in `mark_xfail_ontobee` variable --- dandi/tests/test_metadata.py | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/dandi/tests/test_metadata.py b/dandi/tests/test_metadata.py index 4b6d64e6b..620fac670 100644 --- a/dandi/tests/test_metadata.py +++ b/dandi/tests/test_metadata.py @@ -54,6 +54,13 @@ METADATA_DIR = Path(__file__).with_name("data") / "metadata" +mark_xfail_ontobee = pytest.mark.xfail( + condition="not config.getoption('--scheduled')", + reason="Flaky ontobee site", + strict=False, + raises=requests.RequestException, +) + def test_get_metadata(simple1_nwb: Path, simple1_nwb_metadata: dict[str, Any]) -> None: target_metadata = simple1_nwb_metadata.copy() @@ -235,12 +242,7 @@ def test_timedelta2duration(td: timedelta, duration: str) -> None: assert timedelta2duration(td) == duration -@pytest.mark.xfail( - condition="not config.getoption('--scheduled')", - reason="Flaky ontobee site", - strict=False, - raises=requests.RequestException, -) +@mark_xfail_ontobee @mark.skipif_no_network @pytest.mark.parametrize( "filename, metadata", @@ -466,12 +468,7 @@ def test_time_extract_gest() -> None: ) -@pytest.mark.xfail( - condition="not config.getoption('--scheduled')", - reason="Flaky ontobee site", - strict=False, - raises=requests.RequestException, -) +@mark_xfail_ontobee @mark.skipif_no_network @pytest.mark.obolibrary @pytest.mark.parametrize( @@ -502,12 +499,7 @@ def test_parseobourl(url, value): assert parse_purlobourl(url) == value -@pytest.mark.xfail( - condition="not config.getoption('--scheduled')", - reason="Flaky ontobee site", - strict=False, - raises=requests.RequestException, -) +@mark_xfail_ontobee @pytest.mark.obolibrary @mark.skipif_no_network def test_species(): From 05f3e3704d9ad44babcac4c18c6c86c2c32bbb34 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 16 Apr 2024 12:35:58 -0400 Subject: [PATCH 10/53] Fix spelling of `--scheduled` option used in scheduled tests --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index afe44cf94..d8089a536 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,7 +91,7 @@ jobs: - name: Use scheduled test configuration if: github.event_name == 'schedule' - run: echo PYTEST_ADDOPTS=--schedule >> "$GITHUB_ENV" + run: echo PYTEST_ADDOPTS=--scheduled >> "$GITHUB_ENV" - name: Run all tests if: matrix.mode != 'dandi-api' From 17bacb8b5a927abecda6ce696d473154d2ab9124 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2024 19:34:01 -0400 Subject: [PATCH 11/53] Add codespell-problem-matcher to lint workflow --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a350fc097..b9e5d8fb1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,6 +20,8 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install --upgrade tox + # Annotate codespell within PR + - uses: codespell-project/codespell-problem-matcher@v1 - name: Run linters run: | tox -e lint From 6885b40749c6cd058af55ca6e73c53db59ea621b Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 22 Apr 2024 19:41:44 -0400 Subject: [PATCH 12/53] Extend codespelling to docs and tools in tox --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 39bd25055..8c7f6093e 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ deps = codespell~=2.0 flake8 commands = - codespell dandi setup.py + codespell dandi docs tools setup.py flake8 --config=setup.cfg {posargs} dandi setup.py [testenv:typing] From 89cf7a6b32f22087ab1bc928be34162ba1584293 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 26 Apr 2024 03:11:33 -0500 Subject: [PATCH 13/53] Fix spelling of netlify --- CHANGELOG.md | 2 +- dandi/dandiarchive.py | 2 +- docs/source/ref/urls.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6085949fb..3d38b5290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2114,7 +2114,7 @@ - Fixes [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) - Rename DANDI_API_KEY to DANDI_GIRDER_API_KEY [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) - Test of uploading & downloading via new API [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) -- RF+ENH: support mapping for direct API urls, and use netflify insstance instead of api+ prefix [#330](https://github.com/dandi/dandi-cli/pull/330) ([@yarikoptic](https://github.com/yarikoptic)) +- RF+ENH: support mapping for direct API urls, and use netlify insstance instead of api+ prefix [#330](https://github.com/dandi/dandi-cli/pull/330) ([@yarikoptic](https://github.com/yarikoptic)) - Delint [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) - RF: account for web UI URL changes/dropped features, remove support for girder URLs [#330](https://github.com/dandi/dandi-cli/pull/330) ([@yarikoptic](https://github.com/yarikoptic)) - Handle uploading already-extant assets [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index 53c3506cb..cf7dec4c6 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -604,7 +604,7 @@ class _dandi_url_parser: ( re.compile(r"https?://[^/]*dandiarchive-org\.netlify\.app/.*"), {"map_instance": "dandi"}, - "https://*dandiarchive-org.netflify.app/...", + "https://*dandiarchive-org.netlify.app/...", ), # Direct urls to our new API ( diff --git a/docs/source/ref/urls.rst b/docs/source/ref/urls.rst index bd3f7ca7f..a1cc1a058 100644 --- a/docs/source/ref/urls.rst +++ b/docs/source/ref/urls.rst @@ -22,7 +22,7 @@ has one, and its draft version will be used otherwise. `parse_dandi_url()` converts this format to a `DandisetURL`. - Any ``https://gui.dandiarchive.org/`` or - ``https://*dandiarchive-org.netflify.app/`` URL which redirects to + ``https://*dandiarchive-org.netlify.app/`` URL which redirects to one of the other URL formats - :samp:`https://{server}[/api]/[#/]dandiset/{dandiset-id}[/{version}][/files]` From eb065ead82689b74da5577c5554ac8fb1a95dc46 Mon Sep 17 00:00:00 2001 From: Jacob Nesbitt Date: Fri, 26 Apr 2024 12:53:35 -0400 Subject: [PATCH 14/53] Add DJANGO_DANDI_DEV_EMAIL env var to archive tests --- dandi/tests/data/dandiarchive-docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 03ae76d6c..a8530b576 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -36,6 +36,7 @@ services: DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 DJANGO_DANDI_API_URL: http://localhost:8000 DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org + DJANGO_DANDI_DEV_EMAIL: test@example.com DANDI_ALLOW_LOCALHOST_URLS: "1" ports: - "8000:8000" From 9637d0e08e7b68bc110a686efd9f6456663656f7 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Fri, 26 Apr 2024 15:29:19 -0500 Subject: [PATCH 15/53] clarify resource identifier docs Fixes: #https://github.com/dandi/dandi-cli/issues/1435 --- dandi/cli/cmd_download.py | 2 ++ dandi/cli/cmd_ls.py | 2 ++ dandi/dandiarchive.py | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index ee43459d6..e40f2b82d 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -18,6 +18,8 @@ help=f"""\ Download files or entire folders from DANDI. +\b +{_dandi_url_parser.resource_identifier_primer} \b {_dandi_url_parser.known_patterns} """ diff --git a/dandi/cli/cmd_ls.py b/dandi/cli/cmd_ls.py index eeb3dbecd..9ac9ed42f 100644 --- a/dandi/cli/cmd_ls.py +++ b/dandi/cli/cmd_ls.py @@ -26,6 +26,8 @@ The arguments may be either resource identifiers or paths to local files/directories. +\b +{_dandi_url_parser.resource_identifier_primer} \b {_dandi_url_parser.known_patterns} """ diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index 53c3506cb..1c1dc5c8b 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -675,6 +675,15 @@ class _dandi_url_parser: "https:///...", ), ] + resource_identifier_primer = """dandi commands and accept URLs and URL-like identifiers called + in the following formats for identifying Dandisets, assets, and asset +collections. + +Text in [brackets] is optional. A server field is a base API or GUI URL +for a DANDI Archive instance. If an optional ``version`` field is omitted from +a URL, the given Dandiset's most recent published version will be used if it +has one, and its draft version will be used otherwise. +""" known_patterns = "Accepted resource identifier patterns:" + "\n - ".join( [""] + [display for _, _, display in known_urls] ) From d8bae4d607adf290f752553bd57b50762b0c38ba Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Fri, 26 Apr 2024 16:51:47 -0500 Subject: [PATCH 16/53] Add examples Co-authored-by: Yaroslav Halchenko --- dandi/cli/cmd_download.py | 21 ++++++++++++++++++++- dandi/dandiarchive.py | 19 ++++++++++--------- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index e40f2b82d..c7aff83dd 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -11,6 +11,20 @@ from ..download import DownloadExisting, DownloadFormat, PathType from ..utils import get_instance, joinurl +_examples = """ +EXAMPLES: \n +# Download only the dandiset.yaml \n +dandi download --download dandiset.yaml DANDI:000027 \n + +# Download only dandiset.yaml if there is a newer version \n +dandi download https://identifiers.org/DANDI:000027 --existing refresh + +# Download only the assets \n +dandi download --download assets DANDI:000027 + +# Download all from a specific version \n +dandi download DANDI:000027/0.210831.2033 +""" # The use of f-strings apparently makes this not a proper docstring, and so # click doesn't use it unless we explicitly assign it to `help`: @@ -20,9 +34,14 @@ \b {_dandi_url_parser.resource_identifier_primer} + \b {_dandi_url_parser.known_patterns} - """ + +\b +{_examples} + +""" ) @click.option( "-o", diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index 1c1dc5c8b..2d355b433 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -675,15 +675,16 @@ class _dandi_url_parser: "https:///...", ), ] - resource_identifier_primer = """dandi commands and accept URLs and URL-like identifiers called - in the following formats for identifying Dandisets, assets, and asset -collections. - -Text in [brackets] is optional. A server field is a base API or GUI URL -for a DANDI Archive instance. If an optional ``version`` field is omitted from -a URL, the given Dandiset's most recent published version will be used if it -has one, and its draft version will be used otherwise. -""" + resource_identifier_primer = """RESOURCE ID/URLS:\n + dandi commands accept URLs and URL-like identifiers called + in the following formats for identifying Dandisets, + assets, and asset collections. + + Text in [brackets] is optional. A server field is a base API or GUI URL + for a DANDI Archive instance. If an optional ``version`` field is omitted from + a URL, the given Dandiset's most recent published version will be used if it + has one, and its draft version will be used otherwise. + """ known_patterns = "Accepted resource identifier patterns:" + "\n - ".join( [""] + [display for _, _, display in known_urls] ) From 16c41b66996215a22a4ac51162ac0c46bb24ea3a Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Fri, 26 Apr 2024 19:18:27 -0500 Subject: [PATCH 17/53] line spacing --- dandi/cli/cmd_download.py | 1 + dandi/dandiarchive.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index c7aff83dd..d44496b0b 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -26,6 +26,7 @@ dandi download DANDI:000027/0.210831.2033 """ + # The use of f-strings apparently makes this not a proper docstring, and so # click doesn't use it unless we explicitly assign it to `help`: @click.command( diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index 2d355b433..54bd9a772 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -676,14 +676,14 @@ class _dandi_url_parser: ), ] resource_identifier_primer = """RESOURCE ID/URLS:\n - dandi commands accept URLs and URL-like identifiers called - in the following formats for identifying Dandisets, - assets, and asset collections. + dandi commands accept URLs and URL-like identifiers called in the following formats for identifying Dandisets, assets, and + asset collections. Text in [brackets] is optional. A server field is a base API or GUI URL - for a DANDI Archive instance. If an optional ``version`` field is omitted from - a URL, the given Dandiset's most recent published version will be used if it - has one, and its draft version will be used otherwise. + for a DANDI Archive instance. If an optional ``version`` field is + omitted from a URL, the given Dandiset's most recent published version + will be used if it has one, and its draft version will be used otherwise. """ known_patterns = "Accepted resource identifier patterns:" + "\n - ".join( [""] + [display for _, _, display in known_urls] From 3afd99b22d9871963ad9ddb6fb70861e2973e053 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Sun, 28 Apr 2024 12:40:52 -0500 Subject: [PATCH 18/53] Add specific file example --- dandi/cli/cmd_download.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index d44496b0b..fd62a7c05 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -24,6 +24,9 @@ # Download all from a specific version \n dandi download DANDI:000027/0.210831.2033 + +# Download a specific file or directory \n +dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/sub-RAT123.nwb """ From ae0f1871ee3db5c1ecf2314daed30d1b8b0f6b51 Mon Sep 17 00:00:00 2001 From: Isaac To Date: Tue, 30 Apr 2024 16:58:34 -0700 Subject: [PATCH 19/53] Set `DJANGO_DANDI_DEV_EMAIL` in `django` and `celery` containers This environment var is needed for running the latest `dandiarchive/dandiarchive-api` image --- dandi/tests/data/dandiarchive-docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 03ae76d6c..a2fd30bc8 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -37,6 +37,7 @@ services: DJANGO_DANDI_API_URL: http://localhost:8000 DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org DANDI_ALLOW_LOCALHOST_URLS: "1" + DJANGO_DANDI_DEV_EMAIL: "test@example.com" ports: - "8000:8000" @@ -80,6 +81,7 @@ services: DJANGO_DANDI_API_URL: http://localhost:8000 DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org DANDI_ALLOW_LOCALHOST_URLS: "1" + DJANGO_DANDI_DEV_EMAIL: "test@example.com" minio: image: minio/minio:RELEASE.2022-04-12T06-55-35Z From 58c7c4c558e8662f19c55687d214b5987326d969 Mon Sep 17 00:00:00 2001 From: Austin Macdonald Date: Wed, 1 May 2024 10:49:13 -0500 Subject: [PATCH 20/53] Add example for directory Co-authored-by: Yaroslav Halchenko --- dandi/cli/cmd_download.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index fd62a7c05..647d21951 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -12,20 +12,23 @@ from ..utils import get_instance, joinurl _examples = """ -EXAMPLES: \n -# Download only the dandiset.yaml \n -dandi download --download dandiset.yaml DANDI:000027 \n +EXAMPLES:\n +# Download only the dandiset.yaml\n +dandi download --download dandiset.yaml DANDI:000027\n -# Download only dandiset.yaml if there is a newer version \n +# Download only dandiset.yaml if there is a newer version\n dandi download https://identifiers.org/DANDI:000027 --existing refresh -# Download only the assets \n +# Download only the assets\n dandi download --download assets DANDI:000027 -# Download all from a specific version \n +# Download all from a specific version\n dandi download DANDI:000027/0.210831.2033 -# Download a specific file or directory \n +# Download a specific directory\n +dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/ + +# Download a specific file\n dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/sub-RAT123.nwb """ From f0def0f8d82308d043d9abb0e83a8f38a452885b Mon Sep 17 00:00:00 2001 From: Isaac To Date: Wed, 1 May 2024 13:19:41 -0700 Subject: [PATCH 21/53] Update factory function for creating sample Dandiset for tests This update is for meeting the requirement of having email for a contributor who is a contact person imposed by dandischema, https://github.com/dandi/dandi-schema/pull/235 --- dandi/tests/fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index 2b9a40e89..f27e1dcea 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -548,6 +548,7 @@ def mkdandiset(self, name: str, embargo: bool = False) -> SampleDandiset: { "schemaKey": "Person", "name": "Tests, Dandi-Cli", + "email": "nemo@example.com", "roleName": ["dcite:Author", "dcite:ContactPerson"], } ], From bacef780f97d255ac8a445ad5d2c2c39eca2bf14 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 3 May 2024 09:00:16 -0400 Subject: [PATCH 22/53] [DATALAD RUNCMD] Update also target records to have email in them === Do not change lines below === { "chain": [], "cmd": "sed -i -e 's/^\\( *\\)\\(\"name\": \"Tests, Dandi-Cli\",\\)/\\1\\2\\n\\1\"email\": \"nemo@example.com\",/g' dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json dandi/cli/tests/data/update_dandiset_from_doi/elife.json dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json dandi/cli/tests/data/update_dandiset_from_doi/nature.json dandi/cli/tests/data/update_dandiset_from_doi/neuron.json", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json | 1 + dandi/cli/tests/data/update_dandiset_from_doi/elife.json | 1 + dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json | 1 + dandi/cli/tests/data/update_dandiset_from_doi/nature.json | 1 + dandi/cli/tests/data/update_dandiset_from_doi/neuron.json | 1 + 5 files changed, 5 insertions(+) diff --git a/dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json b/dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json index 6fe83f612..ac14fb425 100644 --- a/dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json +++ b/dandi/cli/tests/data/update_dandiset_from_doi/biorxiv.json @@ -20,6 +20,7 @@ "contributor": [ { "name": "Tests, Dandi-Cli", + "email": "nemo@example.com", "roleName": [ "dcite:Author", "dcite:ContactPerson" diff --git a/dandi/cli/tests/data/update_dandiset_from_doi/elife.json b/dandi/cli/tests/data/update_dandiset_from_doi/elife.json index 54dbf581f..8d8a5e783 100644 --- a/dandi/cli/tests/data/update_dandiset_from_doi/elife.json +++ b/dandi/cli/tests/data/update_dandiset_from_doi/elife.json @@ -20,6 +20,7 @@ "contributor": [ { "name": "Tests, Dandi-Cli", + "email": "nemo@example.com", "roleName": [ "dcite:Author", "dcite:ContactPerson" diff --git a/dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json b/dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json index 14f081ebd..729e185f4 100644 --- a/dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json +++ b/dandi/cli/tests/data/update_dandiset_from_doi/jneurosci.json @@ -20,6 +20,7 @@ "contributor": [ { "name": "Tests, Dandi-Cli", + "email": "nemo@example.com", "roleName": [ "dcite:Author", "dcite:ContactPerson" diff --git a/dandi/cli/tests/data/update_dandiset_from_doi/nature.json b/dandi/cli/tests/data/update_dandiset_from_doi/nature.json index ce5449d09..11461684e 100644 --- a/dandi/cli/tests/data/update_dandiset_from_doi/nature.json +++ b/dandi/cli/tests/data/update_dandiset_from_doi/nature.json @@ -20,6 +20,7 @@ "contributor": [ { "name": "Tests, Dandi-Cli", + "email": "nemo@example.com", "roleName": [ "dcite:Author", "dcite:ContactPerson" diff --git a/dandi/cli/tests/data/update_dandiset_from_doi/neuron.json b/dandi/cli/tests/data/update_dandiset_from_doi/neuron.json index 82a81a7fa..fe2874f42 100644 --- a/dandi/cli/tests/data/update_dandiset_from_doi/neuron.json +++ b/dandi/cli/tests/data/update_dandiset_from_doi/neuron.json @@ -20,6 +20,7 @@ "contributor": [ { "name": "Tests, Dandi-Cli", + "email": "nemo@example.com", "roleName": [ "dcite:Author", "dcite:ContactPerson" From 1a673b92125be1f3e43a7ad6ddf06194199ba8a6 Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Fri, 3 May 2024 13:57:09 +0000 Subject: [PATCH 23/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6085949fb..541915d12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,30 @@ +# 0.62.0 (Fri May 03 2024) + +#### ๐Ÿš€ Enhancement + +- Convert DOI URLs in `related_publications` to related resources [#1417](https://github.com/dandi/dandi-cli/pull/1417) ([@jwodder](https://github.com/jwodder)) + +#### ๐Ÿ› Bug Fix + +- Adjust tests for the added email requirement for contact person [#1438](https://github.com/dandi/dandi-cli/pull/1438) ([@candleindark](https://github.com/candleindark) [@yarikoptic](https://github.com/yarikoptic)) +- Add DJANGO_DANDI_DEV_EMAIL env var to archive tests [#1436](https://github.com/dandi/dandi-cli/pull/1436) ([@jjnesbitt](https://github.com/jjnesbitt)) +- clarify resource identifier docs [#1437](https://github.com/dandi/dandi-cli/pull/1437) ([@asmacdo](https://github.com/asmacdo)) + +#### ๐Ÿงช Tests + +- Fix spelling of `--scheduled` option used in scheduled tests [#1428](https://github.com/dandi/dandi-cli/pull/1428) ([@jwodder](https://github.com/jwodder)) +- Xfail flaky ontobee tests, unless running daily tests [#1423](https://github.com/dandi/dandi-cli/pull/1423) ([@jwodder](https://github.com/jwodder)) + +#### Authors: 5 + +- Austin Macdonald ([@asmacdo](https://github.com/asmacdo)) +- Isaac To ([@candleindark](https://github.com/candleindark)) +- Jacob Nesbitt ([@jjnesbitt](https://github.com/jjnesbitt)) +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # 0.61.2 (Fri Mar 22 2024) #### ๐Ÿงช Tests From fb8bd638746c4c227a314e8256817d1d8e6e9701 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 3 May 2024 10:05:41 -0400 Subject: [PATCH 24/53] insstance typo fix --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d38b5290..85d96fece 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2114,7 +2114,7 @@ - Fixes [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) - Rename DANDI_API_KEY to DANDI_GIRDER_API_KEY [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) - Test of uploading & downloading via new API [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) -- RF+ENH: support mapping for direct API urls, and use netlify insstance instead of api+ prefix [#330](https://github.com/dandi/dandi-cli/pull/330) ([@yarikoptic](https://github.com/yarikoptic)) +- RF+ENH: support mapping for direct API urls, and use netlify instance instead of api+ prefix [#330](https://github.com/dandi/dandi-cli/pull/330) ([@yarikoptic](https://github.com/yarikoptic)) - Delint [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) - RF: account for web UI URL changes/dropped features, remove support for girder URLs [#330](https://github.com/dandi/dandi-cli/pull/330) ([@yarikoptic](https://github.com/yarikoptic)) - Handle uploading already-extant assets [#330](https://github.com/dandi/dandi-cli/pull/330) ([@jwodder](https://github.com/jwodder)) From b477d41aaa0364aff4b3ac38f77e57198ec2650d Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 3 May 2024 14:59:22 -0400 Subject: [PATCH 25/53] Slight tune up to formatting of examples etc to harmonize appearance/make shorter Original --help is too long and a little more inconsistent -- the "#" are shell script construct and generally not used to demarkate anything there, so kind no point to have/look odd --- dandi/cli/cmd_download.py | 37 +++++++++++++++++++------------------ dandi/dandiarchive.py | 16 ++++++++-------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index 647d21951..7a9ebeff7 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -12,24 +12,26 @@ from ..utils import get_instance, joinurl _examples = """ -EXAMPLES:\n -# Download only the dandiset.yaml\n -dandi download --download dandiset.yaml DANDI:000027\n +EXAMPLES: -# Download only dandiset.yaml if there is a newer version\n -dandi download https://identifiers.org/DANDI:000027 --existing refresh - -# Download only the assets\n -dandi download --download assets DANDI:000027 - -# Download all from a specific version\n -dandi download DANDI:000027/0.210831.2033 - -# Download a specific directory\n -dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/ - -# Download a specific file\n -dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/sub-RAT123.nwb +\b + - Download only the dandiset.yaml + dandi download --download dandiset.yaml DANDI:000027 +\b + - Download only dandiset.yaml if there is a newer version + dandi download https://identifiers.org/DANDI:000027 --existing refresh +\b + - Download only the assets + dandi download --download assets DANDI:000027 +\b + - Download all from a specific version + dandi download DANDI:000027/0.210831.2033 +\b + - Download a specific directory + dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/ +\b + - Download a specific file + dandi download dandi://DANDI/000027@0.210831.2033/sub-RAT123/sub-RAT123.nwb """ @@ -45,7 +47,6 @@ \b {_dandi_url_parser.known_patterns} -\b {_examples} """ diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index b931ce871..9c0efd23a 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -676,14 +676,14 @@ class _dandi_url_parser: ), ] resource_identifier_primer = """RESOURCE ID/URLS:\n - dandi commands accept URLs and URL-like identifiers called in the following formats for identifying Dandisets, assets, and - asset collections. - - Text in [brackets] is optional. A server field is a base API or GUI URL - for a DANDI Archive instance. If an optional ``version`` field is - omitted from a URL, the given Dandiset's most recent published version - will be used if it has one, and its draft version will be used otherwise. + dandi commands accept URLs and URL-like identifiers called in the following formats for identifying Dandisets, assets, and + asset collections. + + Text in [brackets] is optional. A server field is a base API or GUI URL + for a DANDI Archive instance. If an optional ``version`` field is + omitted from a URL, the given Dandiset's most recent published version + will be used if it has one, and its draft version will be used otherwise. """ known_patterns = "Accepted resource identifier patterns:" + "\n - ".join( [""] + [display for _, _, display in known_urls] From 659c237e8eabc1754968dab064b93bf1720c0e28 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 3 May 2024 17:24:21 -0400 Subject: [PATCH 26/53] ENH: add timeout of 300 (5 minutes) to any test running We recently started to encounter stalling test runs which lead to hours of stalled operation. Hopefully this would lead to failed test instead of a stall --- setup.cfg | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0e0ebcd8b..773264b1b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -89,6 +89,7 @@ test = pytest-cov pytest-mock pytest-rerunfailures + pytest-timeout responses != 0.24.0 vcrpy tools= diff --git a/tox.ini b/tox.ini index 8c7f6093e..d54eb4439 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,7 @@ changedir = docs commands = sphinx-build -E -W -b html source build [pytest] -addopts = --tb=short --durations=10 +addopts = --tb=short --durations=10 --timeout=300 markers = integration obolibrary From c0df9af77caa259019bb44d7ce0037e6f3fdd1b3 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 14 May 2024 16:11:16 -0400 Subject: [PATCH 27/53] RF: do not require fetching all files for a zarr asset to start initiating downloads Delay assignment of file_qty until it is known, but otherwise proceed to initiating downloads --- dandi/download.py | 48 +++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/dandi/download.py b/dandi/download.py index 79e2f575f..1c4016f47 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -878,33 +878,40 @@ def _download_zarr( # Avoid heavy import by importing within function: from .support.digests import get_zarr_checksum - download_gens = {} - entries = list(asset.iterfiles()) + # we will collect them all while starting the download + # with the first page of entries received from the server. + entries = [] digests = {} + pc = ProgressCombiner(zarr_size=asset.size) def digest_callback(path: str, algoname: str, d: str) -> None: if algoname == "md5": digests[path] = d - for entry in entries: - etag = entry.digest - assert etag.algorithm is DigestType.md5 - download_gens[str(entry)] = _download_file( - entry.get_download_file_iter(), - download_path / str(entry), - toplevel_path=toplevel_path, - size=entry.size, - mtime=entry.modified, - existing=existing, - digests={"md5": etag.value}, - lock=lock, - digest_callback=partial(digest_callback, str(entry)), - ) + def downloads_gen(): + for entry in asset.iterfiles(): + entries.append(entry) + etag = entry.digest + assert etag.algorithm is DigestType.md5 + yield pairing( + str(entry), + _download_file( + entry.get_download_file_iter(), + download_path / str(entry), + toplevel_path=toplevel_path, + size=entry.size, + mtime=entry.modified, + existing=existing, + digests={"md5": etag.value}, + lock=lock, + digest_callback=partial(digest_callback, str(entry)), + ), + ) + pc.file_qty = len(entries) - pc = ProgressCombiner(zarr_size=asset.size, file_qty=len(download_gens)) final_out: dict | None = None with interleave( - [pairing(p, gen) for p, gen in download_gens.items()], + downloads_gen(), onerror=FINISH_CURRENT, max_workers=jobs or 4, ) as it: @@ -988,7 +995,7 @@ class DownloadProgress: @dataclass class ProgressCombiner: zarr_size: int - file_qty: int + file_qty: int = -1 # set to specific known value whenever full sweep is complete files: dict[str, DownloadProgress] = field(default_factory=dict) #: Total size of all files that were not skipped and did not error out #: during download @@ -1032,7 +1039,8 @@ def set_status(self, statusdict: dict) -> None: state_qtys = Counter(s.state for s in self.files.values()) total = len(self.files) if ( - total == self.file_qty + self.file_qty >= 0 # if already known + and total == self.file_qty and state_qtys[DLState.STARTING] == state_qtys[DLState.DOWNLOADING] == 0 ): # All files have finished From 743d034b064f0ae233e16c8f97e5e8e99a53c6d1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 14 May 2024 16:41:23 -0400 Subject: [PATCH 28/53] BF: use total zarr_size to compute done% for zarr Since maxsize is dynamically computed as we go through the files. The idea, I guess, was that it would grow rapidly before actual downloads commense but it is not the case, so we endup with done% being always close to 100% since we get those reports on final downloads completed close to when individual files are downloaded. So this should close https://github.com/dandi/dandi-cli/issues/1407 . But for total zarr file to be used, we needed to account also for skipped files. I added reporting of sizes for skipped files as well. It seems there is no negative side effect on regular files download. So now for the %done of zarr we might be getting to 100% of original size having downloaded nothing. But IMHO it is ok since user does not care as much of how many "subparts" are downloaded, but rather to have adequate progress report back. There also could be side effects if -e skip and we skip download of some updated files which would be smaller than the local ones so altogether we would get over 100% total at the end. --- dandi/download.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/dandi/download.py b/dandi/download.py index 1c4016f47..07d215f5b 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -481,8 +481,8 @@ def agg_done(self, done_sizes: Iterator[int]) -> list[str]: return v -def _skip_file(msg: Any) -> dict: - return {"status": "skipped", "message": str(msg)} +def _skip_file(msg: Any, **kwargs: Any) -> dict: + return dict(**kwargs, status="skipped", message=str(msg)) def _populate_dandiset_yaml( @@ -514,7 +514,7 @@ def _populate_dandiset_yaml( existing is DownloadExisting.REFRESH and os.lstat(dandiset_yaml).st_mtime >= mtime.timestamp() ): - yield _skip_file("already exists") + yield _skip_file("already exists", size=os.lstat(dandiset_yaml).st_mtime) return ds = Dandiset(dandiset_path, allow_empty=True) ds.path_obj.mkdir(exist_ok=True) # exist_ok in case of parallel race @@ -637,7 +637,7 @@ def _download_file( # but mtime is different if same == ["mtime", "size"]: # TODO: add recording and handling of .nwb object_id - yield _skip_file("same time and size") + yield _skip_file("same time and size", size=size) return lgr.debug(f"{path!r} - same attributes: {same}. Redownloading") @@ -1028,11 +1028,17 @@ def get_done(self) -> dict: total_downloaded = sum( s.downloaded for s in self.files.values() - if s.state in (DLState.DOWNLOADING, DLState.CHECKSUM_ERROR, DLState.DONE) + if s.state + in ( + DLState.DOWNLOADING, + DLState.CHECKSUM_ERROR, + DLState.SKIPPED, + DLState.DONE, + ) ) return { "done": total_downloaded, - "done%": total_downloaded / self.maxsize * 100, + "done%": total_downloaded / self.zarr_size * 100, } def set_status(self, statusdict: dict) -> None: @@ -1061,16 +1067,24 @@ def set_status(self, statusdict: dict) -> None: def feed(self, path: str, status: dict) -> Iterator[dict]: keys = list(status.keys()) self.files.setdefault(path, DownloadProgress()) + size = status.get("size") + if size is not None: + if not self.yielded_size: + # this thread will yield + self.yielded_size = True + yield {"size": self.zarr_size} if status.get("status") == "skipped": self.files[path].state = DLState.SKIPPED out = {"message": self.message} + # Treat skipped as "downloaded" for the matter of accounting + if size is not None: + self.files[path].downloaded = size + self.maxsize += size self.set_status(out) yield out + yield self.get_done() elif keys == ["size"]: - if not self.yielded_size: - yield {"size": self.zarr_size} - self.yielded_size = True - self.files[path].size = status["size"] + self.files[path].size = size self.maxsize += status["size"] if any(s.state is DLState.DOWNLOADING for s in self.files.values()): yield self.get_done() From b17aa6358e590cf090269683100d56f1a26af6cf Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 14 May 2024 18:36:10 -0400 Subject: [PATCH 29/53] Fix up test for the changes in prior commit + robustify few places in the modified code logic --- dandi/download.py | 5 +- dandi/tests/test_download.py | 99 +++++++++++++++++++++--------------- 2 files changed, 61 insertions(+), 43 deletions(-) diff --git a/dandi/download.py b/dandi/download.py index 07d215f5b..fc9c71b6e 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -1038,7 +1038,7 @@ def get_done(self) -> dict: ) return { "done": total_downloaded, - "done%": total_downloaded / self.zarr_size * 100, + "done%": total_downloaded / self.zarr_size * 100 if self.zarr_size else 0, } def set_status(self, statusdict: dict) -> None: @@ -1082,7 +1082,8 @@ def feed(self, path: str, status: dict) -> Iterator[dict]: self.maxsize += size self.set_status(out) yield out - yield self.get_done() + if self.zarr_size: + yield self.get_done() elif keys == ["size"]: self.files[path].size = size self.maxsize += status["size"] diff --git a/dandi/tests/test_download.py b/dandi/tests/test_download.py index b6d8f8430..57f0c572a 100644 --- a/dandi/tests/test_download.py +++ b/dandi/tests/test_download.py @@ -485,9 +485,10 @@ def test_download_zarr_subdir_has_only_subdirs( @pytest.mark.parametrize( - "file_qty,inputs,expected", + "zarr_size,file_qty,inputs,expected", [ - ( + ( # 0 + 42, 1, [ ("lonely.txt", {"size": 42}), @@ -501,7 +502,7 @@ def test_download_zarr_subdir_has_only_subdirs( ("lonely.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 42}, {"status": "downloading"}, {"done": 0, "done%": 0.0}, {"done": 20, "done%": 20 / 42 * 100}, @@ -510,7 +511,8 @@ def test_download_zarr_subdir_has_only_subdirs( {"status": "done", "message": "1 done"}, ], ), - ( + ( # 1 + 169, 2, [ ("apple.txt", {"size": 42}), @@ -534,7 +536,7 @@ def test_download_zarr_subdir_has_only_subdirs( ("banana.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 169}, {"status": "downloading"}, {"done": 0, "done%": 0.0}, {"done": 0, "done%": 0.0}, @@ -549,7 +551,8 @@ def test_download_zarr_subdir_has_only_subdirs( {"status": "done", "message": "2 done"}, ], ), - ( + ( # 2 + 169, 2, [ ("apple.txt", {"size": 42}), @@ -573,10 +576,10 @@ def test_download_zarr_subdir_has_only_subdirs( ("banana.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 169}, {"status": "downloading"}, {"done": 0, "done%": 0.0}, - {"done": 20, "done%": 20 / 42 * 100}, + {"done": 20, "done%": 20 / 169 * 100}, {"done": 20, "done%": 20 / 169 * 100}, {"done": 40, "done%": 40 / 169 * 100}, {"done": 42, "done%": 42 / 169 * 100}, @@ -589,7 +592,8 @@ def test_download_zarr_subdir_has_only_subdirs( {"status": "done", "message": "2 done"}, ], ), - ( + ( # 3 + 169, 2, [ ("apple.txt", {"size": 42}), @@ -613,12 +617,12 @@ def test_download_zarr_subdir_has_only_subdirs( ("banana.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 169}, {"status": "downloading"}, {"done": 0, "done%": 0.0}, - {"done": 20, "done%": 20 / 42 * 100}, - {"done": 40, "done%": 40 / 42 * 100}, - {"done": 42, "done%": 42 / 42 * 100}, + {"done": 20, "done%": 20 / 169 * 100}, + {"done": 40, "done%": 40 / 169 * 100}, + {"done": 42, "done%": 42 / 169 * 100}, {"message": "1 done"}, {"done": 42, "done%": 42 / 169 * 100}, {"done": 82, "done%": 82 / 169 * 100}, @@ -628,7 +632,8 @@ def test_download_zarr_subdir_has_only_subdirs( {"status": "done", "message": "2 done"}, ], ), - ( + ( # 4 + 169, 2, [ ("apple.txt", {"size": 42}), @@ -647,7 +652,7 @@ def test_download_zarr_subdir_has_only_subdirs( ("apple.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 169}, {"status": "downloading"}, {"done": 0, "done%": 0.0}, {"done": 0, "done%": 0.0}, @@ -655,21 +660,26 @@ def test_download_zarr_subdir_has_only_subdirs( {"done": 60, "done%": 60 / 169 * 100}, {"done": 80, "done%": 80 / 169 * 100}, {"message": "1 errored"}, - {"done": 40, "done%": 40 / 42 * 100}, - {"done": 42, "done%": 100.0}, + {"done": 40, "done%": 40 / 169 * 100}, + {"done": 42, "done%": 42 / 169 * 100}, {"status": "error", "message": "1 done, 1 errored"}, ], ), - ( + ( # 5 + 0, 1, [("lonely.txt", {"status": "skipped", "message": "already exists"})], [{"status": "skipped", "message": "1 skipped"}], ), - ( + ( # 6 + 169, 2, [ ("apple.txt", {"size": 42}), - ("banana.txt", {"status": "skipped", "message": "already exists"}), + ( + "banana.txt", + {"size": 127, "status": "skipped", "message": "already exists"}, + ), ("apple.txt", {"status": "downloading"}), ("apple.txt", {"done": 0, "done%": 0.0}), ("apple.txt", {"done": 20, "done%": 20 / 42 * 100}), @@ -680,17 +690,19 @@ def test_download_zarr_subdir_has_only_subdirs( ("apple.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 169}, {"message": "1 skipped"}, + {"done": 127, "done%": (127 + 0) / 169 * 100}, {"status": "downloading"}, - {"done": 0, "done%": 0.0}, - {"done": 20, "done%": 20 / 42 * 100}, - {"done": 40, "done%": 40 / 42 * 100}, - {"done": 42, "done%": 100.0}, + {"done": 127 + 0, "done%": (127 + 0) / 169 * 100}, + {"done": 127 + 20, "done%": (127 + 20) / 169 * 100}, + {"done": 127 + 40, "done%": (127 + 40) / 169 * 100}, + {"done": 127 + 42, "done%": 100.0}, {"status": "done", "message": "1 done, 1 skipped"}, ], ), - ( + ( # 7 + 169, 2, [ ("apple.txt", {"size": 42}), @@ -719,7 +731,7 @@ def test_download_zarr_subdir_has_only_subdirs( ("apple.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 169}, {"status": "downloading"}, {"done": 0, "done%": 0.0}, {"done": 0, "done%": 0.0}, @@ -734,14 +746,18 @@ def test_download_zarr_subdir_has_only_subdirs( {"status": "error", "message": "1 done, 1 errored"}, ], ), - ( + ( # 8 + 179, 3, [ ("apple.txt", {"size": 42}), ("banana.txt", {"size": 127}), ("apple.txt", {"status": "downloading"}), ("banana.txt", {"status": "downloading"}), - ("coconut", {"status": "skipped", "message": "already exists"}), + ( + "coconut", + {"size": 10, "status": "skipped", "message": "already exists"}, + ), ("apple.txt", {"done": 0, "done%": 0.0}), ("banana.txt", {"done": 0, "done%": 0.0}), ("apple.txt", {"done": 20, "done%": 20 / 42 * 100}), @@ -764,28 +780,29 @@ def test_download_zarr_subdir_has_only_subdirs( ("banana.txt", {"status": "done"}), ], [ - {"size": 69105}, + {"size": 179}, {"status": "downloading"}, {"message": "1 skipped"}, - {"done": 0, "done%": 0.0}, - {"done": 0, "done%": 0.0}, - {"done": 20, "done%": 20 / 169 * 100}, - {"done": 60, "done%": 60 / 169 * 100}, - {"done": 80, "done%": 80 / 169 * 100}, - {"done": 120, "done%": 120 / 169 * 100}, - {"done": 122, "done%": 122 / 169 * 100}, + {"done": 10, "done%": 10 / 179 * 100}, + {"done": 10, "done%": 10 / 179 * 100}, + {"done": 10, "done%": 10 / 179 * 100}, + {"done": 10 + 20, "done%": (10 + 20) / 179 * 100}, + {"done": 10 + 60, "done%": (10 + 60) / 179 * 100}, + {"done": 10 + 80, "done%": (10 + 80) / 179 * 100}, + {"done": 10 + 120, "done%": (10 + 120) / 179 * 100}, + {"done": 10 + 122, "done%": (10 + 122) / 179 * 100}, {"message": "1 errored, 1 skipped"}, - {"done": 162, "done%": 162 / 169 * 100}, - {"done": 169, "done%": 100.0}, + {"done": 10 + 162, "done%": (10 + 162) / 179 * 100}, + {"done": 179, "done%": 100.0}, {"status": "error", "message": "1 done, 1 errored, 1 skipped"}, ], ), ], ) def test_progress_combiner( - file_qty: int, inputs: list[tuple[str, dict]], expected: list[dict] + zarr_size: int, file_qty: int, inputs: list[tuple[str, dict]], expected: list[dict] ) -> None: - pc = ProgressCombiner(zarr_size=69105, file_qty=file_qty) + pc = ProgressCombiner(zarr_size=zarr_size, file_qty=file_qty) outputs: list[dict] = [] for path, status in inputs: outputs.extend(pc.feed(path, status)) From 08a4050d26321be99159cd76dd03498aafaf06bc Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Mon, 20 May 2024 16:20:42 -0400 Subject: [PATCH 30/53] Code review: minor typing etc recommendations Co-authored-by: John T. Wodder II --- dandi/download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dandi/download.py b/dandi/download.py index fc9c71b6e..d1627af08 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -482,7 +482,7 @@ def agg_done(self, done_sizes: Iterator[int]) -> list[str]: def _skip_file(msg: Any, **kwargs: Any) -> dict: - return dict(**kwargs, status="skipped", message=str(msg)) + return {"status": "skipped", "message": str(msg), **kwargs} def _populate_dandiset_yaml( @@ -995,7 +995,7 @@ class DownloadProgress: @dataclass class ProgressCombiner: zarr_size: int - file_qty: int = -1 # set to specific known value whenever full sweep is complete + file_qty: int | None = None # set to specific known value whenever full sweep is complete files: dict[str, DownloadProgress] = field(default_factory=dict) #: Total size of all files that were not skipped and did not error out #: during download @@ -1045,7 +1045,7 @@ def set_status(self, statusdict: dict) -> None: state_qtys = Counter(s.state for s in self.files.values()) total = len(self.files) if ( - self.file_qty >= 0 # if already known + self.file_qty is not None # if already known and total == self.file_qty and state_qtys[DLState.STARTING] == state_qtys[DLState.DOWNLOADING] == 0 ): From 1910e8a708e22670c9f6f4cfa1f6a3cf8c38f2d9 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 22 May 2024 18:51:39 -0400 Subject: [PATCH 31/53] Rewind filehandle request bodies before retrying requests --- dandi/dandiapi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index a7169c3af..9fb7af431 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -233,6 +233,8 @@ def request( url, result.text, ) + if data is not None and hasattr(data, "seek"): + data.seek(0) result.raise_for_status() except Exception as e: if isinstance(e, requests.HTTPError): From 61a184877f0d16389631228eec14d6bfa3b2ab89 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Thu, 23 May 2024 08:34:18 -0400 Subject: [PATCH 32/53] Separate time components in logfile names with periods --- dandi/cli/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/cli/command.py b/dandi/cli/command.py index bc2ef2e80..87a6c618f 100644 --- a/dandi/cli/command.py +++ b/dandi/cli/command.py @@ -99,7 +99,7 @@ def main(ctx, log_level, pdb=False): logdir = platformdirs.user_log_dir("dandi-cli", "dandi") logfile = os.path.join( - logdir, f"{datetime.now(timezone.utc):%Y%m%d%H%M%SZ}-{os.getpid()}.log" + logdir, f"{datetime.now(timezone.utc):%Y.%m.%d.%H.%M.%SZ}-{os.getpid()}.log" ) os.makedirs(logdir, exist_ok=True) handler = logging.FileHandler(logfile, encoding="utf-8") From a847a9b6ad81a7d073f82e8caf5e697b0020fe50 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Thu, 23 May 2024 10:58:58 -0400 Subject: [PATCH 33/53] Use hyphen to separate date & time --- dandi/cli/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dandi/cli/command.py b/dandi/cli/command.py index 87a6c618f..e278c6f38 100644 --- a/dandi/cli/command.py +++ b/dandi/cli/command.py @@ -99,7 +99,7 @@ def main(ctx, log_level, pdb=False): logdir = platformdirs.user_log_dir("dandi-cli", "dandi") logfile = os.path.join( - logdir, f"{datetime.now(timezone.utc):%Y.%m.%d.%H.%M.%SZ}-{os.getpid()}.log" + logdir, f"{datetime.now(timezone.utc):%Y.%m.%d-%H.%M.%SZ}-{os.getpid()}.log" ) os.makedirs(logdir, exist_ok=True) handler = logging.FileHandler(logfile, encoding="utf-8") From f9eb87c3ca8c0714a68a726893ff21027f783f6c Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Thu, 23 May 2024 22:22:55 +0000 Subject: [PATCH 34/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f07b5ccc4..7ac55d3b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,27 @@ +# 0.62.1 (Thu May 23 2024) + +#### ๐Ÿ› Bug Fix + +- Rewind filehandle request bodies before retrying requests [#1444](https://github.com/dandi/dandi-cli/pull/1444) ([@jwodder](https://github.com/jwodder)) +- Slight tune up to formatting of examples etc to harmonize appearance/make shorter [#1439](https://github.com/dandi/dandi-cli/pull/1439) ([@yarikoptic](https://github.com/yarikoptic)) +- Fix spelling of netlify and insstance [#1433](https://github.com/dandi/dandi-cli/pull/1433) ([@rly](https://github.com/rly) [@yarikoptic](https://github.com/yarikoptic)) + +#### ๐Ÿ  Internal + +- ENH: add/use codespell-project/codespell-problem-matcher to annotate PRs on where typos added [#1429](https://github.com/dandi/dandi-cli/pull/1429) ([@yarikoptic](https://github.com/yarikoptic)) + +#### ๐Ÿงช Tests + +- ENH: add timeout of 300 (5 minutes) to any test running [#1440](https://github.com/dandi/dandi-cli/pull/1440) ([@yarikoptic](https://github.com/yarikoptic)) + +#### Authors: 3 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Ryan Ly ([@rly](https://github.com/rly)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # 0.62.0 (Fri May 03 2024) #### ๐Ÿš€ Enhancement From 0ebbb3585b5e126ed052329b4b63403fe9bbfd06 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 14 Jun 2024 08:39:32 -0400 Subject: [PATCH 35/53] RF test to avoid duplication Duplication leads to bugs and also makes it harder to read the test since requires mental "filtering" to disregard unnecessary duplicated code while ensuring that not missing some critical difference. E.g. I did miss that some have sorted() and some not. Now it is more obvious IMHO. Ideally other tests with similar pattern should be redone as well. Ideally there might be a way to use parametric testing with a common fixture (did not check how/if possible yet) --- dandi/tests/test_dandiapi.py | 50 +++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 38d558da7..1d30a4420 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -7,6 +7,7 @@ import random import re from shutil import rmtree +from typing import Any import anys import click @@ -46,7 +47,7 @@ def test_upload( d.upload_raw_asset(simple1_nwb, {"path": "testing/simple1.nwb"}) (asset,) = d.get_assets() assert asset.path == "testing/simple1.nwb" - d.download_directory("", tmp_path) + d.download_directory("./", tmp_path) paths = list_paths(tmp_path) assert paths == [tmp_path / "testing" / "simple1.nwb"] assert paths[0].stat().st_size == simple1_nwb.stat().st_size @@ -678,26 +679,33 @@ def test_get_assets_order(text_dandiset: SampleDandiset) -> None: def test_get_assets_with_path_prefix(text_dandiset: SampleDandiset) -> None: - assert sorted( - asset.path - for asset in text_dandiset.dandiset.get_assets_with_path_prefix("subdir") - ) == ["subdir1/apple.txt", "subdir2/banana.txt", "subdir2/coconut.txt"] - assert sorted( - asset.path - for asset in text_dandiset.dandiset.get_assets_with_path_prefix("subdir2") - ) == ["subdir2/banana.txt", "subdir2/coconut.txt"] - assert [ - asset.path - for asset in text_dandiset.dandiset.get_assets_with_path_prefix( - "subdir", order="path" - ) - ] == ["subdir1/apple.txt", "subdir2/banana.txt", "subdir2/coconut.txt"] - assert [ - asset.path - for asset in text_dandiset.dandiset.get_assets_with_path_prefix( - "subdir", order="-path" - ) - ] == ["subdir2/coconut.txt", "subdir2/banana.txt", "subdir1/apple.txt"] + def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: + return [ + asset.path + for asset in text_dandiset.dandiset.get_assets_with_path_prefix( + prefix, **kw + ) + ] + + assert sorted(_get_assets_with_path_prefix("subdir")) == [ + "subdir1/apple.txt", + "subdir2/banana.txt", + "subdir2/coconut.txt", + ] + assert sorted(_get_assets_with_path_prefix("subdir2")) == [ + "subdir2/banana.txt", + "subdir2/coconut.txt", + ] + assert _get_assets_with_path_prefix("subdir", order="path") == [ + "subdir1/apple.txt", + "subdir2/banana.txt", + "subdir2/coconut.txt", + ] + assert _get_assets_with_path_prefix("subdir", order="-path") == [ + "subdir2/coconut.txt", + "subdir2/banana.txt", + "subdir1/apple.txt", + ] def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: From 4a318809daf3e6ca23b619b82ee7d516c102c5f1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 14 Jun 2024 08:46:26 -0400 Subject: [PATCH 36/53] BF: normalize path before passing to API for querying assets What we call "path" is really just a prefix for dandi-archive. To ensure more consistent operation for a user, let's normalize those paths, even though normalization would seemingly allow for obnoxious cases like "nonexistent/../realfolder/". According to https://github.com/dandi/dandi-cli/issues/1452 it is likely that we had operation more robust at server-side before and were accepting "./" whenever now it just returns an empty list. Not sure if worth seeking change at dandi-archive level ATM. --- dandi/dandiapi.py | 27 ++++++++++++++++++++++++++- dandi/tests/test_dandiapi.py | 34 +++++++++++++++++++++++++--------- 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 9fb7af431..af2a67b35 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -937,6 +937,29 @@ def from_data(cls, client: DandiAPIClient, data: dict[str, Any]) -> RemoteDandis client=client, identifier=data["identifier"], version=version, data=data ) + @staticmethod + def _normalize_path(path: str) -> str: + """ + Helper to normalize path before passing it to the server. + + We and API call it "path" but it is really a "prefix" with inherent + semantics of containing directory divider '/' and emphasizing with + trailing '/' to point to a directory. + """ + # Server (now) expects path to be a proper prefix, so to account for user + # possibly specifying ./ or some other relative paths etc, let's normalize + # the path. + # Ref: https://github.com/dandi/dandi-cli/issues/1452 + path_normed = os.path.normpath(path) + if path_normed == ".": + path_normed = "" + elif path.endswith("/"): + # we need to make sure that we have a trailing slash if we had it before + path_normed += "/" + if path_normed != path: + lgr.debug("Normalized path %r to %r", path, path_normed) + return path_normed + def json_dict(self) -> dict[str, Any]: """ Convert to a JSONable `dict`, omitting the ``client`` attribute and @@ -1154,6 +1177,8 @@ def get_assets_with_path_prefix( Returns an iterator of all assets in this version of the Dandiset whose `~RemoteAsset.path` attributes start with ``path`` + ``path`` is normalized first to possibly remove leading `./` or relative + paths (e.g., `../`) within it. Assets can be sorted by a given field by passing the name of that field as the ``order`` parameter. The accepted field names are ``"created"``, ``"modified"``, and ``"path"``. Prepend a hyphen to the @@ -1162,7 +1187,7 @@ def get_assets_with_path_prefix( try: for a in self.client.paginate( f"{self.version_api_path}assets/", - params={"path": path, "order": order}, + params={"path": self._normalize_path(path), "order": order}, ): yield RemoteAsset.from_data(self, a) except HTTP404Error: diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 1d30a4420..d445ea90f 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -687,15 +687,31 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: ) ] - assert sorted(_get_assets_with_path_prefix("subdir")) == [ - "subdir1/apple.txt", - "subdir2/banana.txt", - "subdir2/coconut.txt", - ] - assert sorted(_get_assets_with_path_prefix("subdir2")) == [ - "subdir2/banana.txt", - "subdir2/coconut.txt", - ] + assert ( + sorted(_get_assets_with_path_prefix("subdir")) + == sorted(_get_assets_with_path_prefix("./subdir")) + == sorted(_get_assets_with_path_prefix("./subdir2/../sub")) + == [ + "subdir1/apple.txt", + "subdir2/banana.txt", + "subdir2/coconut.txt", + ] + ) + assert ( + _get_assets_with_path_prefix("subdir/") + == _get_assets_with_path_prefix("./subdir/") + == _get_assets_with_path_prefix("../subdir1/") + == [] + ) + assert ( + sorted(_get_assets_with_path_prefix("subdir2")) + == sorted(_get_assets_with_path_prefix("a/../subdir2")) + == sorted(_get_assets_with_path_prefix("./subdir2")) + == [ + "subdir2/banana.txt", + "subdir2/coconut.txt", + ] + ) assert _get_assets_with_path_prefix("subdir", order="path") == [ "subdir1/apple.txt", "subdir2/banana.txt", From c291755da9df976ebc9b6ba953953a34ec4b5493 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 14 Jun 2024 09:35:53 -0400 Subject: [PATCH 37/53] Address comments from review: use posixpath and duplicate doc entry --- dandi/dandiapi.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index af2a67b35..f2a7f4750 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -10,6 +10,7 @@ import json import os.path from pathlib import Path, PurePosixPath +import posixpath import re from time import sleep, time from types import TracebackType @@ -943,14 +944,14 @@ def _normalize_path(path: str) -> str: Helper to normalize path before passing it to the server. We and API call it "path" but it is really a "prefix" with inherent - semantics of containing directory divider '/' and emphasizing with - trailing '/' to point to a directory. + semantics of containing directory divider '/' and referring to a + directory when terminated with '/'. """ # Server (now) expects path to be a proper prefix, so to account for user # possibly specifying ./ or some other relative paths etc, let's normalize # the path. # Ref: https://github.com/dandi/dandi-cli/issues/1452 - path_normed = os.path.normpath(path) + path_normed = posixpath.normpath(path) if path_normed == ".": path_normed = "" elif path.endswith("/"): @@ -1177,8 +1178,9 @@ def get_assets_with_path_prefix( Returns an iterator of all assets in this version of the Dandiset whose `~RemoteAsset.path` attributes start with ``path`` - ``path`` is normalized first to possibly remove leading `./` or relative - paths (e.g., `../`) within it. + ``path`` is normalized first to possibly remove leading ``./`` or relative + paths (e.g., ``../``) within it. + Assets can be sorted by a given field by passing the name of that field as the ``order`` parameter. The accepted field names are ``"created"``, ``"modified"``, and ``"path"``. Prepend a hyphen to the @@ -1225,7 +1227,11 @@ def get_asset_by_path(self, path: str) -> RemoteAsset: Fetch the asset in this version of the Dandiset whose `~RemoteAsset.path` equals ``path``. If the given asset does not exist, a `NotFoundError` is raised. + + ``path`` is normalized first to possibly remove leading ``./`` or relative + paths (e.g., ``../``) within it. """ + path = self._normalize_path(path) try: # Weed out any assets that happen to have the given path as a # proper prefix: @@ -1246,6 +1252,9 @@ def download_directory( """ Download all assets under the virtual directory ``assets_dirpath`` to the directory ``dirpath``. Downloads are synchronous. + + ``path`` is normalized first to possibly remove leading ``./`` or relative + paths (e.g., ``../``) within it. """ if assets_dirpath and not assets_dirpath.endswith("/"): assets_dirpath += "/" From 0aabf1626f47800734d54bf7f5bd098634b5e5f1 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Fri, 14 Jun 2024 10:20:29 -0400 Subject: [PATCH 38/53] BF: fix download_directory for side-effect of normalizing paths while getting listing --- dandi/dandiapi.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index f2a7f4750..5ea504de7 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1258,6 +1258,9 @@ def download_directory( """ if assets_dirpath and not assets_dirpath.endswith("/"): assets_dirpath += "/" + # need to normalize explicitly since we do use it below also + # to deduce portion of the path to strip off. + assets_dirpath = self._normalize_path(assets_dirpath) assets = list(self.get_assets_with_path_prefix(assets_dirpath)) for a in assets: filepath = Path(dirpath, a.path[len(assets_dirpath) :]) From e2d963d8207735b2b45d61652e7139c0a667e99c Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Fri, 14 Jun 2024 14:55:24 +0000 Subject: [PATCH 39/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ac55d3b9..ccf0d66bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +# 0.62.2 (Fri Jun 14 2024) + +#### ๐Ÿ› Bug Fix + +- Normalize path while requesting list of assets from the server [#1454](https://github.com/dandi/dandi-cli/pull/1454) ([@yarikoptic](https://github.com/yarikoptic)) +- OPT+RF of zarr downloads: do not wait for full files listing + compute %done from total zarr size [#1443](https://github.com/dandi/dandi-cli/pull/1443) ([@yarikoptic](https://github.com/yarikoptic)) +- Separate datetime components in logfile names with punctuation [#1445](https://github.com/dandi/dandi-cli/pull/1445) ([@jwodder](https://github.com/jwodder)) + +#### Authors: 2 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) +- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic)) + +--- + # 0.62.1 (Thu May 23 2024) #### ๐Ÿ› Bug Fix From ccf43297da2c505dc6bba21c8b551a0404d7e234 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Fri, 12 Jul 2024 08:25:05 -0400 Subject: [PATCH 40/53] Temporarily restrict dandischema requirement to `< 0.10.2` --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 773264b1b..c8ed6dcef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,9 @@ install_requires = bidsschematools ~= 0.7.0 click >= 7.1 click-didyoumean - dandischema >= 0.9.0, < 0.11 + # Don't allow v0.10.2 of the schema until + # is resolved: + dandischema >= 0.9.0, < 0.10.2 etelemetry >= 0.2.2 fasteners fscacher >= 0.3.0 From 547d7b8bcc8fde161710b06268c1039a7c3ea143 Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Fri, 12 Jul 2024 19:35:44 +0000 Subject: [PATCH 41/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf0d66bc..de99b1b34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 0.62.3 (Fri Jul 12 2024) + +#### ๐Ÿ”ฉ Dependency Updates + +- Temporarily restrict dandischema requirement to `< 0.10.2` [#1458](https://github.com/dandi/dandi-cli/pull/1458) ([@jwodder](https://github.com/jwodder)) + +#### Authors: 1 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) + +--- + # 0.62.2 (Fri Jun 14 2024) #### ๐Ÿ› Bug Fix From ead3d6efc2ae8afdad4120b633c9c86c271a1639 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 15 Jul 2024 15:24:22 -0400 Subject: [PATCH 42/53] Revert "Temporarily restrict dandischema requirement to `< 0.10.2`" This reverts commit ccf43297da2c505dc6bba21c8b551a0404d7e234. --- setup.cfg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index c8ed6dcef..773264b1b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,9 +33,7 @@ install_requires = bidsschematools ~= 0.7.0 click >= 7.1 click-didyoumean - # Don't allow v0.10.2 of the schema until - # is resolved: - dandischema >= 0.9.0, < 0.10.2 + dandischema >= 0.9.0, < 0.11 etelemetry >= 0.2.2 fasteners fscacher >= 0.3.0 From e975d9a039c9fa1e7f61b0bce7ebcce2c9ec1e53 Mon Sep 17 00:00:00 2001 From: DANDI Bot Date: Mon, 15 Jul 2024 20:44:41 +0000 Subject: [PATCH 43/53] Update CHANGELOG.md [skip ci] --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de99b1b34..8483df856 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +# 0.62.4 (Mon Jul 15 2024) + +#### ๐Ÿ”ฉ Dependency Updates + +- Revert "Temporarily restrict dandischema requirement to `< 0.10.2`" [#1459](https://github.com/dandi/dandi-cli/pull/1459) ([@jwodder](https://github.com/jwodder)) + +#### Authors: 1 + +- John T. Wodder II ([@jwodder](https://github.com/jwodder)) + +--- + # 0.62.3 (Fri Jul 12 2024) #### ๐Ÿ”ฉ Dependency Updates From 21b696f9c32faa72e9d95af12b6f3eaf7f2a8fbd Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 23 Jul 2024 09:52:46 -0400 Subject: [PATCH 44/53] Do not use mypy 1.11.0 Broken on Python 3.8: https://github.com/python/mypy/pull/17543 --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d54eb4439..064c68daa 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ commands = [testenv:typing] deps = - mypy + mypy != 1.11.0 types-python-dateutil types-requests commands = From ea0edc1bac34f20e63c4f17b57489fed655ffeda Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 23 Jul 2024 08:59:25 -0400 Subject: [PATCH 45/53] Remove duplicate `DJANGO_DANDI_DEV_EMAIL` key in `docker-compose.yml` --- dandi/tests/data/dandiarchive-docker/docker-compose.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index c128113ce..1e557223d 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -36,9 +36,8 @@ services: DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 DJANGO_DANDI_API_URL: http://localhost:8000 DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org - DJANGO_DANDI_DEV_EMAIL: test@example.com - DANDI_ALLOW_LOCALHOST_URLS: "1" DJANGO_DANDI_DEV_EMAIL: "test@example.com" + DANDI_ALLOW_LOCALHOST_URLS: "1" ports: - "8000:8000" @@ -81,8 +80,8 @@ services: DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 DJANGO_DANDI_API_URL: http://localhost:8000 DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org - DANDI_ALLOW_LOCALHOST_URLS: "1" DJANGO_DANDI_DEV_EMAIL: "test@example.com" + DANDI_ALLOW_LOCALHOST_URLS: "1" minio: image: minio/minio:RELEASE.2022-04-12T06-55-35Z From f3d2f7f055c5dccf1e708642a7fc44d48e9ec2fa Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 23 Jul 2024 09:56:40 -0400 Subject: [PATCH 46/53] Secure Docker Compose ports --- .../data/dandiarchive-docker/docker-compose.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 1e557223d..5a9e75d85 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -39,7 +39,7 @@ services: DJANGO_DANDI_DEV_EMAIL: "test@example.com" DANDI_ALLOW_LOCALHOST_URLS: "1" ports: - - "8000:8000" + - "127.0.0.1:8000:8000" celery: image: dandiarchive/dandiarchive-api @@ -89,7 +89,7 @@ services: tty: true command: ["server", "/data"] ports: - - "9000:9000" + - "127.0.0.1:9000:9000" environment: MINIO_ACCESS_KEY: minioAccessKey MINIO_SECRET_KEY: minioSecretKey @@ -104,8 +104,8 @@ services: POSTGRES_DB: django POSTGRES_PASSWORD: postgres image: postgres - ports: - - "5432:5432" + expose: + - "5432" healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] interval: 7s @@ -114,5 +114,5 @@ services: rabbitmq: image: rabbitmq:management - ports: - - "5672:5672" + expose: + - "5672" From 76e7d14f060bd199a0fa843b6515cab351557ca4 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 23 Jul 2024 09:58:14 -0400 Subject: [PATCH 47/53] Switch from `docker-compose` to `docker compose` --- .github/workflows/test.yml | 4 ++-- dandi/tests/fixtures.py | 31 ++++++++++++++++++++++++------- dandi/tests/skip.py | 2 +- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d8089a536..9e940a6ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,14 +115,14 @@ jobs: - name: Dump Docker Compose logs if: failure() && startsWith(matrix.os, 'ubuntu') run: | - docker-compose \ + docker compose \ -f dandi/tests/data/dandiarchive-docker/docker-compose.yml \ logs --timestamps - name: Shut down Docker Compose if: startsWith(matrix.os, 'ubuntu') run: | - docker-compose \ + docker compose \ -f dandi/tests/data/dandiarchive-docker/docker-compose.yml \ down -v diff --git a/dandi/tests/fixtures.py b/dandi/tests/fixtures.py index f27e1dcea..9fe097154 100644 --- a/dandi/tests/fixtures.py +++ b/dandi/tests/fixtures.py @@ -395,16 +395,27 @@ def docker_compose_setup() -> Iterator[dict[str, str]]: try: if create: if os.environ.get("DANDI_TESTS_PULL_DOCKER_COMPOSE", "1") not in ("", "0"): - run(["docker-compose", "pull"], cwd=str(LOCAL_DOCKER_DIR), check=True) + run( + ["docker", "compose", "pull"], cwd=str(LOCAL_DOCKER_DIR), check=True + ) run( - ["docker-compose", "run", "--rm", "django", "./manage.py", "migrate"], + [ + "docker", + "compose", + "run", + "--rm", + "django", + "./manage.py", + "migrate", + ], cwd=str(LOCAL_DOCKER_DIR), env=env, check=True, ) run( [ - "docker-compose", + "docker", + "compose", "run", "--rm", "django", @@ -417,7 +428,8 @@ def docker_compose_setup() -> Iterator[dict[str, str]]: ) run( [ - "docker-compose", + "docker", + "compose", "run", "--rm", "-e", @@ -436,7 +448,8 @@ def docker_compose_setup() -> Iterator[dict[str, str]]: r = check_output( [ - "docker-compose", + "docker", + "compose", "run", "--rm", "-T", @@ -458,7 +471,7 @@ def docker_compose_setup() -> Iterator[dict[str, str]]: if create: run( - ["docker-compose", "up", "-d", "django", "celery"], + ["docker", "compose", "up", "-d", "django", "celery"], cwd=str(LOCAL_DOCKER_DIR), env=env, check=True, @@ -476,7 +489,11 @@ def docker_compose_setup() -> Iterator[dict[str, str]]: yield {"django_api_key": django_api_key} finally: if persist in (None, "0"): - run(["docker-compose", "down", "-v"], cwd=str(LOCAL_DOCKER_DIR), check=True) + run( + ["docker", "compose", "down", "-v"], + cwd=str(LOCAL_DOCKER_DIR), + check=True, + ) @dataclass diff --git a/dandi/tests/skip.py b/dandi/tests/skip.py index 0e08b5f99..54d864b76 100644 --- a/dandi/tests/skip.py +++ b/dandi/tests/skip.py @@ -101,7 +101,7 @@ def windows(): def no_docker_commands(): missing_cmds = [] - for cmd in ("docker", "docker-compose"): + for cmd in ("docker",): if shutil.which(cmd) is None: missing_cmds.append(cmd) msg = "missing Docker commands: {}".format(", ".join(missing_cmds)) From c583a8993d72dfe6ecbcc78c501303f495b5e8ae Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 23 Jul 2024 14:32:22 -0400 Subject: [PATCH 48/53] Add `--preserve-tree` option to `dandi download` --- dandi/cli/cmd_download.py | 15 +++++++++-- dandi/cli/tests/test_download.py | 7 +++++ dandi/dandiarchive.py | 42 +++++++++++++++++++++-------- dandi/download.py | 28 +++++++++++++------- dandi/tests/test_download.py | 45 +++++++++++++++++++++++++++++++- docs/source/cmdline/download.rst | 6 +++++ 6 files changed, 120 insertions(+), 23 deletions(-) diff --git a/dandi/cli/cmd_download.py b/dandi/cli/cmd_download.py index 7a9ebeff7..32c1ff66d 100644 --- a/dandi/cli/cmd_download.py +++ b/dandi/cli/cmd_download.py @@ -100,6 +100,15 @@ default="all", show_default=True, ) +@click.option( + "--preserve-tree", + is_flag=True, + help=( + "When downloading only part of a Dandiset, also download" + " `dandiset.yaml` and do not strip leading directories from asset" + " paths. Implies `--download all`." + ), +) @click.option( "--sync", is_flag=True, help="Delete local assets that do not exist on the server" ) @@ -138,6 +147,7 @@ def download( sync: bool, dandi_instance: str, path_type: PathType, + preserve_tree: bool, ) -> None: # We need to import the download module rather than the download function # so that the tests can properly patch the function with a mock. @@ -171,8 +181,9 @@ def download( format=format, jobs=jobs[0], jobs_per_zarr=jobs[1], - get_metadata="dandiset.yaml" in download_types, - get_assets="assets" in download_types, + get_metadata="dandiset.yaml" in download_types or preserve_tree, + get_assets="assets" in download_types or preserve_tree, + preserve_tree=preserve_tree, sync=sync, path_type=path_type, # develop_debug=develop_debug diff --git a/dandi/cli/tests/test_download.py b/dandi/cli/tests/test_download.py index 0a8131cff..4f10227c5 100644 --- a/dandi/cli/tests/test_download.py +++ b/dandi/cli/tests/test_download.py @@ -23,6 +23,7 @@ def test_download_defaults(mocker): jobs_per_zarr=None, get_metadata=True, get_assets=True, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) @@ -41,6 +42,7 @@ def test_download_all_types(mocker): jobs_per_zarr=None, get_metadata=True, get_assets=True, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) @@ -59,6 +61,7 @@ def test_download_metadata_only(mocker): jobs_per_zarr=None, get_metadata=True, get_assets=False, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) @@ -77,6 +80,7 @@ def test_download_assets_only(mocker): jobs_per_zarr=None, get_metadata=False, get_assets=True, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) @@ -110,6 +114,7 @@ def test_download_gui_instance_in_dandiset(mocker): jobs_per_zarr=None, get_metadata=True, get_assets=True, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) @@ -135,6 +140,7 @@ def test_download_api_instance_in_dandiset(mocker): jobs_per_zarr=None, get_metadata=True, get_assets=True, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) @@ -160,6 +166,7 @@ def test_download_url_instance_match(mocker): jobs_per_zarr=None, get_metadata=True, get_assets=True, + preserve_tree=False, sync=False, path_type=PathType.EXACT, ) diff --git a/dandi/dandiarchive.py b/dandi/dandiarchive.py index 9c0efd23a..40eb5fb3f 100644 --- a/dandi/dandiarchive.py +++ b/dandi/dandiarchive.py @@ -183,11 +183,17 @@ def navigate( yield (client, dandiset, assets) @abstractmethod - def get_asset_download_path(self, asset: BaseRemoteAsset) -> str: + def get_asset_download_path( + self, asset: BaseRemoteAsset, preserve_tree: bool + ) -> str: """ Returns the path (relative to the base download directory) at which the asset ``asset`` (assumed to have been returned by this object's - `get_assets()` method) should be downloaded + `get_assets()` method) should be downloaded. + + If ``preserve_tree`` is `True`, then the download is being performed + with ``--download tree`` option, and the method's return value should + be adjusted accordingly. :meta private: """ @@ -231,7 +237,9 @@ def get_assets( assert d is not None yield from d.get_assets(order=order) - def get_asset_download_path(self, asset: BaseRemoteAsset) -> str: + def get_asset_download_path( + self, asset: BaseRemoteAsset, preserve_tree: bool + ) -> str: return asset.path.lstrip("/") def is_under_download_path(self, path: str) -> bool: @@ -242,13 +250,17 @@ def is_under_download_path(self, path: str) -> bool: class SingleAssetURL(ParsedDandiURL): """Superclass for parsed URLs that refer to a single asset""" - def get_asset_download_path(self, asset: BaseRemoteAsset) -> str: - return posixpath.basename(asset.path.lstrip("/")) + def get_asset_download_path( + self, asset: BaseRemoteAsset, preserve_tree: bool + ) -> str: + path = asset.path.lstrip("/") + if preserve_tree: + return path + else: + return posixpath.basename(path) def is_under_download_path(self, path: str) -> bool: - raise TypeError( - f"{type(self).__name__}.is_under_download_path() should not be called" - ) + return False @dataclass @@ -257,8 +269,14 @@ class MultiAssetURL(ParsedDandiURL): path: str - def get_asset_download_path(self, asset: BaseRemoteAsset) -> str: - return multiasset_target(self.path, asset.path.lstrip("/")) + def get_asset_download_path( + self, asset: BaseRemoteAsset, preserve_tree: bool + ) -> str: + path = asset.path.lstrip("/") + if preserve_tree: + return path + else: + return multiasset_target(self.path, path) def is_under_download_path(self, path: str) -> bool: prefix = posixpath.dirname(self.path.strip("/")) @@ -487,7 +505,9 @@ def get_assets( if strict and not any_assets: raise NotFoundError(f"No assets found matching glob {self.path!r}") - def get_asset_download_path(self, asset: BaseRemoteAsset) -> str: + def get_asset_download_path( + self, asset: BaseRemoteAsset, preserve_tree: bool + ) -> str: return asset.path.lstrip("/") def is_under_download_path(self, path: str) -> bool: diff --git a/dandi/download.py b/dandi/download.py index d1627af08..2df711c78 100644 --- a/dandi/download.py +++ b/dandi/download.py @@ -93,6 +93,7 @@ def download( jobs_per_zarr: int | None = None, get_metadata: bool = True, get_assets: bool = True, + preserve_tree: bool = False, sync: bool = False, path_type: PathType = PathType.EXACT, ) -> None: @@ -141,6 +142,7 @@ def download( existing=existing, get_metadata=get_metadata, get_assets=get_assets, + preserve_tree=preserve_tree, jobs_per_zarr=jobs_per_zarr, on_error="yield" if format is DownloadFormat.PYOUT else "raise", **kw, @@ -201,6 +203,7 @@ class Downloader: existing: DownloadExisting get_metadata: bool get_assets: bool + preserve_tree: bool jobs_per_zarr: int | None on_error: Literal["raise", "yield"] #: which will be set .gen to assets. Purpose is to make it possible to get @@ -214,19 +217,24 @@ def __post_init__(self, output_dir: str | Path) -> None: # TODO: if we are ALREADY in a dandiset - we can validate that it is # the same dandiset and use that dandiset path as the one to download # under - if isinstance(self.url, DandisetURL): + if isinstance(self.url, DandisetURL) or ( + self.preserve_tree and self.url.dandiset_id is not None + ): assert self.url.dandiset_id is not None self.output_prefix = Path(self.url.dandiset_id) else: self.output_prefix = Path() self.output_path = Path(output_dir, self.output_prefix) + def is_dandiset_yaml(self) -> bool: + return isinstance(self.url, AssetItemURL) and self.url.path == "dandiset.yaml" + def download_generator(self) -> Iterator[dict]: """ A generator for downloads of files, folders, or entire dandiset from DANDI (as identified by URL) - This function is a generator which would yield records on ongoing + This function is a generator which yields records on ongoing activities. Activities include traversal of the remote resource (DANDI archive), download of individual assets while yielding records (TODO: schema) while validating their checksums "on the fly", etc. @@ -235,10 +243,8 @@ def download_generator(self) -> Iterator[dict]: with self.url.navigate(strict=True) as (client, dandiset, assets): if ( isinstance(self.url, DandisetURL) - or ( - isinstance(self.url, AssetItemURL) - and self.url.path == "dandiset.yaml" - ) + or self.is_dandiset_yaml() + or self.preserve_tree ) and self.get_metadata: assert dandiset is not None for resp in _populate_dandiset_yaml( @@ -248,7 +254,7 @@ def download_generator(self) -> Iterator[dict]: "path": str(self.output_prefix / dandiset_metadata_file), **resp, } - if isinstance(self.url, AssetItemURL): + if self.is_dandiset_yaml(): return # TODO: do analysis of assets for early detection of needed renames @@ -262,7 +268,9 @@ def download_generator(self) -> Iterator[dict]: assets = self.assets_it.feed(assets) lock = Lock() for asset in assets: - path = self.url.get_asset_download_path(asset) + path = self.url.get_asset_download_path( + asset, preserve_tree=self.preserve_tree + ) self.asset_download_paths.add(path) download_path = Path(self.output_path, path) path = str(self.output_prefix / path) @@ -995,7 +1003,9 @@ class DownloadProgress: @dataclass class ProgressCombiner: zarr_size: int - file_qty: int | None = None # set to specific known value whenever full sweep is complete + file_qty: int | None = ( + None # set to specific known value whenever full sweep is complete + ) files: dict[str, DownloadProgress] = field(default_factory=dict) #: Total size of all files that were not skipped and did not error out #: during download diff --git a/dandi/tests/test_download.py b/dandi/tests/test_download.py index 57f0c572a..2a8c044ce 100644 --- a/dandi/tests/test_download.py +++ b/dandi/tests/test_download.py @@ -172,6 +172,28 @@ def test_download_folder(text_dandiset: SampleDandiset, tmp_path: Path) -> None: assert (tmp_path / "subdir2" / "coconut.txt").read_text() == "Coconut\n" +def test_download_folder_preserve_tree( + text_dandiset: SampleDandiset, tmp_path: Path +) -> None: + dandiset_id = text_dandiset.dandiset_id + download( + f"dandi://{text_dandiset.api.instance_id}/{dandiset_id}/subdir2/", + tmp_path, + preserve_tree=True, + ) + assert list_paths(tmp_path, dirs=True) == [ + tmp_path / dandiset_id, + tmp_path / dandiset_id / "dandiset.yaml", + tmp_path / dandiset_id / "subdir2", + tmp_path / dandiset_id / "subdir2" / "banana.txt", + tmp_path / dandiset_id / "subdir2" / "coconut.txt", + ] + assert (tmp_path / dandiset_id / "subdir2" / "banana.txt").read_text() == "Banana\n" + assert ( + tmp_path / dandiset_id / "subdir2" / "coconut.txt" + ).read_text() == "Coconut\n" + + def test_download_item(text_dandiset: SampleDandiset, tmp_path: Path) -> None: dandiset_id = text_dandiset.dandiset_id download( @@ -182,6 +204,26 @@ def test_download_item(text_dandiset: SampleDandiset, tmp_path: Path) -> None: assert (tmp_path / "coconut.txt").read_text() == "Coconut\n" +def test_download_item_preserve_tree( + text_dandiset: SampleDandiset, tmp_path: Path +) -> None: + dandiset_id = text_dandiset.dandiset_id + download( + f"dandi://{text_dandiset.api.instance_id}/{dandiset_id}/subdir2/coconut.txt", + tmp_path, + preserve_tree=True, + ) + assert list_paths(tmp_path, dirs=True) == [ + tmp_path / dandiset_id, + tmp_path / dandiset_id / "dandiset.yaml", + tmp_path / dandiset_id / "subdir2", + tmp_path / dandiset_id / "subdir2" / "coconut.txt", + ] + assert ( + tmp_path / dandiset_id / "subdir2" / "coconut.txt" + ).read_text() == "Coconut\n" + + def test_download_dandiset_yaml(text_dandiset: SampleDandiset, tmp_path: Path) -> None: dandiset_id = text_dandiset.dandiset_id download( @@ -330,6 +372,7 @@ def test_download_metadata404(text_dandiset: SampleDandiset, tmp_path: Path) -> existing=DownloadExisting.ERROR, get_metadata=True, get_assets=True, + preserve_tree=False, jobs_per_zarr=None, on_error="raise", ).download_generator() @@ -985,4 +1028,4 @@ def test_pyouthelper_time_remaining_1339(): # once done, dont print ETA assert len(done) == 2 else: - assert done[-1] == f"ETA: {10-i} seconds<" + assert done[-1] == f"ETA: {10 - i} seconds<" diff --git a/docs/source/cmdline/download.rst b/docs/source/cmdline/download.rst index e068194a4..b0c6a3760 100644 --- a/docs/source/cmdline/download.rst +++ b/docs/source/cmdline/download.rst @@ -46,6 +46,12 @@ Options Whether to interpret asset paths in URLs as exact matches or glob patterns +.. option:: --preserve-tree + + When downloading only part of a Dandiset, also download + :file:`dandiset.yaml` and do not strip leading directories from asset + paths. Implies ``--download all``. + .. option:: --sync Delete local assets that do not exist on the server after downloading From bd23dfbf9df2af21d005293c493a244a2f83ea6e Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 24 Jul 2024 08:56:47 -0400 Subject: [PATCH 49/53] docker-compose.yml: Remove obsolete `version` key --- dandi/tests/data/dandiarchive-docker/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 5a9e75d85..681d6691d 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -4,8 +4,6 @@ # , # but using images uploaded to Docker Hub instead of building them locally. -version: '2.1' - services: django: image: dandiarchive/dandiarchive-api From c6f369f12ab41f0abcbb23e4a2fafb8b6b1e8c82 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Wed, 24 Jul 2024 08:58:39 -0400 Subject: [PATCH 50/53] docker-compose.yml: Deduplicate django & celery envvars --- .../dandiarchive-docker/docker-compose.yml | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/dandi/tests/data/dandiarchive-docker/docker-compose.yml b/dandi/tests/data/dandiarchive-docker/docker-compose.yml index 681d6691d..e4ba36c56 100644 --- a/dandi/tests/data/dandiarchive-docker/docker-compose.yml +++ b/dandi/tests/data/dandiarchive-docker/docker-compose.yml @@ -17,7 +17,7 @@ services: condition: service_healthy rabbitmq: condition: service_started - environment: + environment: &django_env DJANGO_CELERY_BROKER_URL: amqp://rabbitmq:5672/ DJANGO_CONFIGURATION: DevelopmentConfiguration DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-dandisets @@ -30,7 +30,7 @@ services: DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey DJANGO_STORAGE_BUCKET_NAME: django-storage DJANGO_MINIO_STORAGE_MEDIA_URL: http://localhost:9000/django-storage - DJANGO_DANDI_SCHEMA_VERSION: + DJANGO_DANDI_SCHEMA_VERSION: ~ DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 DJANGO_DANDI_API_URL: http://localhost:8000 DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org @@ -61,25 +61,8 @@ services: rabbitmq: condition: service_started environment: - DJANGO_CELERY_BROKER_URL: amqp://rabbitmq:5672/ - DJANGO_CONFIGURATION: DevelopmentConfiguration - DJANGO_DANDI_DANDISETS_BUCKET_NAME: dandi-dandisets - DJANGO_DANDI_DANDISETS_LOG_BUCKET_NAME: dandiapi-dandisets-logs - DJANGO_DANDI_DANDISETS_EMBARGO_BUCKET_NAME: dandi-embargoed-dandisets - DJANGO_DANDI_DANDISETS_EMBARGO_LOG_BUCKET_NAME: dandiapi-embargo-dandisets-logs - DJANGO_DATABASE_URL: postgres://postgres:postgres@postgres:5432/django - DJANGO_MINIO_STORAGE_ACCESS_KEY: minioAccessKey - DJANGO_MINIO_STORAGE_ENDPOINT: minio:9000 - DJANGO_MINIO_STORAGE_SECRET_KEY: minioSecretKey - DJANGO_STORAGE_BUCKET_NAME: django-storage - DJANGO_MINIO_STORAGE_MEDIA_URL: http://localhost:9000/django-storage - DJANGO_DANDI_SCHEMA_VERSION: + << : *django_env DJANGO_DANDI_VALIDATION_JOB_INTERVAL: "5" - DJANGO_DANDI_WEB_APP_URL: http://localhost:8085 - DJANGO_DANDI_API_URL: http://localhost:8000 - DJANGO_DANDI_JUPYTERHUB_URL: https://hub.dandiarchive.org - DJANGO_DANDI_DEV_EMAIL: "test@example.com" - DANDI_ALLOW_LOCALHOST_URLS: "1" minio: image: minio/minio:RELEASE.2022-04-12T06-55-35Z From b9c726d21a67a685bf7ac04b50984d0e418e70a8 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 29 Jul 2024 08:41:23 -0400 Subject: [PATCH 51/53] Unbreak vcrpy install --- .github/workflows/docs.yml | 7 +++++++ .github/workflows/lint.yml | 11 +++++++++++ .github/workflows/test.yml | 7 +++++++ .github/workflows/typing.yml | 7 +++++++ 4 files changed, 32 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b52284706..5c7dde293 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,5 +39,12 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install --upgrade tox + # + # + - name: Unbreak vcrpy install + run: | + echo 'setuptools<72' > /tmp/pip-constraint.txt + echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" + - name: Build docs run: tox -e docs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b9e5d8fb1..1ae9ce35e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,16 +12,27 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.8' + - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install --upgrade tox + # Annotate codespell within PR - uses: codespell-project/codespell-problem-matcher@v1 + + # + # + - name: Unbreak vcrpy install + run: | + echo 'setuptools<72' > /tmp/pip-constraint.txt + echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" + - name: Run linters run: | tox -e lint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9e940a6ae..7b61b5604 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,6 +63,13 @@ jobs: with: python-version: ${{ matrix.python }} + # + # + - name: Unbreak vcrpy install + run: | + echo 'setuptools<72' > /tmp/pip-constraint.txt + echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" + - name: Install dependencies run: | python -m pip install --upgrade pip wheel diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml index c01c56cba..a2f5f23c0 100644 --- a/.github/workflows/typing.yml +++ b/.github/workflows/typing.yml @@ -23,5 +23,12 @@ jobs: python -m pip install --upgrade pip python -m pip install --upgrade tox + # + # + - name: Unbreak vcrpy install + run: | + echo 'setuptools<72' > /tmp/pip-constraint.txt + echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" + - name: Run type checker run: tox -e typing From 0017ba1298135399e5c8afb7e104ee6291a5f145 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Mon, 29 Jul 2024 11:27:42 -0400 Subject: [PATCH 52/53] Revert "Unbreak vcrpy install" This reverts commit b9c726d21a67a685bf7ac04b50984d0e418e70a8. --- .github/workflows/docs.yml | 7 ------- .github/workflows/lint.yml | 11 ----------- .github/workflows/test.yml | 7 ------- .github/workflows/typing.yml | 7 ------- 4 files changed, 32 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5c7dde293..b52284706 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,12 +39,5 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install --upgrade tox - # - # - - name: Unbreak vcrpy install - run: | - echo 'setuptools<72' > /tmp/pip-constraint.txt - echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" - - name: Build docs run: tox -e docs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1ae9ce35e..b9e5d8fb1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,27 +12,16 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.8' - - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install --upgrade tox - # Annotate codespell within PR - uses: codespell-project/codespell-problem-matcher@v1 - - # - # - - name: Unbreak vcrpy install - run: | - echo 'setuptools<72' > /tmp/pip-constraint.txt - echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" - - name: Run linters run: | tox -e lint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b61b5604..9e940a6ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,13 +63,6 @@ jobs: with: python-version: ${{ matrix.python }} - # - # - - name: Unbreak vcrpy install - run: | - echo 'setuptools<72' > /tmp/pip-constraint.txt - echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" - - name: Install dependencies run: | python -m pip install --upgrade pip wheel diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml index a2f5f23c0..c01c56cba 100644 --- a/.github/workflows/typing.yml +++ b/.github/workflows/typing.yml @@ -23,12 +23,5 @@ jobs: python -m pip install --upgrade pip python -m pip install --upgrade tox - # - # - - name: Unbreak vcrpy install - run: | - echo 'setuptools<72' > /tmp/pip-constraint.txt - echo PIP_CONSTRAINT=/tmp/pip-constraint.txt >> "$GITHUB_ENV" - - name: Run type checker run: tox -e typing From e0109889a8a9997ec5f9d347e09d7e662351f2fc Mon Sep 17 00:00:00 2001 From: Aaron Kanzer Date: Fri, 2 Aug 2024 10:30:52 -0400 Subject: [PATCH 53/53] exclude specific changes --- CHANGELOG.md | 3 +++ tox.ini | 14 +++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7b3588c0..2a1f337cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -269,3 +269,6 @@ - Aaron Kanzer (aaronkanzer@Aarons-MacBook-Pro.local) - Aaron Kanzer (aaronkanzer@dhcp-10-29-194-155.dyn.mit.edu) - Aaron Kanzer (aaronkanzer@dhcp-10-29-194-155.dyn.MIT.EDU) + +--- + diff --git a/tox.ini b/tox.ini index 064c68daa..659f57af8 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ commands = # Using pytest-cov instead of using coverage directly leaves a bunch of # .coverage.$HOSTNAME.#.# files lying around for some reason coverage erase - coverage run -m pytest -v {posargs} dandi + coverage run -m pytest -v {posargs} lincbrain coverage combine coverage report @@ -23,16 +23,16 @@ deps = codespell~=2.0 flake8 commands = - codespell dandi docs tools setup.py - flake8 --config=setup.cfg {posargs} dandi setup.py + codespell lincbrain setup.py + flake8 --config=setup.cfg {posargs} lincbrain setup.py [testenv:typing] deps = - mypy != 1.11.0 + mypy types-python-dateutil types-requests commands = - mypy dandi + mypy lincbrain [testenv:docs] basepython = python3 @@ -42,7 +42,7 @@ changedir = docs commands = sphinx-build -E -W -b html source build [pytest] -addopts = --tb=short --durations=10 --timeout=300 +addopts = --tb=short --durations=10 markers = integration obolibrary @@ -73,7 +73,7 @@ filterwarnings = [coverage:run] parallel = True -source = dandi +source = lincbrain [coverage:report] precision = 2