Skip to content

Commit

Permalink
Raise and handle FolderContentsMaximumLimitError
Browse files Browse the repository at this point in the history
  • Loading branch information
wkentaro committed Apr 13, 2023
1 parent ae32629 commit 2930376
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 23 deletions.
1 change: 1 addition & 0 deletions gdown/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# flake8: noqa

from . import exceptions
from .cached_download import cached_download
from .cached_download import md5sum
from .download import download
Expand Down
10 changes: 10 additions & 0 deletions gdown/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from .download import indent
from .download_folder import MAX_NUMBER_FILES
from .download_folder import download_folder
from .exceptions import FolderContentsMaximumLimitError


class _ShowVersionAction(argparse.Action):
Expand Down Expand Up @@ -169,6 +170,15 @@ def main():
resume=args.continue_,
format=args.format,
)
except FolderContentsMaximumLimitError as e:
print(
"Failed to retrieve folder contents:\n\n{}\n\n"
"You can use `--remaining-ok` option to ignore this error.".format(
indent("\n".join(textwrap.wrap(str(e))), prefix="\t")
),
file=sys.stderr,
)
sys.exit(1)
except requests.exceptions.ProxyError as e:
print(
"Failed to use proxy:\n\n{}\n\n"
Expand Down
36 changes: 13 additions & 23 deletions gdown/download_folder.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@
import os.path as osp
import re
import sys
import textwrap
import warnings

import bs4

from .download import _get_session
from .download import download
from .download import indent
from .exceptions import FolderContentsMaximumLimitError

MAX_NUMBER_FILES = 50

Expand Down Expand Up @@ -162,16 +161,14 @@ def _download_and_parse_google_drive_link(
gdrive_file.children.append(child)
has_at_least_max_files = len(gdrive_file.children) == MAX_NUMBER_FILES
if not remaining_ok and has_at_least_max_files:
err_msg = " ".join(
message = " ".join(
[
"The gdrive folder with url: {url}".format(url=url),
"has more than {max} files,".format(max=MAX_NUMBER_FILES),
"gdrive can't download more than this limit,",
"if you are ok with this,",
"please run again with --remaining-ok flag.",
"gdrive can't download more than this limit.",
]
)
raise RuntimeError(err_msg)
raise FolderContentsMaximumLimitError(message)
return return_code, gdrive_file


Expand Down Expand Up @@ -252,26 +249,19 @@ def download_folder(
sess = _get_session(proxy=proxy, use_cookies=use_cookies)

if not quiet:
print("Retrieving folder list", file=sys.stderr)
try:
return_code, gdrive_file = _download_and_parse_google_drive_link(
sess,
url,
quiet=quiet,
remaining_ok=remaining_ok,
verify=verify,
)
except RuntimeError as e:
print("Failed to retrieve folder contents:", file=sys.stderr)
error = "\n".join(textwrap.wrap(str(e)))
error = indent(error, "\t")
print("\n", error, "\n", file=sys.stderr)
return
print("Retrieving folder contents", file=sys.stderr)
return_code, gdrive_file = _download_and_parse_google_drive_link(
sess,
url,
quiet=quiet,
remaining_ok=remaining_ok,
verify=verify,
)

if not return_code:
return return_code
if not quiet:
print("Retrieving folder list completed", file=sys.stderr)
print("Retrieving folder contents completed", file=sys.stderr)
print("Building directory structure", file=sys.stderr)
if output is None:
output = os.getcwd() + osp.sep
Expand Down
2 changes: 2 additions & 0 deletions gdown/exceptions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
class FolderContentsMaximumLimitError(Exception):
pass

0 comments on commit 2930376

Please sign in to comment.