Skip to content

Commit

Permalink
Fix download of qc data
Browse files Browse the repository at this point in the history
  • Loading branch information
gregorjerse committed Oct 7, 2024
1 parent 74ba4ec commit a164ed7
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 3 deletions.
9 changes: 9 additions & 0 deletions docs/CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,15 @@ Change Log
All notable changes to this project are documented in this file.


==========
Unreleased
==========

Fixed
-----
- Fix fetching ``qc`` data from deploys with local data storage


===================
21.2.0 - 2024-07-10
===================
Expand Down
15 changes: 12 additions & 3 deletions src/resdk/tables/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import asyncio
import json
from io import BytesIO
from urllib.parse import urljoin
from urllib.parse import urljoin, urlparse

import aiohttp
import pandas as pd
Expand Down Expand Up @@ -32,16 +32,25 @@ def _uri_to_url(resolwe, uris):
return json.loads(response.content.decode("utf-8"))


def is_absolute(url: str) -> bool:
"""Return if the given URL absolute."""
return bool(urlparse(url).netloc)


async def _batch_download(resolwe, uris, parser) -> pd.DataFrame:
"""Download multiple files defined by their uri asynchronously."""
try:
uri_to_url = _uri_to_url(resolwe, uris)
except HTTPError:
return pd.DataFrame()

async with aiohttp.ClientSession() as session:
def prepare_url(url):
"""Prepent the base url if it is not absolute."""
return url if is_absolute(url) else urljoin(resolwe.url, url)

async with aiohttp.ClientSession(cookies=resolwe.session.cookies) as session:
futures = [
_download_file(uri, url, session, parser)
_download_file(uri, prepare_url(url), session, parser)
for uri, url in uri_to_url.items()
if url
]
Expand Down

0 comments on commit a164ed7

Please sign in to comment.