From 3bdb24f996a968fd174d9aba0ba81f4d79334eca Mon Sep 17 00:00:00 2001 From: Romeo Kienzler <romeo.kienzler@protonmail.com> Date: Mon, 22 Apr 2024 13:14:06 +0200 Subject: [PATCH 1/5] make pip installable --- setup.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..e537213b --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup + +setup(name='claimed-component-library', + version='0.1', + description='CLAIMED component library', + url='https://github.com/claimed-framework/component-library', + author='The CLAIMED authors', + author_email='claimed-framework@proton.me', + license='Apache2', + packages=[], + zip_safe=True) From 66036e53dd312e220c198a47c924d1d7b0622083 Mon Sep 17 00:00:00 2001 From: Romeo Kienzler <romeo.kienzler@protonmail.com> Date: Mon, 22 Apr 2024 13:45:48 +0200 Subject: [PATCH 2/5] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e537213b..c42cbeeb 100644 --- a/setup.py +++ b/setup.py @@ -7,5 +7,5 @@ author='The CLAIMED authors', author_email='claimed-framework@proton.me', license='Apache2', - packages=[], + packages=['claimed-component-library'], zip_safe=True) From f61195e41c68f0dfdafb0a6c72a8d25de78587ba Mon Sep 17 00:00:00 2001 From: Romeo Kienzler <5694071+romeokienzler@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:19:10 +0200 Subject: [PATCH 3/5] fix setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c42cbeeb..288af1d7 100644 --- a/setup.py +++ b/setup.py @@ -7,5 +7,5 @@ author='The CLAIMED authors', author_email='claimed-framework@proton.me', license='Apache2', - packages=['claimed-component-library'], - zip_safe=True) + packages=['component-library'], + zip_safe=False) From 60136169e1041151639596ce7a3bf963d0a6fc1d Mon Sep 17 00:00:00 2001 From: Lorenz Weingart <168430688+weingartlorenz@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:47:23 +0200 Subject: [PATCH 4/5] Xview Dataset Draftv01 Hello Romeo, here is the draft of my dataset download file, if you could take a look at it, if it is ok, I'll make a claimed component out of it. --- .../input/dataset_download.ipynb | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 component-library/input/dataset_download.ipynb diff --git a/component-library/input/dataset_download.ipynb b/component-library/input/dataset_download.ipynb new file mode 100644 index 00000000..7c8cd13b --- /dev/null +++ b/component-library/input/dataset_download.ipynb @@ -0,0 +1,196 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "818f3c71", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset download started successfully.\n", + "C:\\Users\\loren\\Desktop\\dataset_download\\Tried_downloads\\train_labels.zip\n", + "File downloaded successfully.\n" + ] + } + ], + "source": [ + "import os\n", + "import shutil\n", + "import time\n", + "from selenium import webdriver\n", + "from selenium.webdriver.common.by import By\n", + "from selenium.webdriver.support.ui import WebDriverWait\n", + "from selenium.webdriver.support import expected_conditions as EC\n", + "from urllib.parse import urlparse\n", + "\n", + "def login_and_download(username, password, move_to_dir, chromedriver_path, max_download_time, label): \n", + " \n", + " # Set Chrome options to automatically download files to the specified directory\n", + " options = webdriver.ChromeOptions()\n", + " prefs = {\n", + " \"download.default_directory\": move_to_dir,\n", + " \"download.prompt_for_download\": False,\n", + " \"download.directory_upgrade\": True,\n", + " \"safebrowsing.enabled\": True\n", + " }\n", + " options.add_experimental_option(\"prefs\", prefs)\n", + "\n", + " # Start a new instance of Chrome web browser\n", + " driver = webdriver.Chrome(executable_path=chromedriver_path, options=options)\n", + " \n", + " # Open the login page\n", + " url_login = r'https://challenge.xviewdataset.org/login'\n", + " driver.get(url_login)\n", + "\n", + " # Find the username and password fields and enter credentials\n", + " username_field = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, 'email')))\n", + " password_field = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, 'password')))\n", + " username_field.send_keys(username)\n", + " password_field.send_keys(password)\n", + "\n", + " # Find and click the login button\n", + " login_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, 'btn.primary')))\n", + " login_button.click()\n", + " \n", + " # Wait for the page to load after login\n", + " time.sleep(1)\n", + " \n", + " # Open the Download page\n", + " url_download = r'https://challenge.xviewdataset.org/download-links'\n", + " driver.get(url_download)\n", + " \n", + " # Wait for the overlay element to be present\n", + " overlay_element = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, 'overlay--active')))\n", + "\n", + " # Remove the automaic pop-up overlay \n", + " body_element = driver.find_element_by_tag_name('body')\n", + " body_element.click()\n", + " time.sleep(1)\n", + " \n", + " # Switch between the possible download files\n", + " search_text = \"\"\n", + " match label:\n", + " case \"TI.zip\":\n", + " search_text = '//a[contains(text(), \"Download Training Images (zip)\")]'\n", + " case \"TL.zip\":\n", + " search_text = '//a[contains(text(), \"Download Training Labels (zip)\")]'\n", + " case \"VI.zip\":\n", + " search_text = '//a[contains(text(), \"Download Validation Images (zip)\")]'\n", + " case \"TI.tgz\":\n", + " search_text = '//a[contains(text(), \"Download Training Images (tgz)\")]'\n", + " case \"TL.tgz\":\n", + " search_text = '//a[contains(text(), \"Download Training Labels (tgz)\")]'\n", + " case \"VI.tgz\":\n", + " search_text = '//a[contains(text(), \"Download Validation Images (tgz)\")]'\n", + " case _:\n", + " raise ValueError(\"Error: This is an invalid download option\") \n", + " \n", + " # Wait for the download link to be present\n", + " download_link_element = WebDriverWait(driver, 100).until(EC.presence_of_element_located((By.XPATH, search_text)))\n", + " \n", + " # Get the dynamic download link from the href attribute\n", + " download_link = download_link_element.get_attribute('href')\n", + " \n", + " # Download the dataset using the obtained link\n", + " if download_link:\n", + " driver.get(download_link)\n", + " print(\"Dataset download started successfully.\")\n", + " \n", + " # Extract the filename from the download link URL\n", + " parsed_url = urlparse(download_link)\n", + " filename = parsed_url.path.split('/')[-1]\n", + " downloaded_file = os.path.join(move_to_dir, filename)\n", + " print(downloaded_file)\n", + " \n", + " # Check if the download directory exists\n", + " if not os.path.exists(move_to_dir):\n", + " os.makedirs(move_to_dir)\n", + " \n", + " # Wait for the file to be completely downloaded\n", + " start_time = time.time()\n", + " \n", + " while True:\n", + " if os.path.exists(downloaded_file) and os.path.getsize(downloaded_file) > 0:\n", + " print(\"File downloaded successfully.\")\n", + " break\n", + " elif time.time() - start_time > max_download_time:\n", + " print(\"Error: Maximum wait time exceeded.\")\n", + " break\n", + " else:\n", + " time.sleep(5)\n", + " \n", + " else:\n", + " print(\"Failed to get the download link.\")\n", + "\n", + " # Close the browser\n", + " driver.quit()\n", + " \n", + " \n", + "username = # enter your Username\n", + "password = # enter your password\n", + "move_to_dir = # provide a movetodir\n", + "chromedriver_path = # provide a path to chromedriver\n", + "max_download_time = 2700 # must be ajusted acording to the file size and internet speed\n", + "label = #the label of the file desired to download\n", + "\n", + "login_and_download(username, password, move_to_dir, chromedriver_path, max_download_time, label)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "794506c5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7b2f96d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0047693d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba530873", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 5a06846e1aa850c5702a60f5dc2a6e479913a17a Mon Sep 17 00:00:00 2001 From: weingartlorenz <weingartlorenz@gmail.com> Date: Tue, 30 Apr 2024 17:20:01 +0200 Subject: [PATCH 5/5] first try of a datasetdownload component to me finished and improved Signed-off-by: weingartlorenz <weingartlorenz@gmail.com> --- .../input/{dataset_download.ipynb => input-Xview_dataset.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename component-library/input/{dataset_download.ipynb => input-Xview_dataset.ipynb} (100%) diff --git a/component-library/input/dataset_download.ipynb b/component-library/input/input-Xview_dataset.ipynb similarity index 100% rename from component-library/input/dataset_download.ipynb rename to component-library/input/input-Xview_dataset.ipynb