Merge conflicts from update.

meorg_client/cli.py

@@ -11,7 +11,7 @@
 import json
 
 
-def _get_client():
+def _get_client() -> Client:
     """Get an authenticated client.
 
     Returns
@@ -45,7 +45,7 @@ def _get_client():
     )
 
 
-def _call(func, **kwargs):
+def _call(func: callable, **kwargs) -> dict:
     """Simple wrapper to handle exceptions.
 
     Exceptions are captured broadly and raw error message printed before non-zero exit.
@@ -136,9 +136,30 @@ def file_upload(file_path, attach_to=None):
         click.echo("SUCCESS")
 
 
+@click.command("upload_parallel")
+@click.argument("file_paths", nargs=-1)
+@click.option(
+    "-n", default=2, help="Number of simultaneous parallel uploads (default=2)."
+)
+def file_upload_parallel(file_paths: tuple, n: int = 2):
+    """Upload files in parallel.
+
+    Parameters
+    ----------
+    file_paths : tuple
+        Sequence of file paths.
+    n : int, optional
+        Number of parallel uploads, by default 2
+    """
+    client = _get_client()
+    responses = _call(client.upload_files_parallel, files=list(file_paths), n=n)
+    for response in responses:
+        click.echo(response.get("data").get("files")[0].get("file"))
+
+
 @click.command("list")
 @click.argument("id")
-def file_list(id):
+def file_list(id: str):
     """
     List the files currently attached to a model output.
 
@@ -154,7 +175,7 @@ def file_list(id):
 @click.command("attach")
 @click.argument("file_id")
 @click.argument("output_id")
-def file_attach(file_id, output_id):
+def file_attach(file_id: str, output_id: str):
     """
     Attach a file to a model output.
     """
@@ -167,7 +188,7 @@ def file_attach(file_id, output_id):
 
 @click.command("start")
 @click.argument("id")
-def analysis_start(id):
+def analysis_start(id: str):
     """
     Start the analysis for the model output id.
 
@@ -184,7 +205,7 @@ def analysis_start(id):
 
 @click.command("status")
 @click.argument("id")
-def analysis_status(id):
+def analysis_status(id: str):
     """
     Get the status of the analysis.
 
@@ -211,7 +232,7 @@ def analysis_status(id):
 @click.option(
     "--dev", is_flag=True, default=False, help="Setup for the development server."
 )
-def initialise(dev=False):
+def initialise(dev: bool = False):
     """
     Initialise the client on the system.
     """
@@ -228,7 +249,7 @@ def initialise(dev=False):
         click.echo(ex.msg, err=True)
         sys.exit(1)
 
-    print("Connection established.")
+    click.echo("Connection established.")
 
     # Build out the dictionary and save it to the user home.
     credentials = dict(email=email, password=password)
@@ -262,6 +283,7 @@ def cli_analysis():
 # Add file commands
 cli_file.add_command(file_list)
 cli_file.add_command(file_upload)
+cli_file.add_command(file_upload_parallel)
 cli_file.add_command(file_attach)
 
 # Add endpoint commands

meorg_client/client.py

@@ -128,7 +128,7 @@ def _make_request(
         # For flexibility
         return self.last_response
 
-    def _get_url(self, endpoint, **kwargs):
+    def _get_url(self, endpoint: str, **kwargs):
         """Get the well-formed URL for the call.
 
         Parameters
@@ -217,15 +217,15 @@ def logout(self):
             self.headers.pop("X-User-Id", None)
             self.headers.pop("X-Auth-Token", None)
 
-    def upload_files_parallel(self, files: list, n: int = 2):
+    def upload_files_parallel(self, files: Union[str, Path, list], n: int = 2):
         """Upload files in parallel.
 
         Parameters
         ----------
-        files : list
-            List of file paths.
+        files : Union[str, Path, list]
+            A path to a file, or a list of paths.
         n : int, optional
-            Number of threads to use, by default 2
+            Number of threads to use, by default 2.
 
         Returns
         -------
@@ -236,7 +236,7 @@ def upload_files_parallel(self, files: list, n: int = 2):
         # Ensure the object is actually iterable
         files = mu.ensure_list(files)
 
-        # Sequential case, single file provided
+        # Single file provided, don't bother starting the pool
         if len(files) == 1:
             return self.upload_files(files)
 
@@ -419,7 +419,7 @@ def list_endpoints(self) -> Union[dict, requests.Response]:
         """
         return self._make_request(method=mcc.HTTP_GET, endpoint=endpoints.ENDPOINT_LIST)
 
-    def success(self):
+    def success(self) -> bool:
         """Test if the last request was successful.
 
         Returns

meorg_client/tests/test_cli.py

@@ -8,22 +8,40 @@
 
 
 @pytest.fixture
-def runner():
+def runner() -> CliRunner:
+    """Get a runner object.
+
+    Returns
+    -------
+    click.testing.CliRunner
+        Runner object.
+    """
     return CliRunner()
 
 
-def test_list_endpoints(runner):
+@pytest.fixture
+def test_filepath() -> str:
+    """Get a test filepath from the installation.
+
+    Returns
+    -------
+    str
+        Path to the test filepath.
+    """
+    return os.path.join(mu.get_installed_data_root(), "test/test.txt")
+
+
+def test_list_endpoints(runner: CliRunner):
     """Test list-endpoints via CLI."""
     result = runner.invoke(cli.list_endpoints)
     assert result.exit_code == 0
 
 
-def test_file_upload(runner):
+def test_file_upload(runner: CliRunner, test_filepath: str):
     """Test file-upload via CLI."""
 
     # Upload a tiny test file
-    filepath = os.path.join(mu.get_installed_data_root(), "test/test.txt")
-    result = runner.invoke(cli.file_upload, [filepath])
+    result = runner.invoke(cli.file_upload, [test_filepath])
     assert result.exit_code == 0
 
     # Add the job_id to the store for the next test
@@ -33,12 +51,11 @@ def test_file_upload(runner):
     time.sleep(5)
 
 
-def test_file_multiple(runner):
+def test_file_multiple(runner: CliRunner, test_filepath: str):
     """Test file-upload via CLI."""
 
     # Upload a tiny test file
-    filepath = os.path.join(mu.get_installed_data_root(), "test/test.txt")
-    result = runner.invoke(cli.file_upload, [filepath, filepath])
+    result = runner.invoke(cli.file_upload, [test_filepath, test_filepath])
     assert result.exit_code == 0
 
     # Add the job_id to the store for the next test
@@ -48,6 +65,16 @@ def test_file_multiple(runner):
     time.sleep(5)
 
 
+def test_file_upload_parallel(runner: CliRunner, test_filepath: str):
+    """Test file-upload via CLI."""
+
+    # Upload a tiny test file
+    result = runner.invoke(
+        cli.file_upload_parallel, [test_filepath, test_filepath, "-n 2"]
+    )
+    assert result.exit_code == 0
+
+
 def test_file_list(runner):
     """Test file-list via CLI."""
     result = runner.invoke(cli.file_list, [store.get("model_output_id")])

meorg_client/tests/test_client.py

@@ -8,7 +8,7 @@
 import tempfile as tf
 
 
-def _get_authenticated_client():
+def _get_authenticated_client() -> Client:
     """Get an authenticated client for tests.
 
     Returns
@@ -45,30 +45,39 @@ def _get_test_file():
 
 
 @pytest.fixture
-def client():
+def client() -> Client:
     return _get_authenticated_client()
 
 
+@pytest.fixture
+def test_filepath() -> str:
+    """Get a test filepath from the installation.
+
+    Returns
+    -------
+    str
+        Path to the test filepath.
+    """
+    return os.path.join(mu.get_installed_data_root(), "test/test.txt")
+
+
 def test_login():
     """Test login."""
     _client = _get_authenticated_client()
     assert "X-Auth-Token" in _client.headers.keys()
 
 
-def test_list_endpoints(client):
+def test_list_endpoints(client: Client):
     """Test list_endpoints."""
     response = client.list_endpoints()
     assert client.success()
     assert isinstance(response, dict)
 
 
-def test_upload_file(client):
+def test_upload_file(client: Client, test_filepath: str):
     """Test the uploading of a file."""
-    # Upload the file.
-    filepath = _get_test_file()
-
     # Upload the file
-    response = client.upload_files(filepath)
+    response = client.upload_files(test_filepath)
 
     # Make sure it worked
     assert client.success()
@@ -77,13 +86,11 @@ def test_upload_file(client):
     store.set("file_upload", response)
 
 
-def test_upload_file_multiple(client):
+def test_upload_file_multiple(client: Client, test_filepath: str):
     """Test the uploading of a file."""
-    # Upload the file.
-    filepath = os.path.join(mu.get_installed_data_root(), "test/test.txt")
 
     # Upload the file
-    response = client.upload_files([filepath, filepath])
+    response = client.upload_files([test_filepath, test_filepath])
 
     # Make sure it worked
     assert client.success()
@@ -92,14 +99,14 @@ def test_upload_file_multiple(client):
     store.set("file_upload_multiple", response)
 
 
-def test_file_list(client):
-    """Test the list of files for a model output."""
+def test_file_list(client: Client):
+    """Test the listinf of files for a model output."""
     response = client.list_files(client._model_output_id)
     assert client.success()
     assert isinstance(response.get("data").get("files"), list)
 
 
-def test_attach_files_to_model_output(client):
+def test_attach_files_to_model_output(client: Client):
     # Get the file id from the job id
     file_id = store.get("file_upload").get("data").get("files")[0].get("file")
 
@@ -109,14 +116,14 @@ def test_attach_files_to_model_output(client):
     assert client.success()
 
 
-def test_start_analysis(client):
+def test_start_analysis(client: Client):
     """Test starting an analysis."""
     response = client.start_analysis(client._model_output_id)
     assert client.success()
     store.set("start_analysis", response)
 
 
-def test_get_analysis_status(client):
+def test_get_analysis_status(client: Client):
     """Test getting the analysis status."""
     # Get the analysis id from the store
     analysis_id = store.get("start_analysis").get("data").get("analysisId")
@@ -125,7 +132,7 @@ def test_get_analysis_status(client):
 
 
 @pytest.mark.xfail(strict=False)
-def test_upload_file_large(client):
+def test_upload_file_large(client: Client):
     """Test the uploading of a large-ish file."""
 
     # Create an in-memory 10mb file
@@ -148,14 +155,25 @@ def test_upload_file_large(client):
     assert client.success()
 
 
-def test_upload_files_with_attach(client):
+def test_upload_files_with_attach(client: Client):
     """Test that the upload can also attach in the same method call."""
     filepath = _get_test_file()
     _ = client.upload_files([filepath, filepath], attach_to=client._model_output_id)
     assert client.success()
 
 
-def test_logout(client):
+def test_upload_file_parallel(client: Client, test_filepath: str):
+    """Test the uploading of a file."""
+    # Upload the file
+    responses = client.upload_files_parallel([test_filepath, test_filepath], n=2)
+
+    # Make sure it worked
+    assert all(
+        [response.get("data").get("files")[0].get("file") for response in responses]
+    )
+
+
+def test_logout(client: Client):
     """Test logout."""
     client.logout()
     assert "X-Auth-Token" not in client.headers.keys()