11 setup rtd

.conda/mkdocs-requirements.txt

@@ -0,0 +1,6 @@
+mkdocs>1.1
+mkdocs-material>=7.2.6
+mkdocs-git-revision-date-localized-plugin
+mkdocs-macros-plugin
+mkdocstrings
+mkdocstrings[python]

.readthedocs.yaml

@@ -0,0 +1,20 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+
+mkdocs:
+  configuration: mkdocs.yml
+
+# Optionally declare the Python requirements required to build your docs
+python:
+   install:
+   - requirements: .conda/mkdocs-requirements.txt

meorg_client/cli.py

@@ -4,7 +4,6 @@
 import meorg_client.utilities as mcu
 import os
 import sys
-from inspect import getmembers
 import getpass
 from pathlib import Path
 import json
@@ -21,20 +20,19 @@ def _get_client():
     # Get the dev-mode flag from the environment, better than passing the dev flag everywhere.
     dev_mode = os.getenv("MEORG_DEV_MODE", "0") == "1"
 
-    credentials = mcu.get_user_data_filepath('credentials.json')
-    credentials_dev = mcu.get_user_data_filepath('credentials-dev.json')
+    credentials = mcu.get_user_data_filepath("credentials.json")
+    credentials_dev = mcu.get_user_data_filepath("credentials-dev.json")
 
     # In dev mode and the configuration file exists
     if dev_mode and credentials_dev.is_file():
-        credentials = mcu.load_user_data('credentials-dev.json')
-    
+        credentials = mcu.load_user_data("credentials-dev.json")
+
     # In dev mode and it doesn't (i.e. Actions)
     elif dev_mode and not credentials_dev.is_file():
         credentials = dict(
-            email=os.getenv('MEORG_EMAIL'),
-            password=os.getenv('MEORG_PASSWORD')
+            email=os.getenv("MEORG_EMAIL"), password=os.getenv("MEORG_PASSWORD")
         )
-    
+
     # Production credentials
     else:
         credentials = mcu.load_user_data("credentials.json")
@@ -65,11 +63,10 @@ def _call(func, **kwargs):
     try:
         return func(**kwargs)
     except Exception as ex:
-
         click.echo(ex.msg, err=True)
 
         # Bubble up the exception
-        if os.getenv('MEORG_DEV_MODE') == '1':
+        if os.getenv("MEORG_DEV_MODE") == "1":
             raise
 
         sys.exit(1)
@@ -86,7 +83,7 @@ def cli():
     pass
 
 
-@click.command('list')
+@click.command("list")
 def list_endpoints():
     """
     List the available endpoints for the server.
@@ -107,27 +104,8 @@ def list_endpoints():
             click.echo(out)
 
 
-@click.command('status')
-@click.argument("id")
-def file_status(id):
-    """
-    Check the file status based on the job ID from file-upload.
-
-    Prints the true file ID or a status.
-    """
-    client = _get_client()
-    response_data = _call(client.get_file_status, id=id).get("data")
-
-    # If the file is complete (transferred to object store), get the true ID
-    if response_data.get("status") == "complete":
-        file_id = response_data.get("files")[0].get("file")
-        click.echo(file_id)
-    else:
-        click.echo("Pending")
-
-
-@click.command('upload')
-@click.argument("file_path")
+@click.command("upload")
+@click.argument("file_path", nargs=-1)
 def file_upload(file_path):
     """
     Upload a file to the server.
@@ -137,12 +115,13 @@ def file_upload(file_path):
     client = _get_client()
 
     # Upload the file, get the job ID
-    response = _call(client.upload_file, file_path=file_path)
-    job_id = response.get("data").get("jobId")
-    click.echo(job_id)
+    response = _call(client.upload_file, file_path=list(file_path))
+    files = response.get("data").get("files")
+    for f in files:
+        click.echo(f.get("file"))
 
 
-@click.command('list')
+@click.command("list")
 @click.argument("id")
 def file_list(id):
     """
@@ -157,7 +136,7 @@ def file_list(id):
         click.echo(f)
 
 
-@click.command('attach')
+@click.command("attach")
 @click.argument("file_id")
 @click.argument("output_id")
 def file_attach(file_id, output_id):
@@ -166,12 +145,12 @@ def file_attach(file_id, output_id):
     """
     client = _get_client()
 
-    response = _call(client.attach_files_to_model_output, id=output_id, files=[file_id])
+    _ = _call(client.attach_files_to_model_output, id=output_id, files=[file_id])
 
     click.echo("SUCCESS")
 
 
-@click.command('start')
+@click.command("start")
 @click.argument("id")
 def analysis_start(id):
     """
@@ -188,7 +167,7 @@ def analysis_start(id):
         click.echo(analysis_id)
 
 
-@click.command('status')
+@click.command("status")
 @click.argument("id")
 def analysis_status(id):
     """
@@ -250,22 +229,25 @@ def initialise(dev=False):
 
 
 # Add groups for nested subcommands
-@click.group('endpoints', help='API endpoint commands.')
+@click.group("endpoints", help="API endpoint commands.")
 def cli_endpoints():
     pass
 
-@click.group('file', help='File commands.')
+
+@click.group("file", help="File commands.")
 def cli_file():
     pass
 
-@click.group('analysis', help='Analysis commands.')
+
+@click.group("analysis", help="Analysis commands.")
 def cli_analysis():
     pass
 
+
 # Add file commands
 cli_file.add_command(file_list)
 cli_file.add_command(file_upload)
-cli_file.add_command(file_status)
+# cli_file.add_command(file_status)
 cli_file.add_command(file_attach)
 
 # Add endpoint commands

meorg_client/client.py

@@ -9,6 +9,7 @@
 import meorg_client.endpoints as endpoints
 import meorg_client.exceptions as mx
 import mimetypes as mt
+import io
 
 
 class Client:
@@ -213,45 +214,67 @@ def logout(self):
             self.headers.pop("X-User-Id", None)
             self.headers.pop("X-Auth-Token", None)
 
-    def get_file_status(self, id: str) -> Union[dict, requests.Response]:
-        """Get the file status.
+    def upload_file(
+        self,
+        file_path: Union[str, list],
+        file_obj: Union[io.BufferedReader, list] = None,
+    ) -> Union[dict, requests.Response]:
+        """Upload a file.
 
         Parameters
         ----------
-        id : str
-            Job ID of the file.
+        file_path : str or list
+            Path to the file.
+        file_obj : io.BufferedReader, optional
+            File object (handle) to allow direct supply of file object, by default None
 
         Returns
         -------
         Union[dict, requests.Response]
             Response from ME.org.
         """
-        return self._make_request(
-            method=mcc.HTTP_GET, endpoint=endpoints.FILE_STATUS, url_params=dict(id=id)
-        )
 
-    def upload_file(self, file_path: str) -> Union[dict, requests.Response]:
-        """Upload a file.
+        payload = list()
 
-        Parameters
-        ----------
-        file_path : str
-            Path to the file.
+        # Cast as list for iterative upload
+        if not isinstance(file_path, list):
+            file_path = [file_path]
 
-        Returns
-        -------
-        Union[dict, requests.Response]
-            Response from ME.org.
-        """
-        # Get the filename and extension
-        filename = os.path.basename(file_path)
-        ext = filename.split(".")[-1]
+        # Payload assembly
+        if file_obj is not None:
+            # Cast as a list for iterative upload
+            if not isinstance(file_obj, list):
+                file_objs = [file_obj]
 
-        # Get the MIME type (raises a KeyError if it is unknown)
-        mimetype = mt.types_map[f".{ext}"]
+            if len(file_objs) != len(file_path):
+                raise ValueError("Supplied file paths and file objects do not match")
 
-        # Assemble the file payload
-        payload = dict(file=(filename, open(file_path, "rb"), mimetype))
+            for ix, file_obj in enumerate(file_objs):
+                if not isinstance(file_obj, io.BufferedReader) and not isinstance(
+                    file_obj, io.BytesIO
+                ):
+                    raise TypeError(
+                        f"Supplied file object {ix} is not an io.BufferedReader or io.BytesIO."
+                    )
+
+                _file_path = file_path[ix]
+
+                # Get the filename and extension
+                filename = os.path.basename(_file_path)
+                ext = filename.split(".")[-1]
+                mimetype = mt.types_map[f".{ext}"]
+
+                payload.append(("file", (filename, file_obj, mimetype)))
+
+        else:
+            for _file_path in file_path:
+                # Get the filename and extension
+                filename = os.path.basename(_file_path)
+                ext = filename.split(".")[-1]
+                mimetype = mt.types_map[f".{ext}"]
+                file_obj = open(_file_path, "rb")
+
+                payload.append(("file", (filename, file_obj, mimetype)))
 
         return self._make_request(
             method=mcc.HTTP_POST,

meorg_client/docs/api.md

@@ -0,0 +1,3 @@
+# API Reference
+
+::: meorg_client.client.Client

meorg_client/docs/cli.md

@@ -25,7 +25,17 @@ Alternatively, you can create a credentials file at the target filepath manually
 }
 ```
 
-Once credentials are set up, you may use the command-line utilities listed alphabetically below. However, given the asynchronous nature of the server requests, a typical workflow is more useful.
+## Get your Model Output ID
+
+As most of the commands act with respect to a given model output, you must first establish the `$MODEL_OUTPUT_ID` to use.
+
+1. Go to modelevaluation.org.
+2. Select "Model Outputs" from the main navigation.
+3. Select the appropriate subset (i.e. Owned by Me).
+4. Click the appropriate model output.
+5. The `$MODEL_OUTPUT_ID` will be displayed in the copy box at the top of the page.
+
+Once credentials are set up and you have your `$MODEL_OUTPUT_ID`, you may use the command-line utilities listed alphabetically below. However, given the asynchronous nature of the server requests, a typical workflow is more useful.
 
 ## Typical Workflow
 
@@ -69,7 +79,9 @@ meorg analysis status $ANALYSIS_ID
 # The final command will output the status and URL to the dashboard.
 ```
 
-## analysis start
+## Commands Available
+
+### analysis start
 
 To start an analysis for a given model output using the files provided, execute the following command:
 
@@ -84,7 +96,7 @@ modelevaluation.org/modelOutput/display/**kafS53HgWu2CDXxgC**
 
 This command will return an `$ANALYSIS_ID` upon success which is used in `analysis status`.
 
-## analysis status
+### analysis status
 
 To query the status of an analysis, execute the following command:
 
@@ -94,7 +106,7 @@ meorg analysis status $ANALYSIS_ID
 
 Where `$ANALYSIS_ID` is the ID returned from `analysis start`.
 
-## file attach
+### file attach
 
 To attach a file to a model output prior to executing an analysis, execute the following command:
 
@@ -104,7 +116,7 @@ meorg file attach $FILE_ID $MODEL_OUTPUT_$ID
 
 Where `$FILE_ID` is the ID returned from `file-status` and `$MODEL_OUTPUT_ID` is the ID of the model output in question.
 
-## file status
+### file status (deprecated)
 
 Given that a `file upload` puts a file in a queue to transfer to the object store, the file itself is not available for use until it has been successfully transferred. In order to check the status of this transfer, execute the following command:
 
@@ -116,7 +128,9 @@ Where `$JOB_ID` is the ID returned from `file upload`.
 
 Once a file is listed as completed, the command will return the true `$FILE_ID` which can be used in `file-attach`.
 
-## file upload
+> NOTE: The file status command is likely to be deprecated in the near future.
+
+### file upload
 
 To upload a file to the staging area of the server, execute the following command:
 
@@ -128,11 +142,11 @@ Where `$PATH` is the local path to the file.
 
 This command will return a `$JOB_ID` upon success, which can be used with `file status` to check the transfer status to the object store.
 
-## initialise
+### initialise
 
 A simple helper command to write the user credentials file for password-less interaction with the client over the command-line. See above.
 
-## endpoints list
+### endpoints list
 
 To list all of the available API endpoints, execute the following command:
 

meorg_client/endpoints.py

@@ -9,7 +9,7 @@
 
 # Files
 FILE_LIST = "modeloutput/{id}/files"
-FILE_UPLOAD = "files"
+FILE_UPLOAD = "upload"
 FILE_STATUS = "files/status/{id}"
 
 # Analysis