diff --git a/meltano.yml b/meltano.yml index ba4930e..f392072 100644 --- a/meltano.yml +++ b/meltano.yml @@ -25,10 +25,14 @@ plugins: value: '2010-01-01T00:00:00Z' config: start_date: '2010-01-01T00:00:00Z' - #api_url: https://sprintsapi.zoho.com.au/zsapi - api_url: https://sprintsapi.zoho.com/zsapi - oauth_url: https://accounts.zoho.com/oauth/v2/token - #oauth_url: https://accounts.zoho.com.au/oauth/v2/token + api_url: https://sprintsapi.zoho.com.au/zsapi + oauth_url: https://accounts.zoho.com.au/oauth/v2/token + #api_url: https://sprintsapi.zoho.com/zsapi + #oauth_url: https://accounts.zoho.com/oauth/v2/token + select: + - '!log_hour.*' + - '!project_user.*' + - '*.*' loaders: - name: target-jsonl variant: andyh1203 diff --git a/tap_zohosprints/client.py b/tap_zohosprints/client.py index 7c3de0f..901b2ab 100644 --- a/tap_zohosprints/client.py +++ b/tap_zohosprints/client.py @@ -5,7 +5,7 @@ import requests import time from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable, cast +from typing import Any, Dict, Optional, Union, List, Iterable, cast, Generator from memoization import cached @@ -142,21 +142,6 @@ def validate_response(self, response): if data.get("code") == 7602.1: raise FatalAPIError("Error, locked out of the API") - # Still catch error status codes - super().validate_response(response) - - def validate_response(self, response: requests.Response) -> None: - """Validate HTTP response. - Args: - response: A `requests.Response`_ object. - - Raises: - FatalAPIError: If the request is not retriable. - RetriableAPIError: If the request is retriable. - - .. _requests.Response: - https://docs.python-requests.org/en/latest/api/#requests.Response - """ msg = ( f"{response.status_code} Client Error: " f"{response.reason} for path: {self.path}" @@ -217,7 +202,7 @@ def property_unfurler( ids_key: str, jobj_key: str, primary_key_name: str, -) -> Iterable[dict]: +) -> Generator[dict, None, None]: """ Zohosprints embeds data inside of a JObj key. @@ -249,17 +234,20 @@ def property_unfurler( json = response.json() props: Dict = json.get(prop_key) ids: List = json.get(ids_key) - for id in ids: - record = {} - prop_values: List = json[jobj_key][id] - for property_name, property_index in props.items(): - record[property_name] = prop_values[property_index] + if json.get("hasData") == False: return_object: Dict = copy.deepcopy(json) - return_object[primary_key_name] = id - return_object.pop(prop_key) - return_object.pop(ids_key) - return_object.pop(jobj_key) - return_object["record"] = record - yield return_object - + yield return_object # Data is empty, stop. + else: + for id in ids: + record = {} + prop_values: List = json[jobj_key][id] + for property_name, property_index in props.items(): + record[property_name] = prop_values[property_index] + return_object: Dict = copy.deepcopy(json) + return_object[primary_key_name] = id + return_object.pop(prop_key) + return_object.pop(ids_key) + return_object.pop(jobj_key) + return_object["record"] = record + yield return_object return {} diff --git a/tap_zohosprints/schemas/log_hour.json b/tap_zohosprints/schemas/log_hour.json new file mode 100644 index 0000000..fdb8392 --- /dev/null +++ b/tap_zohosprints/schemas/log_hour.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "logIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "hasData": { + "type": "boolean" + }, + "userDisplayName": { + "type": "object" + }, + "hasNext": { + "type": "boolean" + }, + "nextIndex": { + "type": "integer" + }, + "zsuserIdvsZUID": { + "type": "object" + }, + "status": { + "type": "string" + }, + "record": { + "type": "object" + }, + "tLogId": { + "type": "string" + }, + "team_id": { + "type": "string" + }, + "project_id": { + "type": "string" + }, + "sprint_id": { + "type": "string" + } + } +} diff --git a/tap_zohosprints/streams.py b/tap_zohosprints/streams.py index 468b304..f69a717 100644 --- a/tap_zohosprints/streams.py +++ b/tap_zohosprints/streams.py @@ -420,3 +420,25 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: jobj_key="userJObj", primary_key_name="userId", ) + + +class LogHours(ZohoSprintsPropsStream): + """Log Hours Stream""" + + name = "log_hour" + path = "/team/{team_id}/projects/{project_id}/sprints/{sprint_id}/timesheet/?action=logitems&listviewtype=0" + parent_stream_type = SprintsStream + primary_keys = ["tLogId"] + replication_key = None + schema_filepath = SCHEMAS_DIR / "log_hour.json" + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + # Create a record object + yield from property_unfurler( + response=response, + prop_key="log_prop", + ids_key="logIds", + jobj_key="logJObj", + primary_key_name="tLogId", + ) diff --git a/tap_zohosprints/tap.py b/tap_zohosprints/tap.py index 4db2ac1..11d1e11 100644 --- a/tap_zohosprints/tap.py +++ b/tap_zohosprints/tap.py @@ -20,6 +20,7 @@ TagsStream, SprintUsers, ProjectUsers, + LogHours, ) # TODO: Compile a list of custom stream types here @@ -38,6 +39,7 @@ TagsStream, SprintUsers, ProjectUsers, + LogHours, ] diff --git a/tap_zohosprints/tests/tag_property_unfurl_hasnodata.json b/tap_zohosprints/tests/tag_property_unfurl_hasnodata.json new file mode 100644 index 0000000..cf5399d --- /dev/null +++ b/tap_zohosprints/tests/tag_property_unfurl_hasnodata.json @@ -0,0 +1,4 @@ +{ + "hasData": false, + "status": "success" +} diff --git a/tap_zohosprints/tests/test_core.py b/tap_zohosprints/tests/test_core.py index f0190e3..a985f5f 100644 --- a/tap_zohosprints/tests/test_core.py +++ b/tap_zohosprints/tests/test_core.py @@ -52,7 +52,6 @@ def test_property_unfurler(mocked_responses): content_type="application/json", ) resp = requests.get("https://autoidm.com") - assert resp.status_code == 200 unfurled = property_unfurler( response=resp, @@ -79,3 +78,35 @@ def test_property_unfurler(mocked_responses): }, "tagId": "114398000000007021", } + + +def test_property_unfurler_nodata(mocked_responses): + """ Sometimes the data comes back empty, we need to not attempt to unfurl the data """ + tag_json = "" + # Should probably just define the json inline as it's way easier to read + with open( + Path(__file__).parent / Path("tag_property_unfurl_hasnodata.json") + ) as tag: + tag_json = tag.read() + + mocked_responses.add( + responses.GET, + "https://autoidm.com", + body=tag_json, + status=200, + content_type="application/json", + ) + resp = requests.get("https://autoidm.com") + + unfurled = property_unfurler( + response=resp, + prop_key="abcdef", + ids_key="hijklmn", + jobj_key="opqrst", + primary_key_name="tuVWxYZ", + ) + output = None + for data in unfurled: + print(data) + output = data + assert output == {"hasData": False, "status": "success"}