From 1e01c60d9ae2083d40a931e956c5f500c3036271 Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Tue, 21 Dec 2021 15:41:25 -0500 Subject: [PATCH 1/3] Expand property_unfurler to handle hasData=False --- tap_zohosprints/client.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/tap_zohosprints/client.py b/tap_zohosprints/client.py index af2f267..c7d6d24 100644 --- a/tap_zohosprints/client.py +++ b/tap_zohosprints/client.py @@ -5,7 +5,7 @@ import requests import time from pathlib import Path -from typing import Any, Dict, Optional, Union, List, Iterable, cast +from typing import Any, Dict, Optional, Union, List, Iterable, cast, Generator from memoization import cached @@ -255,7 +255,7 @@ def property_unfurler( ids_key: str, jobj_key: str, primary_key_name: str, -) -> Iterable[dict]: +) -> Generator[dict, None, None]: """ Zohosprints embeds data inside of a JObj key. @@ -287,17 +287,20 @@ def property_unfurler( json = response.json() props: Dict = json.get(prop_key) ids: List = json.get(ids_key) - for id in ids: - record = {} - prop_values: List = json[jobj_key][id] - for property_name, property_index in props.items(): - record[property_name] = prop_values[property_index] + if (json.get("hasData") == False): return_object: Dict = copy.deepcopy(json) - return_object[primary_key_name] = id - return_object.pop(prop_key) - return_object.pop(ids_key) - return_object.pop(jobj_key) - return_object["record"] = record - yield return_object - + yield return_object #Data is empty, stop. + else: + for id in ids: + record = {} + prop_values: List = json[jobj_key][id] + for property_name, property_index in props.items(): + record[property_name] = prop_values[property_index] + return_object: Dict = copy.deepcopy(json) + return_object[primary_key_name] = id + return_object.pop(prop_key) + return_object.pop(ids_key) + return_object.pop(jobj_key) + return_object["record"] = record + yield return_object return {} From dc80e5713f59ae26921ce2a9d88b8f5da2f57f8d Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Tue, 21 Dec 2021 19:31:29 -0500 Subject: [PATCH 2/3] Log Hours working w/o incremental --- meltano.yml | 12 +++-- tap_zohosprints/client.py | 16 ------- tap_zohosprints/schemas/log_hour.json | 44 +++++++++++++++++++ tap_zohosprints/streams.py | 21 +++++++++ tap_zohosprints/tap.py | 2 + .../tests/tag_property_unfurl_hasnodata.json | 4 ++ tap_zohosprints/tests/test_core.py | 33 +++++++++++++- 7 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 tap_zohosprints/schemas/log_hour.json create mode 100644 tap_zohosprints/tests/tag_property_unfurl_hasnodata.json diff --git a/meltano.yml b/meltano.yml index ba4930e..f392072 100644 --- a/meltano.yml +++ b/meltano.yml @@ -25,10 +25,14 @@ plugins: value: '2010-01-01T00:00:00Z' config: start_date: '2010-01-01T00:00:00Z' - #api_url: https://sprintsapi.zoho.com.au/zsapi - api_url: https://sprintsapi.zoho.com/zsapi - oauth_url: https://accounts.zoho.com/oauth/v2/token - #oauth_url: https://accounts.zoho.com.au/oauth/v2/token + api_url: https://sprintsapi.zoho.com.au/zsapi + oauth_url: https://accounts.zoho.com.au/oauth/v2/token + #api_url: https://sprintsapi.zoho.com/zsapi + #oauth_url: https://accounts.zoho.com/oauth/v2/token + select: + - '!log_hour.*' + - '!project_user.*' + - '*.*' loaders: - name: target-jsonl variant: andyh1203 diff --git a/tap_zohosprints/client.py b/tap_zohosprints/client.py index c98fd86..c0f189c 100644 --- a/tap_zohosprints/client.py +++ b/tap_zohosprints/client.py @@ -142,21 +142,6 @@ def validate_response(self, response): if data.get("code") == 7602.1: raise FatalAPIError("Error, locked out of the API") - # Still catch error status codes - super().validate_response(response) - - def validate_response(self, response: requests.Response) -> None: - """Validate HTTP response. - Args: - response: A `requests.Response`_ object. - - Raises: - FatalAPIError: If the request is not retriable. - RetriableAPIError: If the request is retriable. - - .. _requests.Response: - https://docs.python-requests.org/en/latest/api/#requests.Response - """ msg = ( f"{response.status_code} Client Error: " f"{response.reason} for path: {self.path}" @@ -168,7 +153,6 @@ def validate_response(self, response: requests.Response) -> None: elif 500 <= response.status_code < 600: raise RetriableAPIError(msg) - class ZohoSprintsPropsStream(ZohoSprintsStream): next_page_token_jsonpath = "$.nextIndex" diff --git a/tap_zohosprints/schemas/log_hour.json b/tap_zohosprints/schemas/log_hour.json new file mode 100644 index 0000000..fdb8392 --- /dev/null +++ b/tap_zohosprints/schemas/log_hour.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "logIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "hasData": { + "type": "boolean" + }, + "userDisplayName": { + "type": "object" + }, + "hasNext": { + "type": "boolean" + }, + "nextIndex": { + "type": "integer" + }, + "zsuserIdvsZUID": { + "type": "object" + }, + "status": { + "type": "string" + }, + "record": { + "type": "object" + }, + "tLogId": { + "type": "string" + }, + "team_id": { + "type": "string" + }, + "project_id": { + "type": "string" + }, + "sprint_id": { + "type": "string" + } + } +} diff --git a/tap_zohosprints/streams.py b/tap_zohosprints/streams.py index 468b304..e44f65e 100644 --- a/tap_zohosprints/streams.py +++ b/tap_zohosprints/streams.py @@ -420,3 +420,24 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: jobj_key="userJObj", primary_key_name="userId", ) + +class LogHours(ZohoSprintsPropsStream): + """Log Hours Stream""" + + name = "log_hour" + path = "/team/{team_id}/projects/{project_id}/sprints/{sprint_id}/timesheet/?action=logitems&listviewtype=0" + parent_stream_type = SprintsStream + primary_keys = ["tLogId"] + replication_key = None + schema_filepath = SCHEMAS_DIR / "log_hour.json" + + def parse_response(self, response: requests.Response) -> Iterable[dict]: + """Parse the response and return an iterator of result rows.""" + # Create a record object + yield from property_unfurler( + response=response, + prop_key="log_prop", + ids_key="logIds", + jobj_key="logJObj", + primary_key_name="tLogId", + ) diff --git a/tap_zohosprints/tap.py b/tap_zohosprints/tap.py index 4db2ac1..11d1e11 100644 --- a/tap_zohosprints/tap.py +++ b/tap_zohosprints/tap.py @@ -20,6 +20,7 @@ TagsStream, SprintUsers, ProjectUsers, + LogHours, ) # TODO: Compile a list of custom stream types here @@ -38,6 +39,7 @@ TagsStream, SprintUsers, ProjectUsers, + LogHours, ] diff --git a/tap_zohosprints/tests/tag_property_unfurl_hasnodata.json b/tap_zohosprints/tests/tag_property_unfurl_hasnodata.json new file mode 100644 index 0000000..cf5399d --- /dev/null +++ b/tap_zohosprints/tests/tag_property_unfurl_hasnodata.json @@ -0,0 +1,4 @@ +{ + "hasData": false, + "status": "success" +} diff --git a/tap_zohosprints/tests/test_core.py b/tap_zohosprints/tests/test_core.py index f0190e3..0c3b423 100644 --- a/tap_zohosprints/tests/test_core.py +++ b/tap_zohosprints/tests/test_core.py @@ -52,7 +52,6 @@ def test_property_unfurler(mocked_responses): content_type="application/json", ) resp = requests.get("https://autoidm.com") - assert resp.status_code == 200 unfurled = property_unfurler( response=resp, @@ -79,3 +78,35 @@ def test_property_unfurler(mocked_responses): }, "tagId": "114398000000007021", } + +def test_property_unfurler_nodata(mocked_responses): + """ Sometimes the data comes back empty, we need to not attempt to unfurl the data """ + tag_json = "" + #Should probably just define the json inline as it's way easier to read + with open(Path(__file__).parent / Path("tag_property_unfurl_hasnodata.json")) as tag: + tag_json = tag.read() + + mocked_responses.add( + responses.GET, + "https://autoidm.com", + body=tag_json, + status=200, + content_type="application/json", + ) + resp = requests.get("https://autoidm.com") + + unfurled = property_unfurler( + response=resp, + prop_key="abcdef", + ids_key="hijklmn", + jobj_key="opqrst", + primary_key_name="tuVWxYZ", + ) + output = None + for data in unfurled: + print(data) + output = data + assert output == { + "hasData": False, + "status": "success" + } From cf6abd0744eab08f22386929f3e358f714884e35 Mon Sep 17 00:00:00 2001 From: Derek Visch Date: Tue, 21 Dec 2021 19:34:42 -0500 Subject: [PATCH 3/3] Lint fix --- tap_zohosprints/client.py | 5 +++-- tap_zohosprints/streams.py | 1 + tap_zohosprints/tests/test_core.py | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tap_zohosprints/client.py b/tap_zohosprints/client.py index c0f189c..901b2ab 100644 --- a/tap_zohosprints/client.py +++ b/tap_zohosprints/client.py @@ -153,6 +153,7 @@ def validate_response(self, response): elif 500 <= response.status_code < 600: raise RetriableAPIError(msg) + class ZohoSprintsPropsStream(ZohoSprintsStream): next_page_token_jsonpath = "$.nextIndex" @@ -233,9 +234,9 @@ def property_unfurler( json = response.json() props: Dict = json.get(prop_key) ids: List = json.get(ids_key) - if (json.get("hasData") == False): + if json.get("hasData") == False: return_object: Dict = copy.deepcopy(json) - yield return_object #Data is empty, stop. + yield return_object # Data is empty, stop. else: for id in ids: record = {} diff --git a/tap_zohosprints/streams.py b/tap_zohosprints/streams.py index e44f65e..f69a717 100644 --- a/tap_zohosprints/streams.py +++ b/tap_zohosprints/streams.py @@ -421,6 +421,7 @@ def parse_response(self, response: requests.Response) -> Iterable[dict]: primary_key_name="userId", ) + class LogHours(ZohoSprintsPropsStream): """Log Hours Stream""" diff --git a/tap_zohosprints/tests/test_core.py b/tap_zohosprints/tests/test_core.py index 0c3b423..a985f5f 100644 --- a/tap_zohosprints/tests/test_core.py +++ b/tap_zohosprints/tests/test_core.py @@ -79,11 +79,14 @@ def test_property_unfurler(mocked_responses): "tagId": "114398000000007021", } + def test_property_unfurler_nodata(mocked_responses): """ Sometimes the data comes back empty, we need to not attempt to unfurl the data """ tag_json = "" - #Should probably just define the json inline as it's way easier to read - with open(Path(__file__).parent / Path("tag_property_unfurl_hasnodata.json")) as tag: + # Should probably just define the json inline as it's way easier to read + with open( + Path(__file__).parent / Path("tag_property_unfurl_hasnodata.json") + ) as tag: tag_json = tag.read() mocked_responses.add( @@ -106,7 +109,4 @@ def test_property_unfurler_nodata(mocked_responses): for data in unfurled: print(data) output = data - assert output == { - "hasData": False, - "status": "success" - } + assert output == {"hasData": False, "status": "success"}