diff --git a/pgsync/base.py b/pgsync/base.py index 43a361e..4fcf933 100644 --- a/pgsync/base.py +++ b/pgsync/base.py @@ -778,6 +778,8 @@ def parse_value(self, type_: str, value: str) -> t.Optional[str]: NB: All integers are long in python3 and call to convert is just int """ + if self.verbose: + logger.debug(f"type: {type_} value: {value}") if value.lower() == "null": return None if type_.lower() in self.INT_TYPES: diff --git a/pgsync/constants.py b/pgsync/constants.py index e2982c4..22464c4 100644 --- a/pgsync/constants.py +++ b/pgsync/constants.py @@ -208,5 +208,5 @@ r"table\s\"?(?P[\w-]+)\"?.\"?(?P[\w-]+)\"?:\s(?P[A-Z]+):" # noqa E501 ) LOGICAL_SLOT_SUFFIX = re.compile( - r'\s(?P"?\w+"?)\[(?P[\w\s]+)\]:(?P[\w\'"\-\\.\\+ ]+)' # noqa E501 + r'\s(?P"?\w+"?)\[(?P[\w\s]+)\]:(?P(?:"[^"]*"|\'[^\']*\'|null|\d+e[+-]?\d+|\w+))' ) diff --git a/requirements/base.txt b/requirements/base.txt index 17847bb..8144aac 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,9 +6,9 @@ # async-timeout==4.0.3 # via redis -boto3==1.34.142 +boto3==1.34.153 # via -r requirements/base.in -botocore==1.34.142 +botocore==1.34.153 # via # boto3 # s3transfer @@ -61,13 +61,13 @@ python-dotenv==1.0.1 # via # -r requirements/base.in # environs -redis==5.0.7 +redis==5.0.8 # via -r requirements/base.in requests==2.32.3 # via # opensearch-py # requests-aws4auth -requests-aws4auth==1.2.3 +requests-aws4auth==1.3.1 # via -r requirements/base.in s3transfer==0.10.2 # via boto3 @@ -76,10 +76,9 @@ six==1.16.0 # opensearch-dsl # opensearch-py # python-dateutil - # requests-aws4auth sqlalchemy==2.0.31 # via -r requirements/base.in -sqlparse==0.5.0 +sqlparse==0.5.1 # via -r requirements/base.in typing-extensions==4.12.2 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index 11daa89..76e38fc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,11 +6,11 @@ # async-timeout==4.0.3 # via redis -black==24.4.2 +black==24.8.0 # via -r requirements/dev.in -boto3==1.34.142 +boto3==1.34.153 # via -r requirements/base.in -botocore==1.34.142 +botocore==1.34.153 # via # boto3 # s3transfer @@ -27,7 +27,7 @@ click==8.1.7 # via # -r requirements/base.in # black -coverage[toml]==7.5.4 +coverage[toml]==7.6.0 # via # -r requirements/dev.in # pytest-cov @@ -45,9 +45,9 @@ environs==11.0.0 # via -r requirements/base.in events==0.5 # via opensearch-py -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via pytest -faker==26.0.0 +faker==26.1.0 # via -r requirements/dev.in filelock==3.15.4 # via virtualenv @@ -104,7 +104,7 @@ pycodestyle==2.12.0 # via flake8 pyflakes==3.2.0 # via flake8 -pytest==8.2.2 +pytest==8.3.2 # via # -r requirements/dev.in # pytest-cov @@ -127,13 +127,13 @@ python-dotenv==1.0.1 # environs pyyaml==6.0.1 # via pre-commit -redis==5.0.7 +redis==5.0.8 # via -r requirements/base.in requests==2.32.3 # via # opensearch-py # requests-aws4auth -requests-aws4auth==1.2.3 +requests-aws4auth==1.3.1 # via -r requirements/base.in s3transfer==0.10.2 # via boto3 @@ -142,10 +142,9 @@ six==1.16.0 # opensearch-dsl # opensearch-py # python-dateutil - # requests-aws4auth sqlalchemy==2.0.31 # via -r requirements/base.in -sqlparse==0.5.0 +sqlparse==0.5.1 # via -r requirements/base.in tomli==2.0.1 # via diff --git a/tests/test_base.py b/tests/test_base.py index d0195c7..90abf0f 100644 --- a/tests/test_base.py +++ b/tests/test_base.py @@ -433,12 +433,12 @@ def test_parse_logical_slot( assert "No match for row:" in str(excinfo.value) row = """ - table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue3' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel3' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33 + table public."B1_XYZ": INSERT: "ID"[integer]:5 "CREATED_TIMESTAMP"[bigint]:222 "ADDRESS"[character varying]:'from3' "SOME_FIELD_KEY"[character varying]:'key3' "SOME_OTHER_FIELD_KEY"[character varying]:'issue to handle' "CHANNEL_ID"[integer]:3 "CHANNEL_NAME"[character varying]:'channel 45' "ITEM_ID"[integer]:3 "MESSAGE"[character varying]:'message3' "RETRY"[integer]:4 "STATUS"[character varying]:'status' "SUBJECT"[character varying]:'sub3' "TIMESTAMP"[bigint]:33 """ # noqa E501 payload = pg_base.parse_logical_slot(row) assert payload.data == { "CHANNEL_ID": 3, - "CHANNEL_NAME": "channel3", + "CHANNEL_NAME": "channel 45", "CREATED_TIMESTAMP": 222, "ADDRESS": "from3", "ID": 5, @@ -446,7 +446,7 @@ def test_parse_logical_slot( "MESSAGE": "message3", "RETRY": 4, "SOME_FIELD_KEY": "key3", - "SOME_OTHER_FIELD_KEY": "issue3", + "SOME_OTHER_FIELD_KEY": "issue to handle", "STATUS": "status", "SUBJECT": "sub3", "TIMESTAMP": 33, @@ -472,16 +472,20 @@ def test_parse_logical_slot_with_double_precision( """ # noqa E501 payload = pg_base.parse_logical_slot(row) assert payload.data == { + "copyright": None, + "description": "Stephens Kings It", + "doc": '\'{"a": {"b": {"c": [0, 1, 2, 3, 4]}}, "i": 73, "x": [{"y": 0, "z": ' + '5}, {"y": 1, "z": 6}], "bool": true, "lastname": "Judye", ' + '"firstname": "Glenda", "generation": {"name": "X"}, "nick_names": ' + '["Beatriz", "Jean", "Carilyn", "Carol-Jean", "Sara-Ann"], ' + '"coordinates": {"lat": 21.1, "lon": 32.9}}\'', "id": 1, "isbn": "001", - "title": "It", - "description": "Stephens", - "copyright": None, - "tags": "'", - "doc": "'", + "publish_date": "'1980-01-01 00:00:00'", "publisher_id": 1, - "publish_date": "'1980-01-01", "quad": 2e58, + "tags": '\'["a", "b", "c"]\'', + "title": "It", } assert payload.old == {} assert payload.schema == "public" diff --git a/tests/test_search_client.py b/tests/test_search_client.py index f2ca5ef..eade9b3 100644 --- a/tests/test_search_client.py +++ b/tests/test_search_client.py @@ -109,4 +109,5 @@ def test_get_search_client(self, mocker): use_ssl=True, verify_certs=True, node_class=elastic_transport.RequestsHttpNode, + timeout=settings.ELASTICSEARCH_TIMEOUT, )