Add BulkResponse wrapper for improved decoding of HTTP bulk responses

.github/workflows/tests.yml

@@ -19,7 +19,7 @@ jobs:
       matrix:
         os: ['ubuntu-22.04', 'macos-latest']
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
-        cratedb-version: ['5.8.3']
+        cratedb-version: ['5.9.2']
 
         # To save resources, only verify the most recent Python versions on macOS.
         exclude:

CHANGES.txt

@@ -12,6 +12,10 @@ Unreleased
   "Threads may share the module, but not connections."
 - Added ``error_trace`` to string representation of an Error to relay
   server stacktraces into exception messages.
+- Refactoring: The module namespace ``crate.client.test_util`` has been
+  renamed to ``crate.testing.util``.
+- Added ``BulkResponse`` wrapper for improved decoding of CrateDB HTTP bulk
+  responses including ``rowcount=`` items.
 
 .. _Migrate from crate.client to sqlalchemy-cratedb: https://cratedb.com/docs/sqlalchemy-cratedb/migrate-from-crate-client.html
 .. _sqlalchemy-cratedb: https://pypi.org/project/sqlalchemy-cratedb/

DEVELOP.rst

@@ -26,34 +26,40 @@ see, for example, `useful command-line options for zope-testrunner`_.
 
 Run all tests::
 
-    ./bin/test -vvvv
+    bin/test
 
 Run specific tests::
 
-    ./bin/test -vvvv -t test_score
+    # Select modules.
+    bin/test -t test_cursor
+    bin/test -t client
+    bin/test -t testing
+
+    # Select doctests.
+    bin/test -t http.rst
 
 Ignore specific test directories::
 
-    ./bin/test -vvvv --ignore_dir=testing
+    bin/test --ignore_dir=testing
 
 The ``LayerTest`` test cases have quite some overhead. Omitting them will save
 a few cycles (~70 seconds runtime)::
 
-    ./bin/test -t '!LayerTest'
+    bin/test -t '!LayerTest'
 
-Invoke all tests without integration tests (~15 seconds runtime)::
+Invoke all tests without integration tests (~10 seconds runtime)::
 
-    ./bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest'
+    bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest'
 
-Yet ~130 test cases, but only ~5 seconds runtime::
+Yet ~60 test cases, but only ~1 second runtime::
 
-    ./bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest' \
+    bin/test --layer '!crate.testing.layer.crate' --test '!LayerTest' \
         -t '!test_client_threaded' -t '!test_no_retry_on_read_timeout' \
         -t '!test_wait_for_http' -t '!test_table_clustered_by'
 
 To inspect the whole list of test cases, run::
 
-    ./bin/test --list-tests
+    bin/test --list-tests
 
 You can run the tests against multiple Python interpreters with `tox`_::
 

bin/test

@@ -12,6 +12,6 @@ sys.argv[0] = os.path.abspath(sys.argv[0])
 
 if __name__ == '__main__':
     zope.testrunner.run([
-        '-vvv', '--auto-color',
-        '--test-path', join(base, 'src')],
-    )
+        '-vvvv', '--auto-color',
+        '--path', join(base, 'tests'),
+    ])

bootstrap.sh

@@ -17,7 +17,7 @@
 # set -x
 
 # Default variables.
-CRATEDB_VERSION=${CRATEDB_VERSION:-5.8.3}
+CRATEDB_VERSION=${CRATEDB_VERSION:-5.9.2}
 
 
 function print_header() {

docs/by-example/connection.rst

@@ -21,7 +21,7 @@ connect()
 This section sets up a connection object, and inspects some of its attributes.
 
     >>> from crate.client import connect
-    >>> from crate.client.test_util import ClientMocked
+    >>> from crate.testing.util import ClientMocked
 
     >>> connection = connect(client=ClientMocked())
     >>> connection.lowest_server_version.version

docs/by-example/cursor.rst

@@ -23,7 +23,7 @@ up the response for subsequent cursor operations.
     >>> from crate.client import connect
     >>> from crate.client.converter import DefaultTypeConverter
     >>> from crate.client.cursor import Cursor
-    >>> from crate.client.test_util import ClientMocked
+    >>> from crate.testing.util import ClientMocked
 
     >>> connection = connect(client=ClientMocked())
     >>> cursor = connection.cursor()

src/crate/client/result.py

@@ -0,0 +1,68 @@
+import typing as t
+from functools import cached_property
+
+
+class BulkResultItem(t.TypedDict):
+    """
+    Define the shape of a CrateDB bulk request response item.
+    """
+
+    rowcount: int
+
+
+class BulkResponse:
+    """
+    Manage a response to a CrateDB bulk request.
+    Accepts a list of bulk arguments (parameter list) and a list of bulk response items.
+
+    https://cratedb.com/docs/crate/reference/en/latest/interfaces/http.html#bulk-operations
+    """
+
+    def __init__(
+            self,
+            records: t.List[t.Dict[str, t.Any]],
+            results: t.List[BulkResultItem]):
+        if records is None:
+            raise ValueError("Processing a bulk response without records is an invalid operation")
+        if results is None:
+            raise ValueError("Processing a bulk response without results is an invalid operation")
+        self.records = records
+        self.results = results
+
+    @cached_property
+    def failed_records(self) -> t.List[t.Dict[str, t.Any]]:
+        """
+        Compute list of failed records.
+
+        CrateDB signals failed inserts using `rowcount=-2`.
+
+        https://cratedb.com/docs/crate/reference/en/latest/interfaces/http.html#error-handling
+        """
+        errors: t.List[t.Dict[str, t.Any]] = []
+        for record, status in zip(self.records, self.results):
+            if status["rowcount"] == -2:
+                errors.append(record)
+        return errors
+
+    @cached_property
+    def record_count(self) -> int:
+        """
+        Compute bulk size / length of parameter list.
+        """
+        if not self.records:
+            return 0
+        return len(self.records)
+
+    @cached_property
+    def success_count(self) -> int:
+        """
+        Compute number of succeeding records within a batch.
+        """
+        return self.record_count - self.failed_count
+
+    @cached_property
+    def failed_count(self) -> int:
+        """
+        Compute number of failed records within a batch.
+        """
+        return len(self.failed_records)

src/crate/testing/util.py

@@ -1,3 +1,74 @@
+# -*- coding: utf-8; -*-
+#
+# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
+# license agreements.  See the NOTICE file distributed with this work for
+# additional information regarding copyright ownership.  Crate licenses
+# this file to you under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# However, if you have executed another commercial license agreement
+# with Crate these terms will supersede the license and you may use the
+# software solely pursuant to the terms of the relevant commercial agreement.
+import unittest
+
+
+class ClientMocked(object):
+
+    active_servers = ["http://localhost:4200"]
+
+    def __init__(self):
+        self.response = {}
+        self._server_infos = ("http://localhost:4200", "my server", "2.0.0")
+
+    def sql(self, stmt=None, parameters=None, bulk_parameters=None):
+        return self.response
+
+    def server_infos(self, server):
+        return self._server_infos
+
+    def set_next_response(self, response):
+        self.response = response
+
+    def set_next_server_infos(self, server, server_name, version):
+        self._server_infos = (server, server_name, version)
+
+    def close(self):
+        pass
+
+
+class ParametrizedTestCase(unittest.TestCase):
+    """
+    TestCase classes that want to be parametrized should
+    inherit from this class.
+
+    https://eli.thegreenplace.net/2011/08/02/python-unit-testing-parametrized-test-cases
+    """
+    def __init__(self, methodName="runTest", param=None):
+        super(ParametrizedTestCase, self).__init__(methodName)
+        self.param = param
+
+    @staticmethod
+    def parametrize(testcase_klass, param=None):
+        """ Create a suite containing all tests taken from the given
+            subclass, passing them the parameter 'param'.
+        """
+        testloader = unittest.TestLoader()
+        testnames = testloader.getTestCaseNames(testcase_klass)
+        suite = unittest.TestSuite()
+        for name in testnames:
+            suite.addTest(testcase_klass(name, param=param))
+        return suite
+
+
 class ExtraAssertions:
     """
     Additional assert methods for unittest.