duckdb: Incorporate changes from NixOS#257452

pkgs/development/libraries/duckdb/default.nix

@@ -2,6 +2,7 @@
 , stdenv
 , fetchFromGitHub
 , cmake
+, git
 , ninja
 , openssl
 , openjdk11
@@ -20,35 +21,24 @@ stdenv.mkDerivation rec {
   src = fetchFromGitHub {
     owner = pname;
     repo = pname;
+    deepClone = true;
     rev = "v${version}";
     hash = "sha256-EKvDH7RwOC4Gu/lturrfnGpzXnJ9azIwAFeuVoa6L/Y=";
   };
 
-  patches = [ ./version.patch ];
+  nativeBuildInputs = [ cmake git ninja ];
 
-  postPatch = ''
-    substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"
-  '';
-
-  nativeBuildInputs = [ cmake ninja ];
   buildInputs = [ openssl ]
     ++ lib.optionals withJdbc [ openjdk11 ]
     ++ lib.optionals withOdbc [ unixODBC ];
 
   cmakeFlags = [
-    "-DBUILD_AUTOCOMPLETE_EXTENSION=ON"
+    # use similar flags to what is defined in ${src}/.github/workflow/{LinuxRelease,OSX}.yml
     "-DBUILD_ICU_EXTENSION=ON"
+    "-DDEBUG_STACKTRACE=1"
     "-DBUILD_PARQUET_EXTENSION=ON"
+    "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"
     "-DBUILD_TPCH_EXTENSION=ON"
-    "-DBUILD_TPCDS_EXTENSION=ON"
-    "-DBUILD_FTS_EXTENSION=ON"
-    "-DBUILD_HTTPFS_EXTENSION=ON"
-    "-DBUILD_VISUALIZER_EXTENSION=ON"
-    "-DBUILD_JSON_EXTENSION=ON"
-    "-DBUILD_JEMALLOC_EXTENSION=ON"
-    "-DBUILD_EXCEL_EXTENSION=ON"
-    "-DBUILD_INET_EXTENSION=ON"
-    "-DBUILD_TPCE=ON"
     "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"
     "-DJDBC_DRIVER=${enableFeature withJdbc}"
   ] ++ lib.optionals doInstallCheck [
@@ -66,44 +56,34 @@ stdenv.mkDerivation rec {
 
   installCheckPhase =
     let
-      excludes = map (pattern: "exclude:'${pattern}'") [
-        "[s3]"
+      excludes = map (pattern: "exclude:'${pattern}'") ([
+        "Test using a remote optimizer pass in case thats important to someone"
         "Test closing database during long running query"
-        "test/common/test_cast_hugeint.test"
-        "test/sql/copy/csv/test_csv_remote.test"
-        "test/sql/copy/parquet/test_parquet_remote.test"
-        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
-        "test/sql/storage/compression/chimp/chimp_read.test"
-        "test/sql/storage/compression/chimp/chimp_read_float.test"
-        "test/sql/storage/compression/patas/patas_compression_ratio.test_coverage"
-        "test/sql/storage/compression/patas/patas_read.test"
-        "test/sql/json/read_json_objects.test"
-        "test/sql/json/read_json.test"
-        "test/sql/copy/parquet/parquet_5968.test"
         "test/fuzzer/pedro/buffer_manager_out_of_memory.test"
-        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
+        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
+        "test/sql/copy/csv/parallel/test_parallel_csv.test"
+        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
+        "test/sql/copy/csv/test_csv_httpfs.test"
+        "test/sql/copy/csv/test_csv_remote.test"
         "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"
-        "test/sql/function/timestamp/test_icu_strptime.test"
-        "test/sql/timezone/test_icu_timezone.test"
+        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
+        "test/sql/copy/parquet/parquet_5968.test"
         "test/sql/copy/parquet/snowflake_lineitem.test"
         "test/sql/copy/parquet/test_parquet_force_download.test"
-        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"
-        "test/sql/copy/csv/test_csv_httpfs_prepared.test"
-        "test/sql/copy/csv/test_csv_httpfs.test"
-        "test/sql/copy/csv/parallel/test_parallel_csv.test"
-        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"
-        "test/common/test_cast_struct.test"
-        # test is order sensitive
-        "test/sql/copy/parquet/parquet_glob.test"
+        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"
+        "test/sql/copy/parquet/test_parquet_remote.test"
+        "test/sql/json/table/read_json_objects.test"
+        "test/sql/json/table/read_json.test"
+        "test/sql/settings/test_disabled_file_system_httpfs.test"
+        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"
+
         # these are only hidden if no filters are passed in
         "[!hide]"
-        # this test apparently never terminates
-        "test/sql/copy/csv/auto/test_csv_auto.test"
       ] ++ lib.optionals stdenv.isAarch64 [
         "test/sql/aggregate/aggregates/test_kurtosis.test"
         "test/sql/aggregate/aggregates/test_skewness.test"
         "test/sql/function/list/aggregates/skewness.test"
-      ];
+      ]);
     in
     ''
       runHook preInstallCheck

pkgs/development/python-modules/duckdb-engine/default.nix

@@ -14,7 +14,7 @@
 
 buildPythonPackage rec {
   pname = "duckdb-engine";
-  version = "0.7.3";
+  version = "0.9.2";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
@@ -23,9 +23,11 @@ buildPythonPackage rec {
     repo = "duckdb_engine";
     owner = "Mause";
     rev = "refs/tags/v${version}";
-    hash = "sha256-Z9m1+Bc/csWKdPDuwf82xX0qOiD1Y5LBgJjUlLntAO8=";
+    hash = "sha256-T02nGF+YlughRQPinb0I3NC6xsarh4+qRhG8YfhTvhI=";
   };
 
+   patches = [ ./remote_data.patch ];
+
   nativeBuildInputs = [
     poetry-core
   ];
@@ -42,8 +44,11 @@ buildPythonPackage rec {
   disabledTests = [
     # this test tries to download the httpfs extension
     "test_preload_extension"
+    "test_motherduck"
     # test should be skipped based on sqlalchemy version but isn't and fails
     "test_commit"
+    # rowcount no longer generates an attribute error.
+    "test_rowcount"
   ];
 
   nativeCheckInputs = [

pkgs/development/python-modules/duckdb-engine/remote_data.patch

@@ -0,0 +1,24 @@
+diff --git a/duckdb_engine/tests/test_basic.py b/duckdb_engine/tests/test_basic.py
+index 302636f..ed20f12 100644
+--- a/duckdb_engine/tests/test_basic.py
++++ b/duckdb_engine/tests/test_basic.py
+@@ -183,7 +183,6 @@ def test_get_views(engine: Engine) -> None:
+
+
+ @mark.skipif(os.uname().machine == "aarch64", reason="not supported on aarch64")
+-@mark.remote_data
+ def test_preload_extension() -> None:
+     duckdb.default_connection.execute("INSTALL httpfs")
+     engine = create_engine(
+diff --git a/duckdb_engine/tests/test_integration.py b/duckdb_engine/tests/test_integration.py
+index 349c976..bf3dbaa 100644
+--- a/duckdb_engine/tests/test_integration.py
++++ b/duckdb_engine/tests/test_integration.py
+@@ -24,7 +24,6 @@ def test_integration(engine: Engine) -> None:
+         conn.execute(text("select * from test_df"))
+
+
+-@mark.remote_data
+ @mark.skipif(
+     "dev" in duckdb.__version__, reason="md extension not available for dev builds"  # type: ignore[attr-defined]
+ )

pkgs/development/python-modules/duckdb/default.nix

@@ -2,8 +2,10 @@
 , buildPythonPackage
 , duckdb
 , fsspec
+, git
 , google-cloud-storage
 , numpy
+, openssl
 , pandas
 , psutil
 , pybind11
@@ -12,40 +14,45 @@
 }:
 
 buildPythonPackage rec {
-  inherit (duckdb) pname version src patches;
+  inherit (duckdb) pname version src;
   format = "setuptools";
 
+  BUILD_HTTPFS = 1;
+  patches = [ ./setup.patch ];
+
   postPatch = ''
     # we can't use sourceRoot otherwise patches don't apply, because the patches apply to the C++ library
     cd tools/pythonpkg
 
     # 1. let nix control build cores
     # 2. unconstrain setuptools_scm version
     substituteInPlace setup.py \
-      --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES" \
-      --replace "setuptools_scm<7.0.0" "setuptools_scm"
+      --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"
 
-      # avoid dependency on mypy
-      rm tests/stubs/test_stubs.py
+    # avoid dependency on mypy
+    rm tests/stubs/test_stubs.py
   '';
 
   SETUPTOOLS_SCM_PRETEND_VERSION = version;
 
   nativeBuildInputs = [
+    git
     pybind11
     setuptools-scm
   ];
 
+  buildInputs = [ openssl ];
+
   propagatedBuildInputs = [
     numpy
     pandas
   ];
 
   nativeCheckInputs = [
+    fsspec
     google-cloud-storage
     psutil
     pytestCheckHook
-    fsspec
   ];
 
   disabledTests = [

pkgs/development/python-modules/duckdb/setup.patch

@@ -0,0 +1,22 @@
+diff --git a/tools/pythonpkg/setup.py b/tools/pythonpkg/setup.py
+index fdf2911019..c363cc518a 100644
+--- a/tools/pythonpkg/setup.py
++++ b/tools/pythonpkg/setup.py
+@@ -163,8 +163,6 @@ if 'BUILD_HTTPFS' in os.environ:
+ for ext in extensions:
+     toolchain_args.extend(['-DDUCKDB_EXTENSION_{}_LINKED'.format(ext.upper())])
+
+-toolchain_args.extend(['-DDUCKDB_EXTENSION_AUTOLOAD_DEFAULT=1', '-DDUCKDB_EXTENSION_AUTOINSTALL_DEFAULT=1'])
+-
+
+ class get_pybind_include(object):
+     def __init__(self, user=False):
+@@ -343,7 +341,7 @@ setup(
+     packages=packages,
+     include_package_data=True,
+     python_requires='>=3.7.0',
+-    setup_requires=setup_requires + ["setuptools_scm<7.0.0", 'pybind11>=2.6.0'],
++    setup_requires=setup_requires + ["setuptools_scm", 'pybind11>=2.6.0'],
+     use_scm_version=setuptools_scm_conf,
+     tests_require=['google-cloud-storage', 'mypy', 'pytest'],
+     classifiers=[

pkgs/development/python-modules/ibis-framework/default.nix

@@ -130,6 +130,17 @@ buildPythonPackage rec {
     # tries to download duckdb extensions
     "--deselect=ibis/backends/duckdb/tests/test_register.py::test_register_sqlite"
     "--deselect=ibis/backends/duckdb/tests/test_register.py::test_read_sqlite"
+
+    # duckdb does not respect sample_size=2 (reads 3 lines of csv).
+    "--deselect=ibis/backends/tests/test_register.py::test_csv_reregister_schema"
+
+    # duckdb fails with:
+    # "This function can not be called with an active transaction!, commit or abort the existing one first"
+    "--deselect=ibis/backends/tests/test_udf.py::test_vectorized_udf"
+    "--deselect=ibis/backends/tests/test_udf.py::test_map_merge_udf"
+    "--deselect=ibis/backends/tests/test_udf.py::test_udf"
+    "--deselect=ibis/backends/tests/test_udf.py::test_map_udf"
+
     # pyarrow13 is not supported yet.
     "--deselect=ibis/backends/tests/test_temporal.py::test_date_truncate"
     "--deselect=ibis/backends/tests/test_temporal.py::test_integer_to_interval_timestamp"