diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 80ef3c8..e04aa57 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -27,11 +27,12 @@ jobs: extension-template-capi: name: Extension template (C API) uses: ./.github/workflows/_extension_distribution.yml + if: false # TODO: re-enable when require changes are merged in template repo with: extension_name: capi_quack override_repository: duckdb/extension-template-c override_ref: main - duckdb_version: v1.1.3 + duckdb_version: main override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'python3' diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile index e0af9e1..1fc63a2 100644 --- a/makefiles/c_api_extensions/base.Makefile +++ b/makefiles/c_api_extensions/base.Makefile @@ -2,7 +2,8 @@ # # Inputs # EXTENSION_NAME : name of the extension (lower case) -# MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports +# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension targets +# USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) # EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected # DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected # DUCKDB_TEST_VERSION : the version of DuckDB to test with, if left blank will default to latest stable on PyPi @@ -35,9 +36,9 @@ endif ### Main extension parameters ############################################# -# The minimum DuckDB version that this extension supports -ifeq ($(MINIMUM_DUCKDB_VERSION),) - MINIMUM_DUCKDB_VERSION = v0.0.1 +# The target DuckDB version +ifeq ($(TARGET_DUCKDB_VERSION),) + TARGET_DUCKDB_VERSION = v0.0.1 endif EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension @@ -88,6 +89,16 @@ extension_version: configure/extension_version.txt configure/extension_version.txt: @ $(VERSION_COMMAND) +############################################# +### Parse DuckDB Semver +############################################# + +# Either autodetect or use the provided value +PARSE_SEMVER_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py -s $(TARGET_DUCKDB_VERSION) + +parse_duckdb_version: + @ $(PARSE_SEMVER_COMMAND) + ############################################# ### Testing ############################################# @@ -100,14 +111,19 @@ TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/$(EXTENSI TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/$(EXTENSION_NAME).duckdb_extension # By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version -DUCKDB_INSTALL_VERSION?= -DUCKDB_INSTALL_OPTIONS?= -ifneq ($(DUCKDB_TEST_VERSION),) - DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION) -else ifeq ($(DUCKDB_GIT_VERSION),main) - DUCKDB_INSTALL_OPTIONS= --pre +DUCKDB_PIP_INSTALL?=duckdb +ifeq ($(DUCKDB_TEST_VERSION),main) + DUCKDB_PIP_INSTALL=--pre duckdb +else ifneq ($(DUCKDB_TEST_VERSION),) + DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_TEST_VERSION) +endif + +# This allows C API extensions to be tested against a prerelease of DuckDB. This only really makes sense when DuckDB already +# has stabilized the C API for the upcoming release. +ifeq ($(DUCKDB_GIT_VERSION),main) + DUCKDB_PIP_INSTALL=--pre duckdb else ifneq ($(DUCKDB_GIT_VERSION),) - DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION) + DUCKDB_PIP_INSTALL=duckdb==$(DUCKDB_GIT_VERSION) endif TEST_RELEASE_TARGET=test_extension_release_internal @@ -199,14 +215,19 @@ endif ############################################# ### Adding metadata ############################################# +UNSTABLE_C_API_FLAG= +ifeq ($(USE_UNSTABLE_C_API),1) + UNSTABLE_C_API_FLAG+=--abi-type C_STRUCT_UNSTABLE +endif + build_extension_with_metadata_debug: check_configure link_wasm_debug $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ -l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ - -dv $(MINIMUM_DUCKDB_VERSION) \ + -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ - -pf configure/platform.txt + -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" build_extension_with_metadata_release: check_configure link_wasm_release @@ -214,9 +235,9 @@ build_extension_with_metadata_release: check_configure link_wasm_release -l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ -n $(EXTENSION_NAME) \ - -dv $(MINIMUM_DUCKDB_VERSION) \ + -dv $(TARGET_DUCKDB_VERSION) \ -evf configure/extension_version.txt \ - -pf configure/platform.txt + -pf configure/platform.txt $(UNSTABLE_C_API_FLAG) $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" ############################################# @@ -229,8 +250,9 @@ venv: configure/venv configure/venv: $(PYTHON_BIN) -m venv configure/venv - $(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' $(DUCKDB_INSTALL_OPTIONS) + $(PYTHON_VENV_BIN) -m pip install $(DUCKDB_PIP_INSTALL) $(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python + $(PYTHON_VENV_BIN) -m pip install packaging ############################################# ### Configure diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile index 70aab6a..e90e04a 100644 --- a/makefiles/c_api_extensions/c_cpp.Makefile +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -3,10 +3,8 @@ # Inputs # EXTENSION_NAME : name of the extension (lower case) # EXTENSION_LIB_FILENAME : the library name that is produced by the build -# MINIMUM_DUCKDB_VERSION : full version tag (including v) -# MINIMUM_DUCKDB_VERSION_MAJOR : major version -# MINIMUM_DUCKDB_VERSION_MINOR : minor version -# MINIMUM_DUCKDB_VERSION_PATCH : patch version +# USE_UNSTABLE_C_API : if set to 1, will allow usage of the unstable C API. (This pins the produced binaries to the exact DuckDB version) +# TARGET_DUCKDB_VERSION : the target version of DuckDB that the extension targets # CMAKE_EXTRA_BUILD_FLAGS : additional CMake flags to pass # VCPKG_TOOLCHAIN_PATH : path to vcpkg toolchain # VCPKG_TARGET_TRIPLET : vcpkg triplet to override @@ -18,11 +16,31 @@ ### Base config ############################################# +# Get parsed SemVer for Stable C API +FILE_MAJOR := ./configure/duckdb_version_major.txt +FILE_MINOR := ./configure/duckdb_version_minor.txt +FILE_PATCH := ./configure/duckdb_version_patch.txt +MAJOR_VERSION := $(file < $(FILE_MAJOR)) +MINOR_VERSION := $(file < $(FILE_MINOR)) +PATCH_VERSION := $(file < $(FILE_PATCH)) + # Create build params to pass name and version CMAKE_VERSION_PARAMS = -DEXTENSION_NAME=$(EXTENSION_NAME) -CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MAJOR=$(MINIMUM_DUCKDB_VERSION_MAJOR) -CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MINOR=$(MINIMUM_DUCKDB_VERSION_MINOR) -CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_PATCH=$(MINIMUM_DUCKDB_VERSION_PATCH) + +# Set the parsed semver defines +ifneq ($(MAJOR_VERSION),) + CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MAJOR=$(MAJOR_VERSION) +endif +ifneq ($(MINOR_VERSION),) + CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_MINOR=$(MINOR_VERSION) +endif +ifneq ($(PATCH_VERSION),) + CMAKE_VERSION_PARAMS += -DTARGET_DUCKDB_VERSION_PATCH=$(PATCH_VERSION) +endif + +ifeq ($(USE_UNSTABLE_C_API),1) + CMAKE_VERSION_PARAMS += -DDUCKDB_EXTENSION_API_VERSION_UNSTABLE=$(TARGET_DUCKDB_VERSION) +endif CMAKE_BUILD_FLAGS = $(CMAKE_VERSION_PARAMS) $(CMAKE_EXTRA_BUILD_FLAGS) @@ -116,8 +134,14 @@ build_extension_library_release: check_configure ############################################# ### Misc ############################################# -# TODO: switch this to use the $(MINIMUM_DUCKDB_VERSION) after v1.2.0 is released -BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include +# TODO: switch this to use the $(TARGET_DUCKDB_VERSION) after v1.2.0 is released + +BASE_HEADER_URL= +ifneq ($(TARGET_DUCKDB_VERSION),) + BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/$(TARGET_DUCKDB_VERSION)/src/include +else + BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include +endif DUCKDB_C_HEADER_URL=$(BASE_HEADER_URL)/duckdb.h DUCKDB_C_EXTENSION_HEADER_URL=$(BASE_HEADER_URL)/duckdb_extension.h diff --git a/makefiles/c_api_extensions/rust.Makefile b/makefiles/c_api_extensions/rust.Makefile index 7e01edb..797ba4f 100644 --- a/makefiles/c_api_extensions/rust.Makefile +++ b/makefiles/c_api_extensions/rust.Makefile @@ -30,12 +30,12 @@ endif ############################################# build_extension_library_debug: check_configure - DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(TARGET_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" build_extension_library_release: check_configure - DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(TARGET_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" diff --git a/scripts/configure_helper.py b/scripts/configure_helper.py index 9685410..4448868 100644 --- a/scripts/configure_helper.py +++ b/scripts/configure_helper.py @@ -11,6 +11,7 @@ def main(): arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true') arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true') + arg_parser.add_argument('-s', '--parse-duckdb-semver', type=str, help='Write the parsed DuckDB version SemVer') args = arg_parser.parse_args() @@ -40,6 +41,32 @@ def main(): print(f"Writing platform {duckdb_platform} to {platform_file}") f.write(duckdb_platform) + # Write parsed semver + if args.parse_duckdb_semver: + from packaging.version import Version, InvalidVersion + major_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_major.txt")) + minor_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_minor.txt")) + patch_file = Path(os.path.join(OUTPUT_DIR, "duckdb_version_patch.txt")) + + major_version = "" + minor_version = "" + patch_version = "" + + try: + version = Version(args.parse_duckdb_semver) + major_version = f"{version.major}" + minor_version = f"{version.minor}" + patch_version = f"{version.micro}" + print(f"Written parsed DuckDB semver v{version} to {OUTPUT_DIR}/duckdb_version_.txt") + except InvalidVersion: + print(f"DuckDB version is not a semver, writing empty parsed semver files to {OUTPUT_DIR}/duckdb_version_.txt") + + with open(major_file, 'w') as f: + f.write(major_version) + with open(minor_file, 'w') as f: + f.write(minor_version) + with open(patch_file, 'w') as f: + f.write(patch_version) if __name__ == '__main__': main() \ No newline at end of file