diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index c1ffac4..5672617 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -23,7 +23,31 @@ jobs: override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;python3' - custom_toolchain_script: true + + extension-template-capi: + name: Extension template (C API) + uses: ./.github/workflows/_extension_distribution.yml + with: + extension_name: capi_quack + override_repository: duckdb/extension-template-c + override_ref: main + duckdb_version: v1.1.3 + override_ci_tools_repository: ${{ github.repository }} + ci_tools_version: ${{ github.sha }} + extra_toolchains: 'python3' + + extension-template-rust: + name: Extension template (Rust) + uses: ./.github/workflows/_extension_distribution.yml + with: + extension_name: rusty_quack + override_repository: duckdb/extension-template-rs + override_ref: main + duckdb_version: v1.1.3 + override_ci_tools_repository: ${{ github.repository }} + ci_tools_version: ${{ github.sha }} + extra_toolchains: 'rust;python3' + exclude_archs: 'windows_amd64_rtools;windows_amd64_mingw' # TODO: remove once fixed upstream delta-extension-main: name: Rust builds (using Delta extension) diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml index b9912f2..d9d68bc 100644 --- a/.github/workflows/_extension_deploy.yml +++ b/.github/workflows/_extension_deploy.yml @@ -18,6 +18,16 @@ on: duckdb_version: required: true type: string + # The version of the https://github.com/duckdb/extension-ci-tools submodule of the extension. In most cases will be identical to `duckdb_version`. + # Passing this explicitly is required because of https://github.com/actions/toolkit/issues/1264 + ci_tools_version: + required: true + type: string + # Override the repo for the CI tools (for testing CI tools itself) + override_ci_tools_repository: + required: false + type: string + default: "duckdb/extension-ci-tools" # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' exclude_archs: required: false @@ -47,7 +57,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" jobs: generate_matrix: @@ -57,18 +67,15 @@ jobs: deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} steps: - uses: actions/checkout@v4 + name: Checkout Extension CI tools with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} - id: parse-matrices run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty deploy_matrix="`cat deploy_matrix.json`" echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT echo `cat $GITHUB_OUTPUT` @@ -92,7 +99,7 @@ jobs: cd duckdb git checkout ${{ inputs.duckdb_version }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} path: | diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 3bd4cb0..6d527cb 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -49,7 +49,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" # Enable building the DuckDB Shell build_duckdb_shell: required: false @@ -76,11 +76,6 @@ on: required: false type: string default: "" - # If set, the setup will look for a script in `./scripts/custom-toolchain-script.sh` to run - custom_toolchain_script: - required: false - type: boolean - default: false rust_logs: required: false type: boolean @@ -105,6 +100,7 @@ on: required: false type: boolean default: false + jobs: generate_matrix: name: Generate matrix @@ -115,22 +111,6 @@ jobs: osx_matrix: ${{ steps.set-matrix-osx.outputs.osx_matrix }} wasm_matrix: ${{ steps.set-matrix-wasm.outputs.wasm_matrix }} steps: - - uses: actions/checkout@v4 - name: Checkout override repository - if: ${{inputs.override_repository != ''}} - with: - repository: ${{ inputs.override_repository }} - ref: ${{ inputs.override_ref }} - fetch-depth: 0 - submodules: 'true' - - - uses: actions/checkout@v4 - name: Checkout current repository - if: ${{inputs.override_repository == ''}} - with: - fetch-depth: 0 - submodules: 'true' - - uses: actions/checkout@v4 name: Checkout Extension CI tools with: @@ -138,34 +118,13 @@ jobs: ref: ${{ inputs.ci_tools_version }} repository: ${{ inputs.override_ci_tools_repository }} - - name: Checkout DuckDB to version - if: ${{inputs.duckdb_version != ''}} - run: | - DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }} make set_duckdb_version - - - name: Tag extension - if: ${{inputs.extension_tag != ''}} - run: | - git tag ${{ inputs.extension_tag }} - - - name: Tag DuckDB extension - if: ${{inputs.duckdb_tag != '' && (inputs.duckdb_version == 'main' || inputs.duckdb_version[0] == 'v') }} - run: | - echo "When duckdb_tag is provied an explcit git ref is expected" && exit 1 - - - name: Tag DuckDB extension - if: ${{inputs.duckdb_tag != ''}} - run: | - DUCKDB_TAG=${{ inputs.duckdb_tag }} make set_duckdb_tag - - id: parse-matrices run: | mkdir build - make output_distribution_matrix | tail -n +2 > build/distribution_matrix.json - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - python3 ${{ inputs.matrix_parse_script }} --input build/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os linux --output build/linux_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os osx --output build/osx_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os windows --output build/windows_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty + python3 ${{ inputs.matrix_parse_script }} --input extension-ci-tools/config/distribution_matrix.json --select_os wasm --output build/wasm_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - id: set-matrix-linux run: | @@ -283,6 +242,8 @@ jobs: echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> docker_env.txt echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt echo "DUCKDB_PLATFORM=${{ matrix.duckdb_arch }}" >> docker_env.txt + echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt + echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt echo "TOOLCHAIN_FLAGS=${{ matrix.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt - name: Generate timestamp for Ccache entry @@ -304,16 +265,37 @@ jobs: restore-keys: | ccache-extension-distribution-${{ matrix.duckdb_arch }}- - - name: Build extension + - name: Run configure (outside Docker) + shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 + run: | + make configure_ci + + - name: Run configure (inside Docker) + shell: bash + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make configure_ci + + - name: Build extension (inside Docker) run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release - - name: Test extension + - name: Test extension (inside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} run: | - docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release - - uses: actions/upload-artifact@v3 + - name: Test extension (outside docker) + if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 + run: | + make test_release + + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} path: | @@ -362,7 +344,7 @@ jobs: - name: Install Ninja run: | - brew install ninja autoconf make libtool pkg-config automake autoconf-archive + brew install ninja autoconf make libtool automake autoconf-archive - name: Setup Ccache uses: hendrikmuhs/ccache-action@main @@ -439,16 +421,15 @@ jobs: echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build extension shell: bash - env: - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} run: | make release @@ -456,9 +437,9 @@ jobs: if: ${{ matrix.osx_build_arch == 'arm64' && inputs.skip_tests == false }} shell: bash run: | - make test + make test_release - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} @@ -583,21 +564,23 @@ jobs: vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} vcpkgGitURL: ${{ inputs.vcpkg_url }} - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure shell: bash + env: + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build extension - if: ${{ inputs.skip_tests == true }} env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} run: | make release - - name: Build & test extension + - name: Test extension if: ${{ inputs.skip_tests == false }} env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} @@ -605,7 +588,7 @@ jobs: run: | make test_release - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} @@ -701,17 +684,18 @@ jobs: with: key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build Wasm module run: | make ${{ matrix.duckdb_arch }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: if-no-files-found: error name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}} diff --git a/README.md b/README.md index f20124b..2992f46 100644 --- a/README.md +++ b/README.md @@ -18,4 +18,4 @@ DuckDB's [Extension Template](https://github.com/duckdb/extension-template/actio | v0.10.0 | v0.10.0 | no | Each branch in this repository targets a specific version of DuckDB. Note that these branches will be continually updated to ensure the build environment is functional for that version of DuckDB. -Also note that at some point, support for versions will be dropped. Currently, we aim to support the latest 2 DuckDB versions, to allow extensions devs to transition to a new DuckDB version. +Also note that at some point, support for versions will be dropped. Currently, we aim to support the latest 2 DuckDB versions, to allow extensions devs to transition to a new DuckDB version. \ No newline at end of file diff --git a/makefiles/c_api_extensions/base.Makefile b/makefiles/c_api_extensions/base.Makefile new file mode 100644 index 0000000..ecb2e7e --- /dev/null +++ b/makefiles/c_api_extensions/base.Makefile @@ -0,0 +1,258 @@ +# Reusable makefile for C API based extensions +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# MINIMUM_DUCKDB_VERSION : the minimum version of DuckDB that the extension supports +# EXTENSION_VERSION : the version of the extension, if left blank it will be autodetected +# DUCKDB_PLATFORM : the platform of the extension, if left blank it will be autodetected +# DUCKDB_TEST_VERSION : the version of DuckDB to test with, if left blank will default to latest stable on PyPi +# DUCKDB_GIT_VERSION : set by CI currently, should probably be removed at some point +# LINUX_CI_IN_DOCKER : indicates that the build is being run in/out of Docker in the linux CI +# SKIP_TESTS : makes the test targets turn into NOPs + +.PHONY: platform extension_version test_extension_release test_extension_debug test_extension_release_internal test_extension_debug_internal tests_skipped clean_build clean_configure nop set_duckdb_tag set_duckdb_version output_distribution_matrix venv configure_ci check_configure move_wasm_extension + +############################################# +### Platform dependent config +############################################# +PYTHON_BIN=python3 + +ifeq ($(OS),Windows_NT) + EXTENSION_LIB_FILENAME=$(EXTENSION_NAME).dll + PYTHON_VENV_BIN=./configure/venv/Scripts/python.exe +else + PYTHON_VENV_BIN=./configure/venv/bin/python3 + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).so + endif + ifeq ($(UNAME_S),Darwin) + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).dylib + endif +endif + +############################################# +### Main extension parameters +############################################# + +# The minimum DuckDB version that this extension supports +ifeq ($(MINIMUM_DUCKDB_VERSION),) + MINIMUM_DUCKDB_VERSION = v0.0.1 +endif + +EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension +EXTENSION_FILENAME_NO_METADATA=$(EXTENSION_LIB_FILENAME) + +DUCKDB_WASM_PLATFORM=$(filter wasm_mvp wasm_eh wasm_threads,$(DUCKDB_PLATFORM)) + +ifneq ($(DUCKDB_WASM_PLATFORM),) + EXTENSION_FILENAME=$(EXTENSION_NAME).duckdb_extension.wasm + EXTENSION_FILENAME_NO_METADATA=$(EXTENSION_NAME).no_metadata.wasm + EXTENSION_LIB_FILENAME=lib$(EXTENSION_NAME).a +endif + +############################################# +### Platform Detection +############################################# + +# Write the platform we are building for +platform: configure/platform.txt + +# Either autodetect or use the provided value +PLATFORM_COMMAND?= +ifeq ($(DUCKDB_PLATFORM),) + PLATFORM_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --duckdb-platform +else + # Sets the platform using DUCKDB_PLATFORM variable + PLATFORM_COMMAND=echo $(DUCKDB_PLATFORM) > configure/platform.txt +endif + +configure/platform.txt: + @ $(PLATFORM_COMMAND) + +############################################# +### Extension Version Detection +############################################# + +# Either autodetect or use the provided value +VERSION_COMMAND?= +ifeq ($(EXTENSION_VERSION),) + VERSION_COMMAND=$(PYTHON_VENV_BIN) extension-ci-tools/scripts/configure_helper.py --extension-version +else + # Sets the platform using DUCKDB_PLATFORM variable + VERSION_COMMAND=echo "$(EXTENSION_VERSION)" > configure/extension_version.txt +endif + +extension_version: configure/extension_version.txt + +configure/extension_version.txt: + @ $(VERSION_COMMAND) + +############################################# +### Testing +############################################# + +# Note: to override the default test runner, create a symlink to a different venv +TEST_RUNNER=$(PYTHON_VENV_BIN) -m duckdb_sqllogictest + +TEST_RUNNER_BASE=$(TEST_RUNNER) --test-dir test/sql $(EXTRA_EXTENSIONS_PARAM) +TEST_RUNNER_DEBUG=$(TEST_RUNNER_BASE) --external-extension build/debug/$(EXTENSION_NAME).duckdb_extension +TEST_RUNNER_RELEASE=$(TEST_RUNNER_BASE) --external-extension build/release/$(EXTENSION_NAME).duckdb_extension + +# By default latest duckdb is installed, set DUCKDB_TEST_VERSION to switch to a different version +DUCKDB_INSTALL_VERSION?= +ifneq ($(DUCKDB_TEST_VERSION),) + DUCKDB_INSTALL_VERSION===$(DUCKDB_TEST_VERSION) +endif + +ifneq ($(DUCKDB_GIT_VERSION),) + DUCKDB_INSTALL_VERSION===$(DUCKDB_GIT_VERSION) +endif + +TEST_RELEASE_TARGET=test_extension_release_internal +TEST_DEBUG_TARGET=test_extension_debug_internal + +# Disable testing outside docker: the unittester is currently dynamically linked by default +ifeq ($(LINUX_CI_IN_DOCKER),1) + SKIP_TESTS=1 +endif + +# TODO: for some weird reason the Ubuntu 22.04 Runners on Github Actions don't actually grab the glibc 2.24 wheels but the +# gilbc 2.17 ones. What this means is that we can't run the tests on linux_amd64 because we are installing the duckdb +# linux_amd64_gcc4 test runner +ifeq ($(DUCKDB_PLATFORM),linux_amd64) + SKIP_TESTS=1 +endif + +# The mingw/rtools can not be tested using the python test runner unfortunately +ifeq ($(DUCKDB_PLATFORM),windows_amd64_rtools) + SKIP_TESTS=1 +endif +ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw) + SKIP_TESTS=1 +endif + +ifeq ($(SKIP_TESTS),1) + TEST_RELEASE_TARGET=tests_skipped + TEST_DEBUG_TARGET=tests_skipped +endif + +test_extension_release: $(TEST_RELEASE_TARGET) +test_extension_debug: $(TEST_DEBUG_TARGET) + +test_extension_release_internal: check_configure + @echo "Running RELEASE tests.." + @$(TEST_RUNNER_RELEASE) + +test_extension_debug_internal: check_configure + @echo "Running DEBUG tests.." + @$(TEST_RUNNER_DEBUG) + +tests_skipped: + @echo "Skipping tests.." + + +############################################# +### Misc +############################################# + +clean_build: + rm -rf build + rm -rf duckdb_unittest_tempdir + +clean_configure: + rm -rf configure + +nop: + @echo "NOP" + +set_duckdb_tag: nop + +set_duckdb_version: nop + +output_distribution_matrix: + cat extension-ci-tools/config/distribution_matrix.json + +############################################# +### Linking +############################################# +ifneq ($(DUCKDB_WASM_PLATFORM),) + +link_wasm_debug: + emcc build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME) -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) -O3 -g -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" + +link_wasm_release: + emcc build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME) -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) -O3 -sSIDE_MODULE=2 -sEXPORTED_FUNCTIONS="_$(EXTENSION_NAME)_init_c_api" + +else +link_wasm_debug: +link_wasm_release: + +endif + +############################################# +### Adding metadata +############################################# +build_extension_with_metadata_debug: check_configure link_wasm_debug + $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ + -l build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME) \ + -n $(EXTENSION_NAME) \ + -dv $(MINIMUM_DUCKDB_VERSION) \ + -evf configure/extension_version.txt \ + -pf configure/platform.txt + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +build_extension_with_metadata_release: check_configure link_wasm_release + $(PYTHON_VENV_BIN) extension-ci-tools/scripts/append_extension_metadata.py \ + -l build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME_NO_METADATA) \ + -o build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME) \ + -n $(EXTENSION_NAME) \ + -dv $(MINIMUM_DUCKDB_VERSION) \ + -evf configure/extension_version.txt \ + -pf configure/platform.txt + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +############################################# +### Python +############################################# + +# Installs the test runner using the selected DuckDB version (latest stable by default) +# TODO: switch to PyPI distribution +venv: configure/venv + +configure/venv: + $(PYTHON_BIN) -m venv configure/venv + $(PYTHON_VENV_BIN) -m pip install 'duckdb$(DUCKDB_INSTALL_VERSION)' + $(PYTHON_VENV_BIN) -m pip install git+https://github.com/duckdb/duckdb-sqllogictest-python + +############################################# +### Configure +############################################# + +CONFIGURE_CI_STEP?= +ifeq ($(LINUX_CI_IN_DOCKER),1) + CONFIGURE_CI_STEP=nop +else + CONFIGURE_CI_STEP=configure +endif + +configure_ci: $(CONFIGURE_CI_STEP) + +# Because the configure_ci may differ from configure, we don't automatically run configure on make build, this makes the error a bit nicer +check_configure: + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/platform.txt'), 'The configure step appears to not be run. Please try running make configure'" + @$(PYTHON_BIN) -c "import os; assert os.path.exists('configure/venv'), 'The configure step appears to not be run. Please try running make configure'" + +move_wasm_extension: + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/extension/$(EXTENSION_NAME)/$(EXTENSION_FILENAME)')" + +wasm_mvp: + DUCKDB_PLATFORM=wasm_mvp make configure release move_wasm_extension + +wasm_eh: + DUCKDB_PLATFORM=wasm_eh make configure release move_wasm_extension + +wasm_threads: + DUCKDB_PLATFORM=wasm_threads make configure release move_wasm_extension diff --git a/makefiles/c_api_extensions/c_cpp.Makefile b/makefiles/c_api_extensions/c_cpp.Makefile new file mode 100644 index 0000000..70aab6a --- /dev/null +++ b/makefiles/c_api_extensions/c_cpp.Makefile @@ -0,0 +1,129 @@ +# Reusable makefile for the C/C++ extensions targeting the C extension API +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# EXTENSION_LIB_FILENAME : the library name that is produced by the build +# MINIMUM_DUCKDB_VERSION : full version tag (including v) +# MINIMUM_DUCKDB_VERSION_MAJOR : major version +# MINIMUM_DUCKDB_VERSION_MINOR : minor version +# MINIMUM_DUCKDB_VERSION_PATCH : patch version +# CMAKE_EXTRA_BUILD_FLAGS : additional CMake flags to pass +# VCPKG_TOOLCHAIN_PATH : path to vcpkg toolchain +# VCPKG_TARGET_TRIPLET : vcpkg triplet to override +# GEN : allow specifying ninja as generator + +.PHONY: build_extension_library_debug build_extension_library_release update_duckdb_headers + +############################################# +### Base config +############################################# + +# Create build params to pass name and version +CMAKE_VERSION_PARAMS = -DEXTENSION_NAME=$(EXTENSION_NAME) +CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MAJOR=$(MINIMUM_DUCKDB_VERSION_MAJOR) +CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_MINOR=$(MINIMUM_DUCKDB_VERSION_MINOR) +CMAKE_VERSION_PARAMS += -DMINIMUM_DUCKDB_VERSION_PATCH=$(MINIMUM_DUCKDB_VERSION_PATCH) + +CMAKE_BUILD_FLAGS = $(CMAKE_VERSION_PARAMS) $(CMAKE_EXTRA_BUILD_FLAGS) + +############################################# +### Vcpkg +############################################# + +ifneq ("${VCPKG_TOOLCHAIN_PATH}", "") + CMAKE_BUILD_FLAGS += -DCMAKE_TOOLCHAIN_FILE='${VCPKG_TOOLCHAIN_PATH}' + ifneq ($(DUCKDB_WASM_PLATFORM),) + CMAKE_BUILD_FLAGS += -DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake + endif +endif +ifneq ("${VCPKG_TARGET_TRIPLET}", "") + CMAKE_BUILD_FLAGS += -DVCPKG_TARGET_TRIPLET='${VCPKG_TARGET_TRIPLET}' +endif + +############################################# +### Ninja +############################################# + +MAKE_INVOCATION = make + +ifeq ($(GEN),ninja) + CMAKE_BUILD_FLAGS += -G "Ninja" + MAKE_INVOCATION = ninja +endif + +############################################# +### Windows weirdness +############################################# + +ifeq ($(OS),Windows_NT) + OUTPUT_LIB_PATH_DEBUG=cmake_build/debug/Debug/$(EXTENSION_LIB_FILENAME) + OUTPUT_LIB_PATH_RELEASE=cmake_build/release/Release/$(EXTENSION_LIB_FILENAME) +else + OUTPUT_LIB_PATH_DEBUG=cmake_build/debug/$(EXTENSION_LIB_FILENAME) + OUTPUT_LIB_PATH_RELEASE=cmake_build/release/$(EXTENSION_LIB_FILENAME) +endif + +ifeq ($(DUCKDB_PLATFORM),windows_amd64_rtools) + MINGW=1 +endif +ifeq ($(DUCKDB_PLATFORM),windows_amd64_mingw) + MINGW=1 +endif +ifeq ($(MINGW),1) + EXTRA_COPY_STEP_DEBUG=$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./cmake_build/debug/Debug').mkdir(parents=True, exist_ok=True);import shutil;shutil.copyfile('cmake_build/debug/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_DEBUG)')" + EXTRA_COPY_STEP_RELEASE=$(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./cmake_build/release/Release').mkdir(parents=True, exist_ok=True);import shutil;shutil.copyfile('cmake_build/release/lib$(EXTENSION_LIB_FILENAME)', '$(OUTPUT_LIB_PATH_RELEASE)')" +endif + +CMAKE_WRAPPER= +CMAKE_BUILD_DEBUG = cmake --build cmake_build/debug --config Debug +CMAKE_BUILD_RELEASE = cmake --build cmake_build/release --config Release +EXTRA_CMAKE_FLAGS ?= + +ifneq ($(DUCKDB_WASM_PLATFORM),) + CMAKE_BUILD_DEBUG = $(MAKE_INVOCATION) -C cmake_build/debug + CMAKE_BUILD_RELEASE = $(MAKE_INVOCATION) -C cmake_build/release + CMAKE_WRAPPER=emcmake + CMAKE_BUILD_FLAGS += -DCMAKE_POSITION_INDEPENDENT_CODE=ON + EXTRA_CMAKE_FLAGS += -DDUCKDB_WASM_EXTENSION=1 + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_mvp') + endif + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_eh') + CMAKE_CXX_FLAGS += -fwasm-exceptions + endif + ifeq ($(DUCKDB_WASM_PLATFORM), 'wasm_threads') + CMAKE_CXX_FLAGS += -fwasm-exceptions -DWITH_WASM_THREADS=1 -DWITH_WASM_SIMD=1 -DWITH_WASM_BULK_MEMORY=1 -pthread + endif +endif + +############################################# +### Build targets +############################################# + +build_extension_library_debug: check_configure + $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Debug -S $(PROJ_DIR) -B cmake_build/debug $(EXTRA_CMAKE_FLAGS) + $(CMAKE_BUILD_DEBUG) + $(EXTRA_COPY_STEP_DEBUG) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_DEBUG)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" + +build_extension_library_release: check_configure + $(CMAKE_WRAPPER) cmake $(CMAKE_BUILD_FLAGS) -DCMAKE_BUILD_TYPE=Release -S $(PROJ_DIR) -B cmake_build/release $(EXTRA_CMAKE_FLAGS) + $(CMAKE_BUILD_RELEASE) + $(EXTRA_COPY_STEP_RELEASE) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('$(OUTPUT_LIB_PATH_RELEASE)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" + +############################################# +### Misc +############################################# +# TODO: switch this to use the $(MINIMUM_DUCKDB_VERSION) after v1.2.0 is released +BASE_HEADER_URL=https://raw.githubusercontent.com/duckdb/duckdb/refs/heads/main/src/include +DUCKDB_C_HEADER_URL=$(BASE_HEADER_URL)/duckdb.h +DUCKDB_C_EXTENSION_HEADER_URL=$(BASE_HEADER_URL)/duckdb_extension.h + +update_duckdb_headers: + $(PYTHON_VENV_BIN) -c "import urllib.request;urllib.request.urlretrieve('$(DUCKDB_C_HEADER_URL)', 'duckdb_capi/duckdb.h')" + $(PYTHON_VENV_BIN) -c "import urllib.request;urllib.request.urlretrieve('$(DUCKDB_C_EXTENSION_HEADER_URL)', 'duckdb_capi/duckdb_extension.h')" + +clean_cmake: + rm -rf cmake_build diff --git a/makefiles/c_api_extensions/rust.Makefile b/makefiles/c_api_extensions/rust.Makefile new file mode 100644 index 0000000..7e01edb --- /dev/null +++ b/makefiles/c_api_extensions/rust.Makefile @@ -0,0 +1,47 @@ +# Reusable makefile for the Rust extensions targeting the C extension API +# +# Inputs +# EXTENSION_NAME : name of the extension (lower case) +# EXTENSION_LIB_FILENAME : the library name that is produced by the build +# LOCAL_DUCKDB_RS_PATH : overrides the duckdb-rs path + +.PHONY: build_extension_library_debug build_extension_library_release + +############################################# +### Development config +############################################# + +# Allows overriding the duckdb-rs crates with a local version +CARGO_OVERRIDE_DUCKDB_RS_FLAG?= +ifneq ($(LOCAL_DUCKDB_RS_PATH),) + CARGO_OVERRIDE_DUCKDB_RS_FLAG=--config 'patch.crates-io.duckdb.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb"' --config 'patch.crates-io.libduckdb-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/libduckdb-sys"' --config 'patch.crates-io.duckdb-loadable-macros-sys.path="$(LOCAL_DUCKDB_RS_PATH)/crates/duckdb-loadable-macros-sys"' +endif + +IS_EXAMPLE= + +ifneq ($(DUCKDB_WASM_PLATFORM),) + TARGET=wasm32-unknown-emscripten + TARGET_INFO=--target $(TARGET) --example $(EXTENSION_NAME) + IS_EXAMPLE=examples +endif + +############################################# +### Rust Build targets +############################################# + +build_extension_library_debug: check_configure + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) $(TARGET_INFO) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/debug/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/debug/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/debug/$(EXTENSION_LIB_FILENAME)')" + +build_extension_library_release: check_configure + DUCKDB_EXTENSION_NAME=$(EXTENSION_NAME) DUCKDB_EXTENSION_MIN_DUCKDB_VERSION=$(MINIMUM_DUCKDB_VERSION) cargo build $(CARGO_OVERRIDE_DUCKDB_RS_FLAG) --release $(TARGET_INFO) + $(PYTHON_VENV_BIN) -c "from pathlib import Path;Path('./build/$(DUCKDB_WASM_PLATFORM)/release/extension/$(EXTENSION_NAME)').mkdir(parents=True, exist_ok=True)" + $(PYTHON_VENV_BIN) -c "import shutil;shutil.copyfile('target/$(TARGET)/release/$(IS_EXAMPLE)/$(EXTENSION_LIB_FILENAME)', 'build/$(DUCKDB_WASM_PLATFORM)/release/$(EXTENSION_LIB_FILENAME)')" + +############################################# +### Misc +############################################# + +clean_rust: + cargo clean diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 7e43936..47f32b4 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -1,4 +1,4 @@ -# Reusable makefile for building out-of-tree extension with the DuckDB extension template +# Reusable makefile for building out-of-tree extension with the DuckDB C++ based extension template # # Inputs # EXT_NAME : Upper case string describing the name of the out-of-tree extension @@ -6,8 +6,9 @@ # EXT_FLAGS : Extra CMake flags to pass to the build # EXT_RELEASE_FLAGS : Extra CMake flags to pass to the release build # EXT_DEBUG_FLAGS : Extra CMake flags to pass to the debug build +# SKIP_TESTS : Replaces all test targets with a NOP step -.PHONY: all clean format debug release pull update wasm_mvp wasm_eh wasm_threads +.PHONY: all clean clean-python format debug release pull update wasm_mvp wasm_eh wasm_threads test test_release test_debug test_reldebug test_release_internal test_debug_internal test_reldebug_internal set_duckdb_version set_duckdb_tag output_distribution_matrix all: release @@ -78,15 +79,35 @@ reldebug: # Main tests test: test_release -test_release: release - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" +TEST_RELEASE_TARGET=test_release_internal +TEST_DEBUG_TARGET=test_debug_internal +TEST_RELDEBUG_TARGET=test_reldebug_internal -test_debug: debug - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +# Disable testing outside docker: the unittester is currently dynamically linked by default +ifeq ($(LINUX_CI_IN_DOCKER),0) + SKIP_TESTS=1 +endif + +ifeq ($(SKIP_TESTS),1) + TEST_RELEASE_TARGET=tests_skipped + TEST_DEBUG_TARGET=tests_skipped + TEST_RELDEBUG_TARGET=tests_skipped +endif -test_reldebug: reldebug +test_release: $(TEST_RELEASE_TARGET) +test_debug: $(TEST_DEBUG_TARGET) +test_reldebug: $(TEST_RELDEBUG_TARGET) + +test_release_internal: + ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_debug_internal: + ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_reldebug_internal: ./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*" +tests_skipped: + @echo "Tests are skipped in this run..." + # WASM config VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS) @@ -138,3 +159,6 @@ set_duckdb_tag: output_distribution_matrix: cat duckdb/.github/config/distribution_matrix.json + +configure_ci: + @echo "configure_ci step is skipped for this extension build..." diff --git a/scripts/append_extension_metadata.py b/scripts/append_extension_metadata.py index 7ea59a0..d15262d 100644 --- a/scripts/append_extension_metadata.py +++ b/scripts/append_extension_metadata.py @@ -33,10 +33,15 @@ def main(): arg_parser.add_argument('-o', '--out-file', type=str, help='Explicit path for the output file', default='') - arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode', required=True) + # The platform + arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode') + arg_parser.add_argument('-pf', '--duckdb-platform-file', type=str, help='The file containing the DuckDB platform to encode') + arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type ' 'this encodes the duckdb version or the C API version', required=True) - arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True) + arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode') + arg_parser.add_argument('-evf', '--extension-version-file', type=str, help='The file containing the Extension version to encode') + arg_parser.add_argument('--abi-type', type=str, help='The ABI type to encode, set to C_STRUCT by default', default='C_STRUCT') args = arg_parser.parse_args() @@ -51,6 +56,39 @@ def main(): print(f" - Output file: {OUTPUT_FILE}") shutil.copyfile(args.library_file, OUTPUT_FILE_TMP) + # Handle the platform + PLATFORM = "" + if args.duckdb_platform: + PLATFORM = args.duckdb_platform + elif args.duckdb_platform_file: + try: + with open(args.duckdb_platform_file, 'r') as file: + PLATFORM = file.read().strip() + except Exception as e: + print(f"Failed to read platform from file {args.duckdb_platform_file}") + raise + if not PLATFORM: + raise Exception(f"Platform file passed to script is empty: {args.duckdb_platform_file}") + else: + raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either") + + EXTENSION_VERSION = "" + if args.extension_version: + EXTENSION_VERSION = args.extension_version + elif args.extension_version_file: + try: + with open(args.extension_version_file, 'r') as file: + EXTENSION_VERSION = file.read().strip() + except Exception as e: + print(f"Failed to read extension version from file {args.extension_version_file}") + raise + if not EXTENSION_VERSION: + raise Exception(f"Extension version file passed to script is empty: {args.extension_version_file}") + else: + raise Exception(f"Neither --extension-version nor --extension-version-file found, please specify the extension version using either") + + + # Then append the metadata to the tmp file print(f" - Metadata:") with open(OUTPUT_FILE_TMP, 'ab') as file: @@ -63,12 +101,12 @@ def main(): file.write(padded_byte_string("")) print(f" - FIELD5 (abi_type) = {args.abi_type}") file.write(padded_byte_string(args.abi_type)) - print(f" - FIELD4 (extension_version) = {args.extension_version}") - file.write(padded_byte_string(args.extension_version)) + print(f" - FIELD4 (extension_version) = {EXTENSION_VERSION}") + file.write(padded_byte_string(EXTENSION_VERSION)) print(f" - FIELD3 (duckdb_version) = {args.duckdb_version}") file.write(padded_byte_string(args.duckdb_version)) - print(f" - FIELD2 (duckdb_platform) = {args.duckdb_platform}") - file.write(padded_byte_string(args.duckdb_platform)) + print(f" - FIELD2 (duckdb_platform) = {PLATFORM}") + file.write(padded_byte_string(PLATFORM)) print(f" - FIELD1 (header signature) = 4 (special value to identify a duckdb extension)") file.write(padded_byte_string("4")) diff --git a/scripts/configure_helper.py b/scripts/configure_helper.py new file mode 100644 index 0000000..9685410 --- /dev/null +++ b/scripts/configure_helper.py @@ -0,0 +1,45 @@ +import subprocess +import argparse +from pathlib import Path +import os + +def main(): + arg_parser = argparse.ArgumentParser(description='Script to aid in running the configure step of the extension build process') + + + arg_parser.add_argument('-o', '--output-directory', type=str, help='Specify the output directory', default='configure') + + arg_parser.add_argument('-ev', '--extension-version', help='Write the autodetected extension version', action='store_true') + arg_parser.add_argument('-p', '--duckdb-platform', help='Write the auto-detected duckdb platform', action='store_true') + + args = arg_parser.parse_args() + + OUTPUT_DIR = args.output_directory + + Path(OUTPUT_DIR).mkdir(parents=True, exist_ok=True) + + # Write version + if args.extension_version: + git_tag = subprocess.getoutput("git tag --points-at HEAD") + if git_tag: + EXTENSION_VERSION = git_tag + else: + EXTENSION_VERSION = subprocess.getoutput("git --no-pager log -1 --format=%h") + + version_file = Path(os.path.join(OUTPUT_DIR, "extension_version.txt")) + with open(version_file, 'w') as f: + print(f"Writing version {EXTENSION_VERSION} to {version_file}") + f.write(EXTENSION_VERSION) + + # Write duck + if args.duckdb_platform: + import duckdb + platform_file = Path(os.path.join(OUTPUT_DIR, "platform.txt")) + duckdb_platform = duckdb.execute('pragma platform').fetchone()[0] + with open(platform_file, 'w') as f: + print(f"Writing platform {duckdb_platform} to {platform_file}") + f.write(duckdb_platform) + + +if __name__ == '__main__': + main() \ No newline at end of file