diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 0f86112..c9e525a 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -23,7 +23,6 @@ jobs: override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} extra_toolchains: 'parser_tools;fortran;omp;go;python3' - custom_toolchain_script: true delta-extension-main: name: Rust builds (using Delta extension) diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 3bd4cb0..81a8d03 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -49,7 +49,7 @@ on: matrix_parse_script: required: false type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" + default: "./extension-ci-tools/scripts/modify_distribution_matrix.py" # Enable building the DuckDB Shell build_duckdb_shell: required: false @@ -76,11 +76,6 @@ on: required: false type: string default: "" - # If set, the setup will look for a script in `./scripts/custom-toolchain-script.sh` to run - custom_toolchain_script: - required: false - type: boolean - default: false rust_logs: required: false type: boolean @@ -283,6 +278,8 @@ jobs: echo "OPENSSL_DIR=/duckdb_build_dir/build/release/vcpkg_installed/${{ matrix.vcpkg_triplet }}" >> docker_env.txt echo "OPENSSL_USE_STATIC_LIBS=true" >> docker_env.txt echo "DUCKDB_PLATFORM=${{ matrix.duckdb_arch }}" >> docker_env.txt + echo "DUCKDB_GIT_VERSION=${{ inputs.duckdb_version }}" >> docker_env.txt + echo "LINUX_CI_IN_DOCKER=1" >> docker_env.txt echo "TOOLCHAIN_FLAGS=${{ matrix.duckdb_arch == 'linux_arm64' && '-DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ -DCMAKE_Fortran_COMPILER=aarch64-linux-gnu-gfortran' || '' }}" >> docker_env.txt - name: Generate timestamp for Ccache entry @@ -304,14 +301,35 @@ jobs: restore-keys: | ccache-extension-distribution-${{ matrix.duckdb_arch }}- - - name: Build extension + - name: Run configure (outside Docker) + shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 + run: | + make configure_ci + + - name: Run configure (inside Docker) + shell: bash + run: | + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make configure_ci + + - name: Build extension (inside Docker) run: | docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make release - - name: Test extension + - name: Test extension (inside docker) if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} run: | - docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test + docker run --env-file=docker_env.txt -v `pwd`:/duckdb_build_dir -v `pwd`/ccache_dir:/ccache_dir duckdb/${{ matrix.duckdb_arch }} make test_release + + - name: Test extension (outside docker) + if: ${{ matrix.duckdb_arch != 'linux_arm64' && inputs.skip_tests == false }} + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} + LINUX_CI_IN_DOCKER: 0 + run: | + make test_release - uses: actions/upload-artifact@v3 with: @@ -439,16 +457,15 @@ jobs: echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build extension shell: bash - env: - DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} run: | make release @@ -456,7 +473,7 @@ jobs: if: ${{ matrix.osx_build_arch == 'arm64' && inputs.skip_tests == false }} shell: bash run: | - make test + make test_release - uses: actions/upload-artifact@v3 with: @@ -583,21 +600,23 @@ jobs: vcpkgGitCommitId: ${{ inputs.vcpkg_commit }} vcpkgGitURL: ${{ inputs.vcpkg_url }} - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure shell: bash + env: + DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} + DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build extension - if: ${{ inputs.skip_tests == true }} env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} DUCKDB_PLATFORM_RTOOLS: ${{ matrix.duckdb_arch == 'windows_amd64_rtools' && 1 || 0 }} run: | make release - - name: Build & test extension + - name: Test extension if: ${{ inputs.skip_tests == false }} env: DUCKDB_PLATFORM: ${{ matrix.duckdb_arch }} @@ -701,11 +720,12 @@ jobs: with: key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - name: Run custom toolchain script - if: ${{ inputs.custom_toolchain_script }} + - name: Run configure shell: bash + env: + DUCKDB_GIT_VERSION: ${{ inputs.duckdb_version }} run: | - bash scripts/setup-custom-toolchain.sh + make configure_ci - name: Build Wasm module run: | diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 7e43936..c248f7c 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -6,8 +6,9 @@ # EXT_FLAGS : Extra CMake flags to pass to the build # EXT_RELEASE_FLAGS : Extra CMake flags to pass to the release build # EXT_DEBUG_FLAGS : Extra CMake flags to pass to the debug build +# SKIP_TESTS : Replaces all test targets with a NOP step -.PHONY: all clean format debug release pull update wasm_mvp wasm_eh wasm_threads +.PHONY: all clean clean-python format debug release pull update wasm_mvp wasm_eh wasm_threads test test_release test_debug test_reldebug test_release_internal test_debug_internal test_reldebug_internal set_duckdb_version set_duckdb_tag output_distribution_matrix all: release @@ -78,15 +79,35 @@ reldebug: # Main tests test: test_release -test_release: release - ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" +TEST_RELEASE_TARGET=test_release_internal +TEST_DEBUG_TARGET=test_debug_internal +TEST_RELDEBUG_TARGET=test_reldebug_internal -test_debug: debug - ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +# Disable testing outside docker: the unittester is currently dynamically linked by default +ifeq ($(LINUX_CI_IN_DOCKER),0) + SKIP_TESTS=1 +endif + +ifeq ($(SKIP_TESTS),1) + TEST_RELEASE_TARGET=tests_skipped + TEST_DEBUG_TARGET=tests_skipped + TEST_RELDEBUG_TARGET=tests_skipped +endif -test_reldebug: reldebug +test_release: $(TEST_RELEASE_TARGET) +test_debug: $(TEST_DEBUG_TARGET) +test_reldebug: $(TEST_RELDEBUG_TARGET) + +test_release_internal: + ./build/release/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_debug_internal: + ./build/debug/$(TEST_PATH) "$(PROJ_DIR)test/*" +test_reldebug_internal: ./build/reldebug/$(TEST_PATH) "$(PROJ_DIR)test/*" +tests_skipped: + @echo "Tests are skipped in this run..." + # WASM config VCPKG_EMSDK_FLAGS=-DVCPKG_CHAINLOAD_TOOLCHAIN_FILE=$(EMSDK)/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake WASM_COMPILE_TIME_COMMON_FLAGS=-DWASM_LOADABLE_EXTENSIONS=1 -DBUILD_EXTENSIONS_ONLY=1 $(TOOLCHAIN_FLAGS) $(VCPKG_EMSDK_FLAGS) @@ -138,3 +159,6 @@ set_duckdb_tag: output_distribution_matrix: cat duckdb/.github/config/distribution_matrix.json + +configure_ci: + @echo "configure_ci step is skipped for this extension build..." diff --git a/scripts/append_extension_metadata.py b/scripts/append_extension_metadata.py index 7ea59a0..277d5d3 100644 --- a/scripts/append_extension_metadata.py +++ b/scripts/append_extension_metadata.py @@ -33,7 +33,10 @@ def main(): arg_parser.add_argument('-o', '--out-file', type=str, help='Explicit path for the output file', default='') - arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode', required=True) + # The platform + arg_parser.add_argument('-p', '--duckdb-platform', type=str, help='The DuckDB platform to encode') + arg_parser.add_argument('-pf', '--duckdb-platform-file', type=str, help='The file containing the DuckDB platform to encode') + arg_parser.add_argument('-dv', '--duckdb-version', type=str, help='The DuckDB version to encode, depending on the ABI type ' 'this encodes the duckdb version or the C API version', required=True) arg_parser.add_argument('-ev', '--extension-version', type=str, help='The Extension version to encode', required=True) @@ -51,6 +54,22 @@ def main(): print(f" - Output file: {OUTPUT_FILE}") shutil.copyfile(args.library_file, OUTPUT_FILE_TMP) + # Handle the platform + PLATFORM = "" + if args.duckdb_platform: + PLATFORM = args.duckdb_platform + elif args.duckdb_platform_file: + try: + with open(args.duckdb_platform_file, 'r') as file: + PLATFORM = file.read().strip() + except Exception as e: + print(f"Failed to read platform from file {args.duckdb_platform_file}") + raise + if not PLATFORM: + raise Exception(f"Platform file passed to script is empty: {args.duckdb_platform_file}") + else: + raise Exception(f"Neither --duckdb-platform nor --duckdb-platform-file found, please specify the platform using either") + # Then append the metadata to the tmp file print(f" - Metadata:") with open(OUTPUT_FILE_TMP, 'ab') as file: @@ -67,8 +86,8 @@ def main(): file.write(padded_byte_string(args.extension_version)) print(f" - FIELD3 (duckdb_version) = {args.duckdb_version}") file.write(padded_byte_string(args.duckdb_version)) - print(f" - FIELD2 (duckdb_platform) = {args.duckdb_platform}") - file.write(padded_byte_string(args.duckdb_platform)) + print(f" - FIELD2 (duckdb_platform) = {PLATFORM}") + file.write(padded_byte_string(PLATFORM)) print(f" - FIELD1 (header signature) = 4 (special value to identify a duckdb extension)") file.write(padded_byte_string("4"))