From 80fc2284996707945821f8d0a50fe2911b3f5305 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 9 Dec 2024 10:59:37 +0100 Subject: [PATCH 1/3] remove depoy script --- .../workflows/MainDistributionPipeline.yml | 12 +- .github/workflows/_extension_deploy.yml | 121 ------------------ docs/README.md | 60 ++++----- 3 files changed, 31 insertions(+), 162 deletions(-) delete mode 100644 .github/workflows/_extension_deploy.yml diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 0b96e22..62ab995 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -26,14 +26,4 @@ jobs: with: duckdb_version: v1.1.3 ci_tools_version: v1.1.3 - extension_name: quack - - duckdb-stable-deploy: - name: Deploy extension binaries - needs: duckdb-stable-build - uses: ./.github/workflows/_extension_deploy.yml - secrets: inherit - with: - duckdb_version: v1.1.3 - extension_name: quack - deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }} + extension_name: quack \ No newline at end of file diff --git a/.github/workflows/_extension_deploy.yml b/.github/workflows/_extension_deploy.yml deleted file mode 100644 index 604a24a..0000000 --- a/.github/workflows/_extension_deploy.yml +++ /dev/null @@ -1,121 +0,0 @@ -# -# Reusable workflow that deploys the artifacts produced by github.com/duckdb/duckdb/.github/workflows/_extension_distribution.yml -# -# note: this workflow needs to be located in the extension repository, as it requires secrets to be passed to the -# deploy script. However, it should generally not be necessary to modify this workflow in your extension repository, as -# this workflow can be configured to use a custom deploy script. - - -name: Extension Deployment -on: - workflow_call: - inputs: - # The name of the extension - extension_name: - required: true - type: string - # DuckDB version to build against - duckdb_version: - required: true - type: string - # ';' separated list of architectures to exclude, for example: 'linux_amd64;osx_arm64' - exclude_archs: - required: false - type: string - default: "" - # Whether to upload this deployment as the latest. This may overwrite a previous deployment. - deploy_latest: - required: false - type: boolean - default: false - # Whether to upload this deployment under a versioned path. These will not be deleted automatically - deploy_versioned: - required: false - type: boolean - default: false - # Postfix added to artifact names. Can be used to guarantee unique names when this workflow is called multiple times - artifact_postfix: - required: false - type: string - default: "" - # Override the default deploy script with a custom script - deploy_script: - required: false - type: string - default: "./scripts/extension-upload.sh" - # Override the default matrix parse script with a custom script - matrix_parse_script: - required: false - type: string - default: "./duckdb/scripts/modify_distribution_matrix.py" - -jobs: - generate_matrix: - name: Generate matrix - runs-on: ubuntu-latest - outputs: - deploy_matrix: ${{ steps.parse-matrices.outputs.deploy_matrix }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - id: parse-matrices - run: | - python3 ${{ inputs.matrix_parse_script }} --input ./duckdb/.github/config/distribution_matrix.json --deploy_matrix --output deploy_matrix.json --exclude "${{ inputs.exclude_archs }}" --pretty - deploy_matrix="`cat deploy_matrix.json`" - echo deploy_matrix=$deploy_matrix >> $GITHUB_OUTPUT - echo `cat $GITHUB_OUTPUT` - - deploy: - name: Deploy - runs-on: ubuntu-latest - needs: generate_matrix - if: ${{ needs.generate_matrix.outputs.deploy_matrix != '{}' && needs.generate_matrix.outputs.deploy_matrix != '' }} - strategy: - matrix: ${{fromJson(needs.generate_matrix.outputs.deploy_matrix)}} - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: 'true' - - - name: Checkout DuckDB to version - run: | - cd duckdb - git checkout ${{ inputs.duckdb_version }} - - - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.extension_name }}-${{ inputs.duckdb_version }}-extension-${{matrix.duckdb_arch}}${{inputs.artifact_postfix}}${{startsWith(matrix.duckdb, 'wasm') && '.wasm' || ''}} - path: | - /tmp/extension - - - name: Deploy - shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.S3_REGION }} - BUCKET_NAME: ${{ secrets.S3_BUCKET }} - DUCKDB_EXTENSION_SIGNING_PK: ${{ secrets.S3_DUCKDB_ORG_EXTENSION_SIGNING_PK }} - run: | - pwd - python3 -m pip install pip awscli - git config --global --add safe.directory '*' - cd duckdb - git fetch --tags - export DUCKDB_VERSION=`git tag --points-at HEAD` - export DUCKDB_VERSION=${DUCKDB_VERSION:=`git log -1 --format=%h`} - cd .. - git fetch --tags - export EXT_VERSION=`git tag --points-at HEAD` - export EXT_VERSION=${EXT_VERSION:=`git log -1 --format=%h`} - ${{ inputs.deploy_script }} ${{ inputs.extension_name }} $EXT_VERSION $DUCKDB_VERSION ${{ matrix.duckdb_arch }} $BUCKET_NAME ${{inputs.deploy_latest || 'true' && 'false'}} ${{inputs.deploy_versioned || 'true' && 'false'}} diff --git a/docs/README.md b/docs/README.md index 7825350..18252e9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -77,42 +77,42 @@ For inspiration/examples on how to extend DuckDB in a more meaningful way, check the [in-tree extensions](https://github.com/duckdb/duckdb/tree/main/extension), and the [out-of-tree extensions](https://github.com/duckdblabs). ## Distributing your extension -Easy distribution of extensions built with this template is facilitated using a similar process used by DuckDB itself. -Binaries are generated for various versions/platforms allowing duckdb to automatically install the correct binary. - -This step requires that you pass the following 4 parameters to your GitHub repo as action secrets: - -| secret name | description | -| ------------- | ----------------------------------- | -| S3_REGION | s3 region holding your bucket | -| S3_BUCKET | the name of the bucket to deploy to | -| S3_DEPLOY_ID | the S3 key id | -| S3_DEPLOY_KEY | the S3 key secret | - -After setting these variables, all pushes to main will trigger a new (dev) release. Note that your AWS token should -have full permissions to the bucket, and you will need to have ACLs enabled. - -### Installing the deployed binaries -To install your extension binaries from S3, you will need to do two things. Firstly, DuckDB should be launched with the -`allow_unsigned_extensions` option set to true. How to set this will depend on the client you're using. Some examples: +To distribute your extension binaries, there are a few options. + +### Community extensions +The recommended way of distributing extensions is through the [community extensions repository](https://github.com/duckdb/community-extensions). +This repository is designed specifically for extensions that are built using this extension template, meaning that as long as your extension can be +built using the default CI in this template, submitting it to the community extensions is a very simple process. The process works similarly to popular +package managers like homebrew and vcpkg, where a PR containing a descriptor file is submitted to the package manager repository. After the CI in the +community extensions repository completes, the extension can be installed and loaded in DuckDB with: +```SQL +INSTALL FROM community; +LOAD +``` +For more information, see the [community extensions documentation](https://duckdb.org/community_extensions/documentation). -CLI: +### Downloading artifacts from GitHub +The default CI in this template will automatically upload the binaries for every push to the main branch as GitHub Actions artifacts. These +can be downloaded manually and then loaded directly using: +```SQL +LOAD '/path/to/downloaded/extension.duckdb_extension'; +``` +Note that this will require starting DuckDB with the +`allow_unsigned_extensions` option set to true. How to set this will depend on the client you're using. For the CLI it is done like: ```shell duckdb -unsigned ``` -Secondly, you will need to set the repository endpoint in DuckDB to the HTTP url of your bucket + version of the extension -you want to install. To do this run the following SQL query in DuckDB: -```sql -SET custom_extension_repository='bucket.s3.eu-west-1.amazonaws.com//latest'; -``` -Note that the `/latest` path will allow you to install the latest extension version available for your current version of -DuckDB. To specify a specific version, you can pass the version instead. +### Uploading to a custom repository +If for some reason distributing through community extensions is not an option, extensions can also be uploaded to a custom extension repository. +This will give some more control over where and how the extensions are distributed, but comes with the downside of requiring the `allow_unsigned_extensions` +option to be set. For examples of how to configure a manual GitHub Actions deploy pipeline, check out the extension deploy script in https://github.com/duckdb/extension-ci-tools. +Some examples of extensions that use this CI/CD workflow check out [spatial](https://github.com/duckdblabs/duckdb_spatial) or [aws](https://github.com/duckdb/duckdb_aws). -After running these steps, you can install and load your extension using the regular INSTALL/LOAD commands in DuckDB: -```sql -INSTALL -LOAD +Extensions in custom repositories can be installed and loaded using: +```SQL +INSTALL FROM 'http://my-custom-repo' +LOAD ``` ### Versioning of your extension From 8ba7fd9c4ddef240171ddf68e1d3612d86034c20 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Mon, 9 Dec 2024 12:29:17 +0100 Subject: [PATCH 2/3] checkout v4 --- .github/workflows/ExtensionTemplate.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ExtensionTemplate.yml b/.github/workflows/ExtensionTemplate.yml index 0b2c52c..10112be 100644 --- a/.github/workflows/ExtensionTemplate.yml +++ b/.github/workflows/ExtensionTemplate.yml @@ -45,7 +45,7 @@ jobs: make prefix=/usr install git --version - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: 'true' @@ -93,7 +93,7 @@ jobs: shell: bash steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: 'true' @@ -144,7 +144,7 @@ jobs: shell: bash steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 submodules: 'true' From 78921da6e02f1edca7e7499f370200513ec842c0 Mon Sep 17 00:00:00 2001 From: Sam Ansmink Date: Fri, 13 Dec 2024 10:44:02 +0100 Subject: [PATCH 3/3] switch to ubuntu latest --- .github/workflows/ExtensionTemplate.yml | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ExtensionTemplate.yml b/.github/workflows/ExtensionTemplate.yml index 10112be..9712532 100644 --- a/.github/workflows/ExtensionTemplate.yml +++ b/.github/workflows/ExtensionTemplate.yml @@ -13,7 +13,6 @@ jobs: name: Linux if: ${{ vars.RUN_RENAME_TEST == 'true' || github.repository == 'duckdb/extension-template' }} runs-on: ubuntu-latest - container: ubuntu:18.04 strategy: matrix: # Add commits/tags to build against other DuckDB versions @@ -28,22 +27,9 @@ jobs: shell: bash steps: - - name: Install required ubuntu packages - run: | - apt-get update -y -qq - apt-get install -y -qq software-properties-common - add-apt-repository ppa:git-core/ppa - apt-get update -y -qq - apt-get install -y -qq ninja-build make gcc-multilib g++-multilib libssl-dev wget openjdk-8-jdk zip maven unixodbc-dev libc6-dev-i386 lib32readline6-dev libssl-dev libcurl4-gnutls-dev libexpat1-dev gettext unzip build-essential checkinstall libffi-dev curl libz-dev openssh-client - - - name: Install Git 2.18.5 - run: | - wget https://github.com/git/git/archive/refs/tags/v2.18.5.tar.gz - tar xvf v2.18.5.tar.gz - cd git-2.18.5 - make - make prefix=/usr install - git --version + - name: Install Ninja + shell: bash + run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build - uses: actions/checkout@v4 with: @@ -56,8 +42,6 @@ jobs: cd duckdb git checkout ${{ matrix.duckdb_version }} - - uses: ./duckdb/.github/actions/ubuntu_18_setup - - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: