Ckan 2.10 upgrade

.github/workflows/wf1.yaml

@@ -3,7 +3,7 @@ run-name: ${{ github.actor }} created a new release
 on: 
   push:
     tags:
-      - v*.*.*-*
+      - twdh-*.*.*
 jobs:
   test-image:
     runs-on: ubuntu-22.04
@@ -48,7 +48,7 @@ jobs:
       - run: |
           cd docker && docker compose --file docker-compose_local.yml up -d --wait
       - name: docker_log
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build_log
           path: build.log
@@ -61,4 +61,4 @@ jobs:
         env:
           SOURCE_NAME: ${{ steps.name_branch_tags.outputs.SOURCE_NAME }}
           SOURCE_BRANCH: ${{ steps.name_branch_tags.outputs.SOURCE_BRANCH }}
-          SOURCE_TAG: ${{ steps.name_branch_tags.outputs.SOURCE_TAG }}
+          SOURCE_TAG: ${{ steps.name_branch_tags.outputs.SOURCE_TAG }}

docker/.env-ckan.env

@@ -22,11 +22,11 @@ DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
 #DB CONNECTION STRINGS
-CKAN_DATASTORE_WRITE_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
+CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
 
 # General Settings
-CKAN_VERSION=2.9.9
+CKAN_VERSION=2.10.5
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://0.0.0.0:5000
 CKAN_PORT=5000
@@ -36,6 +36,7 @@ CKAN___DEBUG=false
 
 # Auth Settings
 CKAN__AUTH__CREATE_USER_VIA_WEB=false
+CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=false
 
 # Tracking Settings
 CKAN__TRACKING_ENABLED=true
@@ -60,6 +61,7 @@ [email protected]
 # CKAN_SMTP_MAIL_FROM=ckan@localhost
 
 # Datapusher configuration
+CKAN__DATAPUSHER__API_TOKEN=${CKAN_DATAPUSHER_API_TOKEN}
 CKAN__DATAPUSHER__URL=http://datapusher:8800
 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://0.0.0.0:5000/
 DATAPUSHER_VERSION=0.0.17
@@ -94,6 +96,7 @@ CKAN___BEAKER__SESSION__TYPE=ext:redis
 CKAN___BEAKER__SESSION__URL=redis://redis:6379/8
 CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true
 CKAN___BEAKER__SESSION__COOKIE_DOMAIN=0.0.0.0
+
 CKANEXT__SECURITY__DOMAIN=0.0.0.0
 CKANEXT__SECURITY__REDIS__HOST=redis
 CKANEXT__SECURITY__REDIS__PORT=6379
@@ -103,3 +106,4 @@ CKANEXT__SECURITY__LOGIN_MAX_COUNT=10
 CKANEXT__SECURITY__BRUTE_FORCE_KEY=user_name
 CKANEXT__SECURITY__DISABLE_PASSWORD_RESET_OVERRIDE=true
 CKANEXT__SECURITY__MFA_HELP_LINK=http://0.0.0.0:5000/pages/mfa
+

docker/ckan/Dockerfile

@@ -6,7 +6,7 @@ FROM ubuntu:focal-20210827 as ckanbuild
 # Set CKAN version to build
 
 ENV GIT_URL=https://github.com/TNRIS/ckan.git
-ENV GIT_BRANCH=twdh-patches-2.9.10
+ENV GIT_BRANCH=twdh-patches-2.10.5
 
 # Set timezone
 ENV TZ=UTC
@@ -119,6 +119,7 @@ ENV CKAN__DEFAULT_VIEWS image_view \
 ENV CKAN__PLUGINS \
     stats \
     twdh_theme \
+    activity \
     security \
     report \
     check_link \
@@ -127,8 +128,6 @@ ENV CKAN__PLUGINS \
     image_view \
     video_view \
     datastore \
-    harvest \
-    ckan_harvester \
     spatial_metadata \
     spatial_query \
     twdh_schema \
@@ -143,7 +142,7 @@ ENV CKAN__PLUGINS \
     wmts_view \
     shp_view \
     datatablesview_plus \
-    datapusher \
+    datapusher_plus \
     openapi_viewer \
     s3filestore \
     googleanalytics \
@@ -191,6 +190,9 @@ RUN apt-get update && \
     libfreetype6 \
     libfreetype6-dev \
     zlib1g-dev \
+    supervisor \
+    file \
+    uchardet \
     apache2-utils && \
     rm -rf /var/lib/apt/lists/* && \
     # Create SRC_DIR
@@ -216,7 +218,7 @@ COPY ./plugins ${SRC_DIR}/plugins
 RUN ls ${SRC_DIR}/plugins
 
 # Additional install steps for build stages artifacts
-RUN pip install uWSGI==2.0.20 gevent==21.12.0
+RUN pip install uWSGI==2.0.20 gevent==22.10.2
 # Create a local user and group to run the app
 RUN groupadd -g 92 ckan && \
     useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan
@@ -263,7 +265,26 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     ckan config-tool ${APP_DIR}/production.ini "googleanalytics.id = G-SJK5R9G5F5" && \
     ckan config-tool ${APP_DIR}/production.ini "googleanalytics.download_handler = ckanext.s3filestore.views:resource_download" && \
     # Change ownership to app user
-    chown -R ckan:ckan /srv/app
+    chown -R ckan:ckan /srv/app && \
+    # Create log directory for `ckan jobs worker` process
+    mkdir -v /var/log/ckan
+
+# Setup supervisor
+COPY ./supervisord.conf /etc/supervisor/
+COPY ./supervisor-ckan-worker.conf /etc/supervisor/conf.d/
+
+# install Rust, then qsvdp
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    # source the cargo env to setup the $PATH for rust
+    . $HOME/.cargo/env && \
+    # install qsvdp from source
+    git clone https://github.com/dathere/qsv.git && \
+    cd qsv && \
+    export CARGO_BUILD_RUSTFLAGS='-C target-cpu=native' && \
+    cargo build --release --locked --bin qsvdp -F datapusher_plus,luau,polars && \
+    # mv the qsvdp binary so that ckan user can run it
+    mv target/release/qsvdp /usr/local/bin/ && \
+    cargo clean
 
 # Remove wheels
 RUN rm -rf /srv/app/wheels
@@ -288,6 +309,6 @@ EXPOSE 5000
 
 HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
-USER ckan
+#USER ckan
 
 CMD ["/srv/app/start_ckan.sh"]

docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
+
+UPGRADE=0
+
+if [ "$UPGRADE" -ne 1 ]; then
+    echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"
+else 
+  ckan -c /srv/app/production.ini db upgrade 
+  ckan -c /srv/app/production.ini db pending-migrations --apply 
+fi
+

docker/ckan/merge.production.ini

@@ -33,4 +33,4 @@ ckanext.security.disable_password_reset_override = true
 
 # Provide a help page to allow 2fa users to contact support or get more information
 # Shows up as 'Need help?' on the 2fa entry form beside the submit button. Does not display a link if none provided
-ckanext.security.mfa_help_link = https://data.govt.nz/catalogue-guide/releasing-data-on-data-govt-nz/how-do-i-set-up-two-factor-authentication/
+ckanext.security.mfa_help_link = https://data.govt.nz/catalogue-guide/releasing-data-on-data-govt-nz/how-do-i-set-up-two-factor-authentication/