From f5230c1521a8de9f7b23e3fd2874485488c8442b Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 4 Mar 2024 19:30:27 +0000
Subject: [PATCH 01/69] Changed dataset.yaml to dataset-druf.yaml for testing

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index bb75ad8..0d8950b 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -252,7 +252,7 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     # upgrade ckan_max_resource_size
     ckan config-tool ${APP_DIR}/production.ini "ckan.max_resource_size = ${CKAN__MAX_RESOURCE_SIZE}" && \
     # add scheming config
-    ckan config-tool ${APP_DIR}/production.ini "scheming.dataset_schemas = ckanext.twdh_schema:schemas/dataset.yaml ckanext.twdh_schema:schemas/application.yaml" && \
+    ckan config-tool ${APP_DIR}/production.ini "scheming.dataset_schemas = ckanext.twdh_schema:schemas/dataset-druf.yaml ckanext.twdh_schema:schemas/application.yaml" && \
     ckan config-tool ${APP_DIR}/production.ini "scheming.organization_schemas = ckanext.twdh_schema:schemas/organization.yaml" && \
     ckan config-tool ${APP_DIR}/production.ini "scheming.presets = ckanext.scheming:presets.json ckanext.composite:presets.json ckanext.twdh_schema:schemas/presets.yaml" && \
     # add ckan.spatial.search_backend to enable solr spatial search

From bab4111c2dc3e5a3db56bf480944fd3771e0948b Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 4 Mar 2024 19:34:26 +0000
Subject: [PATCH 02/69] Updated tags to point to bb-dev repos for work towards
 0.5.3

---
 docker/ckan/plugins/ckan_plugins.json | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 57d6317..ff55d58 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -3,8 +3,8 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-twdh_theme",
-      "tag": "TWDHv0.5.2.rc4",
-      "branch": "",
+      "tag": "",
+      "branch": "bb-dev",
       "cmd": [
         "pip install -e .",
         "pip install -r requirements.txt",
@@ -14,22 +14,22 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-twdh_schema",
-      "tag": "TWDHv0.5.1.rc1",
-      "branch": "",
+      "tag": "",
+      "branch": "bb-dev",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
       "repo": "ckanext-twdh_gazetteer",
-      "tag": "TWDHv0.4.1.rc3",
+      "tag": "bb-dev",
       "branch": "",
       "cmd": ["pip install -e ."]
     },
     {
       "owner": "TNRIS",
       "repo": "ckanext-spatial",
-      "tag": "TWDHv0.5.0.rc1",
-      "branch": "",
+      "tag": "",
+      "branch": "bb-dev",
       "cmd": [
         "pip install -e .",
         "pip install -r requirements.txt",
@@ -88,8 +88,8 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-datatablesview_plus",
-      "tag": "TWDHv0.5.2.rc2",
-      "branch": "",
+      "tag": "",
+      "branch": "bb-dev",
       "cmd": [
         "python setup.py install",
         "pip install botocore==1.27.29",

From 794c8e0f83d0820dad341e5d35762c41142079ee Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 4 Nov 2024 20:13:29 +0000
Subject: [PATCH 03/69] 2.10 upgrade prep

---
 docker/ckan/Dockerfile                |   2 +-
 docker/ckan/plugins/ckan_plugins.json | 146 +++++++++++++-------------
 2 files changed, 74 insertions(+), 74 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 0d8950b..ad5c4cf 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -6,7 +6,7 @@ FROM ubuntu:focal-20210827 as ckanbuild
 # Set CKAN version to build
 
 ENV GIT_URL=https://github.com/TNRIS/ckan.git
-ENV GIT_BRANCH=twdh-patches-2.9.10
+ENV GIT_BRANCH=twdh-patches-2.10.5
 
 # Set timezone
 ENV TZ=UTC
diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 664e9fe..4551379 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -2,40 +2,50 @@
   "plugins": [
     {
       "owner": "TNRIS",
-      "repo": "ckanext-twdh_theme",
-      "tag": "TWDHv0.5.3.rc1",
+      "repo": "ckanext-agsview",
+      "tag": "TWDHv0.2.9a",
       "branch": "",
-      "cmd": [
-        "pip install -e .",
-        "pip install -r requirements.txt",
-        "pip install Pillow"
-      ]
+      "cmd": ["pip install -r requirements.txt", "python setup.py develop"]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-twdh_schema",
-      "tag": "",
-      "branch": "bb-dev",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
+      "repo": "ckanext-check-link",
+      "tag": "TWDHv0.5.0.rc1",
+      "branch": "ckan-2.10-upgrade",
+      "cmd": ["python setup.py install", "pip install -r dev-requirements.txt"]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-twdh_gazetteer",
-      "tag": "bb-dev",
+      "repo": "ckanext-composite",
+      "tag": "TWDHv0.2.6",
       "branch": "",
-      "cmd": ["pip install -e ."]
+      "cmd": ["python setup.py install", "pip install -r dev-requirements.txt"]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-spatial",
+      "repo": "ckanext-contact",
       "tag": "",
-      "branch": "bb-dev",
+      "branch": "ckan-2.10-upgrade",
+      "cmd": ["pip install -r requirements.txt", "python setup.py develop"]
+    },
+    {
+      "owner": "TNRIS",
+      "repo": "ckanext-datatablesview_plus",
+      "tag": "",
+      "branch": "ckan-2.10-upgrade",
       "cmd": [
-        "pip install -e .",
-        "pip install -r requirements.txt",
-        "pip install pika"
+        "python setup.py install",
+        "pip install botocore==1.27.29",
+        "pip install -r dev-requirements.txt"
       ]
     },
+    {
+      "owner": "okfn",
+      "repo": "ckanext-envvars",
+      "tag": "",
+      "branch": "master",
+      "cmd": ["python setup.py install"]
+    },
     {
       "owner": "TNRIS",
       "repo": "ckanext-geoview",
@@ -44,11 +54,18 @@
       "cmd": ["python setup.py install"]
     },
     {
-      "owner": "ckan",
-      "repo": "ckanext-scheming",
+      "owner": "TNRIS",
+      "repo": "ckanext-googleanalytics",
+      "tag": "TWDHv0.5.0.rc1",
+      "branch": "",
+      "cmd": ["python setup.py install", "pip install -r requirements.txt"]
+    },
+    {
+      "owner": "TNRIS",
+      "repo": "ckanext-harvest",
       "tag": "",
       "branch": "master",
-      "cmd": ["pip install -e ."]
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
@@ -59,63 +76,38 @@
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-pages",
-      "tag": "TWDHv0.5.0.rc1",
+      "repo": "ckanext-openapi_viewer",
+      "tag": "TWDHv0.2.6",
       "branch": "",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-contact",
+      "repo": "ckanext-pages",
       "tag": "TWDHv0.5.0.rc1",
       "branch": "",
-      "cmd": ["pip install -r requirements.txt", "python setup.py develop"]
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-composite",
+      "repo": "ckanext-report",
       "tag": "TWDHv0.2.6",
       "branch": "master",
-      "cmd": ["python setup.py install", "pip install -r dev-requirements.txt"]
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
       "repo": "ckanext-s3filestore",
-      "tag": "",
-      "branch": "main",
-      "cmd": ["python setup.py install", "pip install -r requirements.txt"]
-    },
-    {
-      "owner": "TNRIS",
-      "repo": "ckanext-datatablesview_plus",
-      "tag": "",
-      "branch": "bb-dev",
-      "cmd": [
-        "python setup.py install",
-        "pip install botocore==1.27.29",
-        "pip install -r dev-requirements.txt"
-      ]
-    },
-    {
-      "owner": "TNRIS",
-      "repo": "ckanext-googleanalytics",
-      "tag": "TWDHv0.5.0.rc1",
+      "tag": "TWDHv0.5.3",
       "branch": "",
       "cmd": ["python setup.py install", "pip install -r requirements.txt"]
     },
     {
-      "owner": "TNRIS",
-      "repo": "ckanext-harvest",
-      "tag": "",
-      "branch": "master",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
-    },
-    {
-      "owner": "okfn",
-      "repo": "ckanext-envvars",
+      "owner": "ckan",
+      "repo": "ckanext-scheming",
       "tag": "",
       "branch": "master",
-      "cmd": ["python setup.py install"]
+      "cmd": ["pip install -e ."]
     },
     {
       "owner": "TNRIS",
@@ -126,31 +118,39 @@
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-check-link",
-      "tag": "TWDHv0.5.0.rc1",
-      "branch": "",
-      "cmd": ["python setup.py install", "pip install -r dev-requirements.txt"]
+      "repo": "ckanext-spatial",
+      "tag": "",
+      "branch": "bb-dev",
+      "cmd": [
+        "pip install -e .",
+        "pip install -r requirements.txt",
+        "pip install pika"
+      ]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-report",
-      "tag": "TWDHv0.2.6",
-      "branch": "master",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
+      "repo": "ckanext-twdh_gazetteer",
+      "tag": "bb-dev",
+      "branch": "",
+      "cmd": ["pip install -e ."]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-agsview",
-      "tag": "TWDHv0.2.9a",
-      "branch": "",
-      "cmd": ["pip install -r requirements.txt", "python setup.py develop"]
+      "repo": "ckanext-twdh_schema",
+      "tag": "",
+      "branch": "ckan-2.10-upgrade",
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-openapi_viewer",
-      "tag": "TWDHv0.2.6",
-      "branch": "",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
+      "repo": "ckanext-twdh_theme",
+      "tag": "",
+      "branch": "ckan-2.10-upgrade",
+      "cmd": [
+        "pip install -e .",
+        "pip install -r requirements.txt",
+        "pip install Pillow"
+      ]
     }
   ]
 }

From c21295aff34a2d4bfcd0fd78a7a1b1f9b9ab46fe Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 7 Nov 2024 08:27:36 -0600
Subject: [PATCH 04/69] CKAN_DATASTORE_WRITE_URL and CKAN_VERSION updates

Updated CKAN_DATASTORE_WRITE_URL to use CKAN_DB_USER/PASSWORD instead of
incorrect DATASTORE_READONLY_USER/PASSWORD

Updated CKAN_VERSION to 2.10.5
---
 docker/.env-ckan.env | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 4a14415..57659ce 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -22,11 +22,11 @@ DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
 #DB CONNECTION STRINGS
-CKAN_DATASTORE_WRITE_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
+CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
 
 # General Settings
-CKAN_VERSION=2.9.9
+CKAN_VERSION=2.10.5
 CKAN_SITE_ID=default
 CKAN_SITE_URL=http://0.0.0.0:5000
 CKAN_PORT=5000

From ad4d4e9d5de8973a6dc36caec3781647150a0969 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 7 Nov 2024 08:29:23 -0600
Subject: [PATCH 05/69] Updated CKAN_PLUGINS list

added activity
removed harvest, ckan_harvest, scheming_organizations, datapusher

Will add datapusher back once we upgrade to version 1.x

Updated version on gevent to 22.10.2
---
 docker/ckan/Dockerfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index ad5c4cf..9c620eb 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -119,6 +119,7 @@ ENV CKAN__DEFAULT_VIEWS image_view \
 ENV CKAN__PLUGINS \
     stats \
     twdh_theme \
+    activity \
     security \
     report \
     check_link \
@@ -127,13 +128,10 @@ ENV CKAN__PLUGINS \
     image_view \
     video_view \
     datastore \
-    harvest \
-    ckan_harvester \
     spatial_metadata \
     spatial_query \
     twdh_schema \
     scheming_datasets \
-    scheming_organizations \
     composite \
     contact \
     pages \
@@ -143,7 +141,6 @@ ENV CKAN__PLUGINS \
     wmts_view \
     shp_view \
     datatablesview_plus \
-    datapusher \
     openapi_viewer \
     s3filestore \
     googleanalytics \
@@ -152,6 +149,10 @@ ENV CKAN__PLUGINS \
     envvars 
     # ^ envvars always needs to be last in the CKAN__PLUGINS list
     # s3filestore MUST PRECEDE googlealanalytics in the CKAN_PLUGINS list
+    # harvest \
+    # ckan_harvester \
+    #scheming_organizations \
+    # datapusher \
 
 # s3filestore env variables
 ENV CKANEXT__S3FILESTORE__AWS_BUCKET_NAME twdh-s3filestore
@@ -216,7 +217,7 @@ COPY ./plugins ${SRC_DIR}/plugins
 RUN ls ${SRC_DIR}/plugins
 
 # Additional install steps for build stages artifacts
-RUN pip install uWSGI==2.0.20 gevent==21.12.0
+RUN pip install uWSGI==2.0.20 gevent==22.10.2
 # Create a local user and group to run the app
 RUN groupadd -g 92 ckan && \
     useradd -rm -d /srv/app -s /bin/bash -g ckan -u 92 ckan

From bea7dfdb6c320f8d44094ae4d81cc260d75518ea Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 7 Nov 2024 08:33:29 -0600
Subject: [PATCH 06/69] Alphabetized plugins list; updated tags

Rearranged list of plugins to be in alpha order for easier comparison
when checking tag values
---
 docker/ckan/afterinit.d/00_harvest.sh |  3 ---
 docker/ckan/plugins/ckan_plugins.json | 18 ++----------------
 2 files changed, 2 insertions(+), 19 deletions(-)
 delete mode 100644 docker/ckan/afterinit.d/00_harvest.sh

diff --git a/docker/ckan/afterinit.d/00_harvest.sh b/docker/ckan/afterinit.d/00_harvest.sh
deleted file mode 100644
index 49d81f6..0000000
--- a/docker/ckan/afterinit.d/00_harvest.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-echo "@@@@@@ INIT HARVESTER @@@@@@"
-ckan -c /srv/app/production.ini harvester initdb
diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 4551379..102dc6f 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -10,7 +10,7 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-check-link",
-      "tag": "TWDHv0.5.0.rc1",
+      "tag": "",
       "branch": "ckan-2.10-upgrade",
       "cmd": ["python setup.py install", "pip install -r dev-requirements.txt"]
     },
@@ -60,20 +60,6 @@
       "branch": "",
       "cmd": ["python setup.py install", "pip install -r requirements.txt"]
     },
-    {
-      "owner": "TNRIS",
-      "repo": "ckanext-harvest",
-      "tag": "",
-      "branch": "master",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
-    },
-    {
-      "owner": "TNRIS",
-      "repo": "ckanext-officedocs",
-      "tag": "",
-      "branch": "master",
-      "cmd": ["python setup.py install"]
-    },
     {
       "owner": "TNRIS",
       "repo": "ckanext-openapi_viewer",
@@ -113,7 +99,7 @@
       "owner": "TNRIS",
       "repo": "ckanext-security",
       "tag": "",
-      "branch": "bugs-in-schema.py",
+      "branch": "master",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {

From 0b488dc30bd2d7ea2f3a20abe3f2ce7b241c2a0a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 7 Nov 2024 08:35:09 -0600
Subject: [PATCH 07/69] Added port forwarding to db config

---
 docker/docker-compose_local.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/docker-compose_local.yml b/docker/docker-compose_local.yml
index 141b45f..ee4c71a 100644
--- a/docker/docker-compose_local.yml
+++ b/docker/docker-compose_local.yml
@@ -56,6 +56,8 @@ services:
       - POSTGRES_HOST_AUTH_METHOD=trust
     healthcheck:
       test: ["CMD", "pg_isready", "-U", "postgres"]
+    ports: 
+      - "9000:5432"
 
   solr:
     container_name: solr

From 8eabb525c6294968a3e769ed609838a9ce4c9c9d Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 8 Nov 2024 19:05:33 +0000
Subject: [PATCH 08/69] Turn off collaborator functionality

---
 docker/.env-ckan.env | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 4a14415..fefc0d5 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -22,7 +22,7 @@ DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
 #DB CONNECTION STRINGS
-CKAN_DATASTORE_WRITE_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
+CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
 
 # General Settings
@@ -36,6 +36,8 @@ CKAN___DEBUG=false
 
 # Auth Settings
 CKAN__AUTH__CREATE_USER_VIA_WEB=false
+CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=false
+
 
 # Tracking Settings
 CKAN__TRACKING_ENABLED=true

From cafb3a64362c59fda3d1fba9be84e86dc9261194 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 4 Dec 2024 21:52:41 +0000
Subject: [PATCH 09/69] Corrected branch on ckanext-twdh_gazetteer

---
 docker/ckan/plugins/ckan_plugins.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 102dc6f..159a9fe 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -116,8 +116,8 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-twdh_gazetteer",
-      "tag": "bb-dev",
-      "branch": "",
+      "tag": "",
+      "branch": "bb-dev",
       "cmd": ["pip install -e ."]
     },
     {

From 99a0af709c9595b6b25cbfcd0847b6c62d97e337 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 12:06:07 -0600
Subject: [PATCH 10/69] Added 01_ckan_upgrade.sh db upgrade script

---
 docker/ckan/entrypoint.d/01_ckan_upgrade.sh | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 docker/ckan/entrypoint.d/01_ckan_upgrade.sh

diff --git a/docker/ckan/entrypoint.d/01_ckan_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_upgrade.sh
new file mode 100644
index 0000000..9cd6df2
--- /dev/null
+++ b/docker/ckan/entrypoint.d/01_ckan_upgrade.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
+ckan -c /srv/app/production.ini db upgrade 

From 2aaa744a22db747c5ef290ed77fd5c1a04d01038 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 13:09:46 -0600
Subject: [PATCH 11/69] Renamed and improved 01_ckan_db_upgrade.sh

Renamed 01_ckan_upgrade.sh to 01_ckan_db_upgrade.sh

Introduced conditional statement so that we can keep this file in place
but only have it run when we set UPGRADE=1
---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 12 ++++++++++++
 docker/ckan/entrypoint.d/01_ckan_upgrade.sh    |  3 ---
 2 files changed, 12 insertions(+), 3 deletions(-)
 create mode 100644 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
 delete mode 100644 docker/ckan/entrypoint.d/01_ckan_upgrade.sh

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
new file mode 100644
index 0000000..68ed9be
--- /dev/null
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
+
+UPGRADE=1
+
+if [ "$UPGRADE" -ne 1 ]; then
+    echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"
+      exit
+else 
+  ckan -c /srv/app/production.ini db upgrade 
+fi
+
diff --git a/docker/ckan/entrypoint.d/01_ckan_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_upgrade.sh
deleted file mode 100644
index 9cd6df2..0000000
--- a/docker/ckan/entrypoint.d/01_ckan_upgrade.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/sh
-echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
-ckan -c /srv/app/production.ini db upgrade 

From 338536693c0d27fe713750741e3924b17e45b7c4 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 13:11:23 -0600
Subject: [PATCH 12/69] Temporarily disable HEALTHCHECK

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 9c620eb..59c2464 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -287,7 +287,7 @@ COPY ./afterinit.d ${APP_DIR}/docker-afterinit.d
 
 EXPOSE 5000
 
-HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
+# HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
 USER ckan
 

From 852e5b1a5ead41b5e5ac85d738d4628dd7df0e7a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 13:42:27 -0600
Subject: [PATCH 13/69] Turned off UPGRADE toggle

---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
index 68ed9be..1cfb979 100644
--- a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
 
-UPGRADE=1
+UPGRADE=0
 
 if [ "$UPGRADE" -ne 1 ]; then
     echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"

From 266dab76fcfbc9189a1f645651f8f81ffac7a4f8 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 13:55:53 -0600
Subject: [PATCH 14/69] Removed errant exit statement

---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
index 1cfb979..e96a087 100644
--- a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -5,7 +5,6 @@ UPGRADE=0
 
 if [ "$UPGRADE" -ne 1 ]; then
     echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"
-      exit
 else 
   ckan -c /srv/app/production.ini db upgrade 
 fi

From ce9b2e55193ae51e5143c7a719957198ed08190d Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 14:14:19 -0600
Subject: [PATCH 15/69] Reenabled HEALTHCHECK

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 59c2464..9c620eb 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -287,7 +287,7 @@ COPY ./afterinit.d ${APP_DIR}/docker-afterinit.d
 
 EXPOSE 5000
 
-# HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
 USER ckan
 

From 145df271fdad756b431d803fcd8e701bd4b35d5d Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 14:27:11 -0600
Subject: [PATCH 16/69] Set HEALTHCHECK=NONE

---
 docker/ckan/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 9c620eb..8f2cfd7 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -287,7 +287,8 @@ COPY ./afterinit.d ${APP_DIR}/docker-afterinit.d
 
 EXPOSE 5000
 
-HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
+#HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
+HEALTHCHECK NONE
 
 USER ckan
 

From 5b29596515b5f9fb7d3fbfb52f45452e2d648c7b Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 14:38:23 -0600
Subject: [PATCH 17/69] Reenabled HEALTHCHECK

---
 docker/ckan/Dockerfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 8f2cfd7..9c620eb 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -287,8 +287,7 @@ COPY ./afterinit.d ${APP_DIR}/docker-afterinit.d
 
 EXPOSE 5000
 
-#HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
-HEALTHCHECK NONE
+HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
 USER ckan
 

From 8641ddba059a6fa4f5efbba496e643a52eda8e12 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 5 Dec 2024 14:42:52 -0600
Subject: [PATCH 18/69] Set UPGRADE=1 to initiate 2.10.5 db upgrade

---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
index e96a087..6ef8eb9 100644
--- a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
 
-UPGRADE=0
+UPGRADE=1
 
 if [ "$UPGRADE" -ne 1 ]; then
     echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"

From 53e1f8cacf725801447fe0ab6d30725a2c37dadb Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 6 Dec 2024 08:57:08 -0600
Subject: [PATCH 19/69] Set UPGRADE=0

---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
index 6ef8eb9..e96a087 100644
--- a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
 
-UPGRADE=1
+UPGRADE=0
 
 if [ "$UPGRADE" -ne 1 ]; then
     echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"

From dea1a5c21f5bff46a4fbbf7d422f96597be152aa Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 6 Dec 2024 18:02:51 +0000
Subject: [PATCH 20/69] Added variables

CKAN_BEAKER_SESSION_SECRET and CKAN_DATAPUSHER_API_TOKEN
---
 docker/.env-ckan.env | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 55a4925..a71930c 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -21,6 +21,9 @@ CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
 DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
+CKAN_BEAKER_SESSION_SECRET=x9zPgwq50xqN-mv4wAwZCFxHT
+CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJCLVFETlZ5VTJVTUdnTXRSdnZpSTIwcHloMkF5RS1sLWR4M1lZYUJfaC13Smw4Y01iSVAwOU8zYWhFXzVjVHhDWS1HVVRFMlhtTDZoclZtUCIsImlhdCI6MTcxMTcyNzQ4N30.26MVeO-9hKGj1YyoDhm_3Q0xCDlRFHaMg-B9iUnu6IA
+
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
@@ -62,6 +65,7 @@ CKAN_SYSADMIN_EMAIL=sysadmin@ckantest.com
 # CKAN_SMTP_MAIL_FROM=ckan@localhost
 
 # Datapusher configuration
+CKAN__DATAPUSHER__API_TOKEN=${CKAN_DATAPUSHER_API_TOKEN}
 CKAN__DATAPUSHER__URL=http://datapusher:8800
 CKAN__DATAPUSHER__CALLBACK_URL_BASE=http://0.0.0.0:5000/
 DATAPUSHER_VERSION=0.0.17
@@ -73,6 +77,7 @@ DATAPUSHER_SSL_VERIFY=False
 DATAPUSHER_REWRITE_RESOURCES=True
 DATAPUSHER_REWRITE_URL=http://0.0.0.0:5000
 
+
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
@@ -96,6 +101,7 @@ CKAN___BEAKER__SESSION__TYPE=ext:redis
 CKAN___BEAKER__SESSION__URL=redis://redis:6379/8
 CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true
 CKAN___BEAKER__SESSION__COOKIE_DOMAIN=0.0.0.0
+CKAN___BEAKER__SESSION__SECRET=${CKAN_BEAKER_SESSION_SECRET}
 CKANEXT__SECURITY__DOMAIN=0.0.0.0
 CKANEXT__SECURITY__REDIS__HOST=redis
 CKANEXT__SECURITY__REDIS__PORT=6379

From 9540fd3174495b75b594e635a745a1759b14a518 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 6 Dec 2024 18:03:56 +0000
Subject: [PATCH 21/69] Added datapusher_plus to PLUGINS

---
 docker/ckan/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 9c620eb..7786c34 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -141,6 +141,7 @@ ENV CKAN__PLUGINS \
     wmts_view \
     shp_view \
     datatablesview_plus \
+    datapusher_plus \
     openapi_viewer \
     s3filestore \
     googleanalytics \

From c5fb4fcbffced45c9e233594e17db0a7f1076396 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 6 Dec 2024 18:04:38 +0000
Subject: [PATCH 22/69] added ckanext-datapusher_plus block

---
 docker/ckan/plugins/ckan_plugins.json | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 159a9fe..eacab42 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -28,6 +28,13 @@
       "branch": "ckan-2.10-upgrade",
       "cmd": ["pip install -r requirements.txt", "python setup.py develop"]
     },
+    {
+      "owner": "TNRIS",
+      "repo": "ckanext-datapusher_plus",
+      "tag": "1.0.3",
+      "branch": "",
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
+    },
     {
       "owner": "TNRIS",
       "repo": "ckanext-datatablesview_plus",

From f2957c9582f765033bfa1c613d80b11695d164a2 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 9 Dec 2024 14:21:54 -0600
Subject: [PATCH 23/69] DP+ Upgrade - supervisor process and qsv

Added supervisor infrastructure to run `ckan -c production jobs worker` process
Added qsv installation
Added datapusher_plus to ckan_plugins.json and $CKAN_PLUGINS
---
 docker/ckan/Dockerfile                | 17 +++++++++++++++--
 docker/ckan/plugins/ckan_plugins.json |  2 +-
 docker/ckan/setup/app/start_ckan.sh   |  2 ++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 7786c34..1dd8650 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -193,6 +193,7 @@ RUN apt-get update && \
     libfreetype6 \
     libfreetype6-dev \
     zlib1g-dev \
+    supervisor \
     apache2-utils && \
     rm -rf /var/lib/apt/lists/* && \
     # Create SRC_DIR
@@ -265,7 +266,19 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     ckan config-tool ${APP_DIR}/production.ini "googleanalytics.id = G-SJK5R9G5F5" && \
     ckan config-tool ${APP_DIR}/production.ini "googleanalytics.download_handler = ckanext.s3filestore.views:resource_download" && \
     # Change ownership to app user
-    chown -R ckan:ckan /srv/app
+    chown -R ckan:ckan /srv/app && \
+    # Setup supervisor
+    cp -v /srv/app/src/ckan/ckan/config/supervisor-ckan-worker.conf /etc/supervisor/conf.d/ && \
+    mkdir -v /var/log/ckan
+
+
+# install Rust, then qsvdp
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    # source the cargo env to setup the $PATH for rust
+    . $HOME/.cargo/env && \
+    # install qsvdp from source
+    cargo install qsv --locked --bin qsvdp -F datapusher_plus,luau
+
 
 # Remove wheels
 RUN rm -rf /srv/app/wheels
@@ -290,6 +303,6 @@ EXPOSE 5000
 
 HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
-USER ckan
+# USER ckan
 
 CMD ["/srv/app/start_ckan.sh"]
diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index eacab42..038dea7 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -30,7 +30,7 @@
     },
     {
       "owner": "TNRIS",
-      "repo": "ckanext-datapusher_plus",
+      "repo": "datapusher-plus",
       "tag": "1.0.3",
       "branch": "",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
diff --git a/docker/ckan/setup/app/start_ckan.sh b/docker/ckan/setup/app/start_ckan.sh
index 1944e2b..1fdb859 100755
--- a/docker/ckan/setup/app/start_ckan.sh
+++ b/docker/ckan/setup/app/start_ckan.sh
@@ -50,6 +50,8 @@ then
       exit 1
     fi
   else
+    # Start supervisord
+    supervisord --configuration /etc/supervisor/supervisord.conf &
     # Start uwsgi
     uwsgi $UWSGI_OPTS
   fi

From 2ee9badd5b40f926db47a7fdb8e9ca461f1820d5 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 9 Dec 2024 20:37:57 +0000
Subject: [PATCH 24/69] Removed unused extensions from $CKAN_PLUGINS

---
 docker/ckan/Dockerfile | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 7786c34..83d926a 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -150,10 +150,6 @@ ENV CKAN__PLUGINS \
     envvars 
     # ^ envvars always needs to be last in the CKAN__PLUGINS list
     # s3filestore MUST PRECEDE googlealanalytics in the CKAN_PLUGINS list
-    # harvest \
-    # ckan_harvester \
-    #scheming_organizations \
-    # datapusher \
 
 # s3filestore env variables
 ENV CKANEXT__S3FILESTORE__AWS_BUCKET_NAME twdh-s3filestore

From 1efb3ade344727fcc76b7a5861a1996df4877a9b Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 9 Dec 2024 16:12:23 -0600
Subject: [PATCH 25/69] Updated docker/ckan/Dockerfile for supervisor

Changed ckan Dockerfile to pull supervisor-ckan-worker.conf from the
ckanext-twdh_theme instead of the version in CKAN core.
---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 1dd8650..36b6a92 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -268,7 +268,7 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     # Change ownership to app user
     chown -R ckan:ckan /srv/app && \
     # Setup supervisor
-    cp -v /srv/app/src/ckan/ckan/config/supervisor-ckan-worker.conf /etc/supervisor/conf.d/ && \
+    cp -v /srv/app/src/ckan/ckanext-twdh_theme/ckanext/twdh_theme/config/supervisor-ckan-worker.conf /etc/supervisor/conf.d/ && \
     mkdir -v /var/log/ckan
 
 

From 0ea4d80b39c80618bdf0c769b56da7421bf2ec70 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 10 Dec 2024 08:08:20 -0600
Subject: [PATCH 26/69] Changed way supervisor.conf gets copied

---
 docker/ckan/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 4c6bb85..a966124 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -263,10 +263,10 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     ckan config-tool ${APP_DIR}/production.ini "googleanalytics.download_handler = ckanext.s3filestore.views:resource_download" && \
     # Change ownership to app user
     chown -R ckan:ckan /srv/app && \
-    # Setup supervisor
-    cp -v /srv/app/src/ckan/ckanext-twdh_theme/ckanext/twdh_theme/config/supervisor-ckan-worker.conf /etc/supervisor/conf.d/ && \
     mkdir -v /var/log/ckan
 
+# Setup supervisor
+COPY ./supervisor-ckan-worker.conf /etc/supervisor/conf.d/
 
 # install Rust, then qsvdp
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \

From 56f141d7b6713d1e62473f62b102a55e1ddfa3ec Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 10 Dec 2024 08:08:43 -0600
Subject: [PATCH 27/69] Added supervisor-ckan-worker.conf config file

---
 docker/ckan/supervisor-ckan-worker.conf | 44 +++++++++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 docker/ckan/supervisor-ckan-worker.conf

diff --git a/docker/ckan/supervisor-ckan-worker.conf b/docker/ckan/supervisor-ckan-worker.conf
new file mode 100644
index 0000000..3df276e
--- /dev/null
+++ b/docker/ckan/supervisor-ckan-worker.conf
@@ -0,0 +1,44 @@
+; =======================================================
+; Supervisor configuration for CKAN background job worker
+; =======================================================
+
+; 1. Copy this file to /etc/supervisor/conf.d
+; 2. Make sure the paths below match your setup
+
+
+[program:ckan-worker]
+
+; Use the full paths to the virtualenv and your configuration file here.
+; command=/usr/lib/ckan/default/bin/ckan -c /etc/ckan/default/ckan.ini jobs worker
+command=/srv/app/virtualenv/bin/ckan -c production.ini jobs worker
+
+
+; User the worker runs as.
+; user=www-data
+user=ckan
+
+
+; Start just a single worker. Increase this number if you have many or
+; particularly long running background jobs.
+numprocs=1
+process_name=%(program_name)s-%(process_num)02d
+
+
+; Log files.
+stdout_logfile=/var/log/ckan/ckan-worker.stdout.log
+stderr_logfile=/var/log/ckan/ckan-worker.stderr.log
+
+
+; Make sure that the worker is started on system start and automatically
+; restarted if it crashes unexpectedly.
+autostart=true
+autorestart=true
+
+
+; Number of seconds the process has to run before it is considered to have
+; started successfully.
+startsecs=10
+
+; Need to wait for currently executing tasks to finish at shutdown.
+; Increase this if you have very long running tasks.
+stopwaitsecs = 600

From 841d61de2f726b600068b795dac56bb8cb3d72da Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 10 Dec 2024 13:48:46 -0600
Subject: [PATCH 28/69] Added supervisord.conf

---
 docker/ckan/Dockerfile       |  1 +
 docker/ckan/supervisord.conf | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 docker/ckan/supervisord.conf

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index a966124..2fcca14 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -266,6 +266,7 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     mkdir -v /var/log/ckan
 
 # Setup supervisor
+COPY ./supervisord.conf /etc/supervisor/
 COPY ./supervisor-ckan-worker.conf /etc/supervisor/conf.d/
 
 # install Rust, then qsvdp
diff --git a/docker/ckan/supervisord.conf b/docker/ckan/supervisord.conf
new file mode 100644
index 0000000..a3f6671
--- /dev/null
+++ b/docker/ckan/supervisord.conf
@@ -0,0 +1,23 @@
+[unix_http_server]
+file = /tmp/supervisor.sock
+chmod = 0777
+chown = nobody:nogroup
+
+[supervisord]
+logfile = /tmp/supervisord.log
+logfile_maxbytes = 50MB
+logfile_backups=10
+loglevel = info
+pidfile = /tmp/supervisord.pid
+nodaemon = true
+umask = 022
+identifier = supervisor
+
+[supervisorctl]
+serverurl = unix:///tmp/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[include]
+files = /etc/supervisord.d/*.conf

From d071ac40bd984491207892ea050e775b5812344e Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 10 Dec 2024 15:41:41 -0600
Subject: [PATCH 29/69] Fixed /etc/supervisor/conf.d/ path in supervisord.conf

---
 docker/ckan/supervisord.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/supervisord.conf b/docker/ckan/supervisord.conf
index a3f6671..e62a3ca 100644
--- a/docker/ckan/supervisord.conf
+++ b/docker/ckan/supervisord.conf
@@ -20,4 +20,4 @@ serverurl = unix:///tmp/supervisor.sock
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 [include]
-files = /etc/supervisord.d/*.conf
+files = /etc/supervisor/conf.d/*

From 409509a2ee01ee79333441216b5907815f7b92ec Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 17:22:08 +0000
Subject: [PATCH 30/69] Commented out beaker.session.secret in
 merge.production.ini

---
 docker/ckan/merge.production.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/ckan/merge.production.ini b/docker/ckan/merge.production.ini
index afbbfb5..a5118e7 100644
--- a/docker/ckan/merge.production.ini
+++ b/docker/ckan/merge.production.ini
@@ -1,7 +1,7 @@
 # <your other settings here>
 beaker.session.key = ckan_session
 # Your session secret should be a long, random and secret string!
-beaker.session.secret = beaker-secret
+#beaker.session.secret = beaker-secret
 beaker.session.data_serializer = json
 beaker.session.httponly = true
 beaker.session.secure = true
@@ -33,4 +33,4 @@ ckanext.security.disable_password_reset_override = true
 
 # Provide a help page to allow 2fa users to contact support or get more information
 # Shows up as 'Need help?' on the 2fa entry form beside the submit button. Does not display a link if none provided
-ckanext.security.mfa_help_link = https://data.govt.nz/catalogue-guide/releasing-data-on-data-govt-nz/how-do-i-set-up-two-factor-authentication/
\ No newline at end of file
+ckanext.security.mfa_help_link = https://data.govt.nz/catalogue-guide/releasing-data-on-data-govt-nz/how-do-i-set-up-two-factor-authentication/

From ae2a7135e091b273a7fed325270b17090018c868 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 18:06:34 +0000
Subject: [PATCH 31/69] Uncommented beaker.session.secret in
 merge.production.ini

---
 docker/ckan/merge.production.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/merge.production.ini b/docker/ckan/merge.production.ini
index a5118e7..f4903aa 100644
--- a/docker/ckan/merge.production.ini
+++ b/docker/ckan/merge.production.ini
@@ -1,7 +1,7 @@
 # <your other settings here>
 beaker.session.key = ckan_session
 # Your session secret should be a long, random and secret string!
-#beaker.session.secret = beaker-secret
+beaker.session.secret = beaker-secret
 beaker.session.data_serializer = json
 beaker.session.httponly = true
 beaker.session.secure = true

From efd07973b83c14db255902a31ac75f4be58335dc Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 20:28:11 +0000
Subject: [PATCH 32/69] Removed DP+ secrets

---
 docker/.env-ckan.env | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index a71930c..8995ffc 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -21,9 +21,6 @@ CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
 DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
-CKAN_BEAKER_SESSION_SECRET=x9zPgwq50xqN-mv4wAwZCFxHT
-CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJCLVFETlZ5VTJVTUdnTXRSdnZpSTIwcHloMkF5RS1sLWR4M1lZYUJfaC13Smw4Y01iSVAwOU8zYWhFXzVjVHhDWS1HVVRFMlhtTDZoclZtUCIsImlhdCI6MTcxMTcyNzQ4N30.26MVeO-9hKGj1YyoDhm_3Q0xCDlRFHaMg-B9iUnu6IA
-
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore

From 7195b0d5f864e56af2502fd01a5aa3a8bd32b106 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 21:54:17 +0000
Subject: [PATCH 33/69] Added to .env-ckan.env

---
 docker/.env-ckan.env | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 8995ffc..f477633 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -21,6 +21,9 @@ CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
 DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
+CKAN_BEAKER_SESSION_SECRET=RIeHgKbiMu99qo98-i2kyjQxA
+CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJROEZvOWN5Y2NEQklCaVFUbHp5TExiSDlBV1o0dzdBTTZIMkx0cXlwTVRRIiwiaWF0IjoxNzMzOTQzNjI1fQ.dWusfJDXks2ADrRCuZa8aq1QNjU3iv4xZHgAK_kbxSU
+
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore

From ac49cf88ecc594e8bcf8ef849f99af8cdd9a2d69 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 22:28:20 +0000
Subject: [PATCH 34/69] Removed CKAN__DATAPUSHER__API from .ckan-env.env

---
 docker/.env-ckan.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index f477633..2105137 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -22,7 +22,7 @@ DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
 CKAN_BEAKER_SESSION_SECRET=RIeHgKbiMu99qo98-i2kyjQxA
-CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJROEZvOWN5Y2NEQklCaVFUbHp5TExiSDlBV1o0dzdBTTZIMkx0cXlwTVRRIiwiaWF0IjoxNzMzOTQzNjI1fQ.dWusfJDXks2ADrRCuZa8aq1QNjU3iv4xZHgAK_kbxSU
+#CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJOTmNVRGpvVHpkTU9CT3NId3c0NWpWcDFHWlFfc19DWGVLdkhrZW0tY1ZnIiwiaWF0IjoxNzMzOTU1OTc3fQ.Izx7VqBNHV5OAAhseqRa20EPUjxVNgBua1kuIxZ_rpk
 
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore

From 428d4b634c2ba865d106622f95564de0f68fc3db Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 22:56:04 +0000
Subject: [PATCH 35/69] Set CKAN_DATAPUSHER_API_TOKEN in .env-ckan.env

---
 docker/.env-ckan.env | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 2105137..5f63958 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -22,7 +22,7 @@ DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
 CKAN_BEAKER_SESSION_SECRET=RIeHgKbiMu99qo98-i2kyjQxA
-#CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJOTmNVRGpvVHpkTU9CT3NId3c0NWpWcDFHWlFfc19DWGVLdkhrZW0tY1ZnIiwiaWF0IjoxNzMzOTU1OTc3fQ.Izx7VqBNHV5OAAhseqRa20EPUjxVNgBua1kuIxZ_rpk
+CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJOTmNVRGpvVHpkTU9CT3NId3c0NWpWcDFHWlFfc19DWGVLdkhrZW0tY1ZnIiwiaWF0IjoxNzMzOTU1OTc3fQ.Izx7VqBNHV5OAAhseqRa20EPUjxVNgBua1kuIxZ_rpk
 
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore

From 5d4c2518e461f44de7f9e7c1de365fdf82d9b4ec Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 23:51:15 +0000
Subject: [PATCH 36/69] Added comment

---
 docker/ckan/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 2fcca14..dccd072 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -263,6 +263,7 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     ckan config-tool ${APP_DIR}/production.ini "googleanalytics.download_handler = ckanext.s3filestore.views:resource_download" && \
     # Change ownership to app user
     chown -R ckan:ckan /srv/app && \
+    # Create log directory for `ckan jobs worker` process
     mkdir -v /var/log/ckan
 
 # Setup supervisor

From a2c657515d3472efb732ae763a6ffafbedeac2c0 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 11 Dec 2024 23:51:58 +0000
Subject: [PATCH 37/69] TESTING added variables to .ckan-env.env

CKAN___API_TOKEN__JWT__ENCODE__SECRET
CKAN___API_TOKEN__JWT__DECODE__SECRET
---
 docker/.env-ckan.env | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 5f63958..64c3d89 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -102,6 +102,10 @@ CKAN___BEAKER__SESSION__URL=redis://redis:6379/8
 CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true
 CKAN___BEAKER__SESSION__COOKIE_DOMAIN=0.0.0.0
 CKAN___BEAKER__SESSION__SECRET=${CKAN_BEAKER_SESSION_SECRET}
+
+CKAN___API_TOKEN__JWT__ENCODE__SECRET=${CKAN_BEAKER_SESSION_SECRET}
+CKAN___API_TOKEN__JWT__DECODE__SECRET=${CKAN_BEAKER_SESSION_SECRET}
+
 CKANEXT__SECURITY__DOMAIN=0.0.0.0
 CKANEXT__SECURITY__REDIS__HOST=redis
 CKANEXT__SECURITY__REDIS__PORT=6379
@@ -111,3 +115,6 @@ CKANEXT__SECURITY__LOGIN_MAX_COUNT=10
 CKANEXT__SECURITY__BRUTE_FORCE_KEY=user_name
 CKANEXT__SECURITY__DISABLE_PASSWORD_RESET_OVERRIDE=true
 CKANEXT__SECURITY__MFA_HELP_LINK=http://0.0.0.0:5000/pages/mfa
+
+
+CKAN__DATAPUSHER__API_TOKEN=${CKAN_DATAPUSHER_API_TOKEN}

From 5d0f648766e85e30e2208beddc16fb05e2991860 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 12 Dec 2024 00:26:24 +0000
Subject: [PATCH 38/69] Rolling back changes

---
 docker/.env-ckan.env | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 64c3d89..2945f83 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -21,8 +21,8 @@ CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
 DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
-CKAN_BEAKER_SESSION_SECRET=RIeHgKbiMu99qo98-i2kyjQxA
-CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJOTmNVRGpvVHpkTU9CT3NId3c0NWpWcDFHWlFfc19DWGVLdkhrZW0tY1ZnIiwiaWF0IjoxNzMzOTU1OTc3fQ.Izx7VqBNHV5OAAhseqRa20EPUjxVNgBua1kuIxZ_rpk
+#CKAN_BEAKER_SESSION_SECRET=RIeHgKbiMu99qo98-i2kyjQxA
+#CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJOTmNVRGpvVHpkTU9CT3NId3c0NWpWcDFHWlFfc19DWGVLdkhrZW0tY1ZnIiwiaWF0IjoxNzMzOTU1OTc3fQ.Izx7VqBNHV5OAAhseqRa20EPUjxVNgBua1kuIxZ_rpk
 
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
@@ -101,10 +101,7 @@ CKAN___BEAKER__SESSION__TYPE=ext:redis
 CKAN___BEAKER__SESSION__URL=redis://redis:6379/8
 CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true
 CKAN___BEAKER__SESSION__COOKIE_DOMAIN=0.0.0.0
-CKAN___BEAKER__SESSION__SECRET=${CKAN_BEAKER_SESSION_SECRET}
-
-CKAN___API_TOKEN__JWT__ENCODE__SECRET=${CKAN_BEAKER_SESSION_SECRET}
-CKAN___API_TOKEN__JWT__DECODE__SECRET=${CKAN_BEAKER_SESSION_SECRET}
+#CKAN___BEAKER__SESSION__SECRET=${CKAN_BEAKER_SESSION_SECRET}
 
 CKANEXT__SECURITY__DOMAIN=0.0.0.0
 CKANEXT__SECURITY__REDIS__HOST=redis
@@ -116,5 +113,4 @@ CKANEXT__SECURITY__BRUTE_FORCE_KEY=user_name
 CKANEXT__SECURITY__DISABLE_PASSWORD_RESET_OVERRIDE=true
 CKANEXT__SECURITY__MFA_HELP_LINK=http://0.0.0.0:5000/pages/mfa
 
-
-CKAN__DATAPUSHER__API_TOKEN=${CKAN_DATAPUSHER_API_TOKEN}
+#CKAN__DATAPUSHER__API_TOKEN=${CKAN_DATAPUSHER_API_TOKEN}

From f060ae028a3726c3a5e840ad57176547630ed3e6 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 14:50:58 +0000
Subject: [PATCH 39/69] Set USER back to ckan

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index dccd072..d4aa425 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -301,6 +301,6 @@ EXPOSE 5000
 
 HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
-# USER ckan
+USER ckan
 
 CMD ["/srv/app/start_ckan.sh"]

From 3823d428868d9fe8bbcd64b25905433096a503de Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 15:35:03 +0000
Subject: [PATCH 40/69] Copy qsvdp into /usr/local/bin/

---
 docker/ckan/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index d4aa425..6bf10c8 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -275,7 +275,8 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
     # source the cargo env to setup the $PATH for rust
     . $HOME/.cargo/env && \
     # install qsvdp from source
-    cargo install qsv --locked --bin qsvdp -F datapusher_plus,luau
+    cargo install qsv --locked --bin qsvdp -F datapusher_plus,luau && \
+    cp -v /root/.cargo/bin/qsvdp /usr/local/bin/
 
 
 # Remove wheels

From aac256810c2d7a56cf5cf9082c4bd0a6ecb7cc10 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 16:04:42 +0000
Subject: [PATCH 41/69] Changed log dir from /tmp to /var/log/supervisor

---
 docker/ckan/supervisord.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/supervisord.conf b/docker/ckan/supervisord.conf
index e62a3ca..79be3f9 100644
--- a/docker/ckan/supervisord.conf
+++ b/docker/ckan/supervisord.conf
@@ -4,7 +4,7 @@ chmod = 0777
 chown = nobody:nogroup
 
 [supervisord]
-logfile = /tmp/supervisord.log
+logfile = /var/log/supervisor/supervisord.log
 logfile_maxbytes = 50MB
 logfile_backups=10
 loglevel = info

From 8226151fca8c61a767356df070806d3cdcc5087d Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 16:06:28 +0000
Subject: [PATCH 42/69] Trying running as root again

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 6bf10c8..45b913a 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -302,6 +302,6 @@ EXPOSE 5000
 
 HEALTHCHECK --interval=10s --timeout=5s --retries=5 CMD curl --fail http://0.0.0.0:5000/api/3/action/status_show || exit 1
 
-USER ckan
+#USER ckan
 
 CMD ["/srv/app/start_ckan.sh"]

From 2ed0e8f7fe1e59e30a42ae1b1b8bd3f55d3f4dd6 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 16:06:40 +0000
Subject: [PATCH 43/69] Set supervisor to run as user ckan

---
 docker/ckan/supervisord.conf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ckan/supervisord.conf b/docker/ckan/supervisord.conf
index 79be3f9..362396e 100644
--- a/docker/ckan/supervisord.conf
+++ b/docker/ckan/supervisord.conf
@@ -12,6 +12,7 @@ pidfile = /tmp/supervisord.pid
 nodaemon = true
 umask = 022
 identifier = supervisor
+user = ckan
 
 [supervisorctl]
 serverurl = unix:///tmp/supervisor.sock

From d888811911c7ceb35ed0ec7db013c983a7fb910c Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 16:47:00 +0000
Subject: [PATCH 44/69] mv instead of cp qsvdp binary

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 45b913a..ac6cd42 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -276,7 +276,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
     . $HOME/.cargo/env && \
     # install qsvdp from source
     cargo install qsv --locked --bin qsvdp -F datapusher_plus,luau && \
-    cp -v /root/.cargo/bin/qsvdp /usr/local/bin/
+    mv -v /root/.cargo/bin/qsvdp /usr/local/bin/
 
 
 # Remove wheels

From ce286d7342450b6f01ce50cef2e80d05f8182c44 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 16:47:21 +0000
Subject: [PATCH 45/69] Not running as ckan due to permision problem

---
 docker/ckan/supervisord.conf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/ckan/supervisord.conf b/docker/ckan/supervisord.conf
index 362396e..79be3f9 100644
--- a/docker/ckan/supervisord.conf
+++ b/docker/ckan/supervisord.conf
@@ -12,7 +12,6 @@ pidfile = /tmp/supervisord.pid
 nodaemon = true
 umask = 022
 identifier = supervisor
-user = ckan
 
 [supervisorctl]
 serverurl = unix:///tmp/supervisor.sock

From c33e19ef6b0392fb62248d8507701e13d0b38c03 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 17:25:49 +0000
Subject: [PATCH 46/69] Install file command required by qsv

---
 docker/ckan/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index ac6cd42..b52a7cf 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -190,6 +190,7 @@ RUN apt-get update && \
     libfreetype6-dev \
     zlib1g-dev \
     supervisor \
+    file \
     apache2-utils && \
     rm -rf /var/lib/apt/lists/* && \
     # Create SRC_DIR
@@ -276,7 +277,8 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
     . $HOME/.cargo/env && \
     # install qsvdp from source
     cargo install qsv --locked --bin qsvdp -F datapusher_plus,luau && \
-    mv -v /root/.cargo/bin/qsvdp /usr/local/bin/
+    # mv the qsvdp binary so that ckan user can run it
+    mv /root/.cargo/bin/qsvdp /usr/local/bin/
 
 
 # Remove wheels

From 9e6a4c3ce19d8a98264e77246591fe8e2ba234cd Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 13 Dec 2024 19:04:53 +0000
Subject: [PATCH 47/69] Added uchardet to apt-get

---
 docker/ckan/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index b52a7cf..21eafdb 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -191,6 +191,7 @@ RUN apt-get update && \
     zlib1g-dev \
     supervisor \
     file \
+    uchardet \
     apache2-utils && \
     rm -rf /var/lib/apt/lists/* && \
     # Create SRC_DIR

From 0a6469697586d9bdccbf15ff5ea10c347118a981 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 16 Dec 2024 20:40:15 +0000
Subject: [PATCH 48/69] Changed dp+ branch to dev-v1.0

---
 docker/ckan/plugins/ckan_plugins.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 038dea7..33b96c8 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -31,7 +31,7 @@
     {
       "owner": "TNRIS",
       "repo": "datapusher-plus",
-      "tag": "1.0.3",
+      "tag": "dev-v1.0",
       "branch": "",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },

From 4b5b181777045313cbde57d23fc7349f7f230f7a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 16 Dec 2024 15:19:38 -0600
Subject: [PATCH 49/69] SOLR field config

Added solr field for singular 'group' field inaddition to plural
'groups' field. Presuming this fixes the error we are seeing, I will
then investigate if we can/should remove the redundant 'groups' field.
---
 docker/solr8/solr-configset/overwrite.schema.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/solr8/solr-configset/overwrite.schema.xml b/docker/solr8/solr-configset/overwrite.schema.xml
index 43d6048..d005195 100644
--- a/docker/solr8/solr-configset/overwrite.schema.xml
+++ b/docker/solr8/solr-configset/overwrite.schema.xml
@@ -125,6 +125,7 @@ attribute with the form `ckan-X.Y` -->
     <field name="license" type="string" indexed="true" stored="true" />
     <field name="license_id" type="string" indexed="true" stored="true" />
     <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="group" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
 
@@ -195,6 +196,7 @@ attribute with the form `ckan-X.Y` -->
 <copyField source="license" dest="text"/>
 <copyField source="notes" dest="text"/>
 <copyField source="tags" dest="text"/>
+<copyField source="group" dest="text"/>
 <copyField source="groups" dest="text"/>
 <copyField source="organization" dest="text"/>
 <copyField source="res_name" dest="text"/>

From d2e65349198808baa3de2232079cfd6bb84a9a84 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 6 Jan 2025 16:01:01 +0000
Subject: [PATCH 50/69] Added "group" field definition to SOLR config

---
 docker/solr8/solr-configset/overwrite.schema.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker/solr8/solr-configset/overwrite.schema.xml b/docker/solr8/solr-configset/overwrite.schema.xml
index 43d6048..d005195 100644
--- a/docker/solr8/solr-configset/overwrite.schema.xml
+++ b/docker/solr8/solr-configset/overwrite.schema.xml
@@ -125,6 +125,7 @@ attribute with the form `ckan-X.Y` -->
     <field name="license" type="string" indexed="true" stored="true" />
     <field name="license_id" type="string" indexed="true" stored="true" />
     <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="group" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
 
@@ -195,6 +196,7 @@ attribute with the form `ckan-X.Y` -->
 <copyField source="license" dest="text"/>
 <copyField source="notes" dest="text"/>
 <copyField source="tags" dest="text"/>
+<copyField source="group" dest="text"/>
 <copyField source="groups" dest="text"/>
 <copyField source="organization" dest="text"/>
 <copyField source="res_name" dest="text"/>

From 5591ef8eeb7eec5316c5aec79c3b888e8c30d1e1 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 6 Jan 2025 10:07:26 -0600
Subject: [PATCH 51/69] Changed tag template to twdh-*.*.*

---
 .github/workflows/wf1.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/wf1.yaml b/.github/workflows/wf1.yaml
index 5613c3b..da0e80b 100644
--- a/.github/workflows/wf1.yaml
+++ b/.github/workflows/wf1.yaml
@@ -3,7 +3,7 @@ run-name: ${{ github.actor }} created a new release
 on: 
   push:
     tags:
-      - v*.*.*-*
+      - twdh-*.*.*
 jobs:
   test-image:
     runs-on: ubuntu-22.04
@@ -61,4 +61,4 @@ jobs:
         env:
           SOURCE_NAME: ${{ steps.name_branch_tags.outputs.SOURCE_NAME }}
           SOURCE_BRANCH: ${{ steps.name_branch_tags.outputs.SOURCE_BRANCH }}
-          SOURCE_TAG: ${{ steps.name_branch_tags.outputs.SOURCE_TAG }}
\ No newline at end of file
+          SOURCE_TAG: ${{ steps.name_branch_tags.outputs.SOURCE_TAG }}

From 2470496ba39dfc582386d648f2535f30f8225407 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 7 Jan 2025 09:33:28 -0600
Subject: [PATCH 52/69] Added output to show git clone commands

---
 docker/ckan/plugins/plugins.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/plugins.py b/docker/ckan/plugins/plugins.py
index b0697c5..0437ada 100644
--- a/docker/ckan/plugins/plugins.py
+++ b/docker/ckan/plugins/plugins.py
@@ -22,8 +22,10 @@ def install(self):
         git = f"git clone"
         if(self.branch != ""):
             git = f"git clone -b {self.branch}"
+            print(git)
         if(self.tag != ""):
             git = f"git clone -b {self.tag}"
+            print(git)
         
         git = git + f" --depth 1 https://{GH_TOKEN}@github.com/{self.owner}/{self.repo} ./{self.repo}"
 
@@ -47,4 +49,4 @@ def install(self):
         p["cmd"]
     )
 
-    plugin.install()
\ No newline at end of file
+    plugin.install()

From 6461cf5c55d5011641bd14785fc99271b0ae72b1 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 9 Jan 2025 18:28:26 +0000
Subject: [PATCH 53/69] actions/upload-artifact@v3 -> v4

actions/upload-artifact@v3 is deprecated

https://github.blog/changelog/2024-04-16-deprecation-notice-v3-of-the-artifact-actions/
---
 .github/workflows/wf1.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/wf1.yaml b/.github/workflows/wf1.yaml
index 5613c3b..0e16d97 100644
--- a/.github/workflows/wf1.yaml
+++ b/.github/workflows/wf1.yaml
@@ -48,7 +48,7 @@ jobs:
       - run: |
           cd docker && docker compose --file docker-compose_local.yml up -d --wait
       - name: docker_log
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: build_log
           path: build.log
@@ -61,4 +61,4 @@ jobs:
         env:
           SOURCE_NAME: ${{ steps.name_branch_tags.outputs.SOURCE_NAME }}
           SOURCE_BRANCH: ${{ steps.name_branch_tags.outputs.SOURCE_BRANCH }}
-          SOURCE_TAG: ${{ steps.name_branch_tags.outputs.SOURCE_TAG }}
\ No newline at end of file
+          SOURCE_TAG: ${{ steps.name_branch_tags.outputs.SOURCE_TAG }}

From 3f1e9b8b5384f22ce6276963efa3610627fcba4d Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 9 Jan 2025 12:53:38 -0600
Subject: [PATCH 54/69] Reactivated scheming_organizations plugin

---
 docker/ckan/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 21eafdb..9485020 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -132,6 +132,7 @@ ENV CKAN__PLUGINS \
     spatial_query \
     twdh_schema \
     scheming_datasets \
+    scheming_organizations \
     composite \
     contact \
     pages \

From 188215e08ae5b08a292f196d53e491b23d7d45dd Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 13 Jan 2025 14:48:39 -0600
Subject: [PATCH 55/69] Updated plugin branches & tags

---
 docker/ckan/plugins/ckan_plugins.json | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 33b96c8..9d86529 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -3,7 +3,7 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-agsview",
-      "tag": "TWDHv0.2.9a",
+      "tag": "twdh-0.6.0",
       "branch": "",
       "cmd": ["pip install -r requirements.txt", "python setup.py develop"]
     },
@@ -31,8 +31,8 @@
     {
       "owner": "TNRIS",
       "repo": "datapusher-plus",
-      "tag": "dev-v1.0",
-      "branch": "",
+      "tag": "",
+      "branch": "dev-v1.0",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
@@ -56,14 +56,14 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-geoview",
-      "tag": "TWDHv0.2.6",
-      "branch": "master",
+      "tag": "twdh-0.6.0",
+      "branch": "",
       "cmd": ["python setup.py install"]
     },
     {
       "owner": "TNRIS",
       "repo": "ckanext-googleanalytics",
-      "tag": "TWDHv0.5.0.rc1",
+      "tag": "twdh-0.6.0",
       "branch": "",
       "cmd": ["python setup.py install", "pip install -r requirements.txt"]
     },
@@ -77,7 +77,7 @@
     {
       "owner": "TNRIS",
       "repo": "ckanext-pages",
-      "tag": "TWDHv0.5.0.rc1",
+      "tag": "twdh-0.6.0",
       "branch": "",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },

From 4dd727cc18955f9e0b02d3002c618e8ee28920ff Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Mon, 13 Jan 2025 15:16:18 -0600
Subject: [PATCH 56/69] Updated plugin branches & tags

---
 docker/ckan/plugins/ckan_plugins.json | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 9d86529..10a07f1 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -47,10 +47,10 @@
       ]
     },
     {
-      "owner": "okfn",
+      "owner": "TNRIS",
       "repo": "ckanext-envvars",
-      "tag": "",
-      "branch": "master",
+      "tag": "twdh-0.6.0",
+      "branch": "",
       "cmd": ["python setup.py install"]
     },
     {
@@ -106,14 +106,14 @@
       "owner": "TNRIS",
       "repo": "ckanext-security",
       "tag": "",
-      "branch": "master",
+      "branch": "dp+-conflict",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",
       "repo": "ckanext-spatial",
-      "tag": "",
-      "branch": "bb-dev",
+      "tag": "twdh-0.6.0",
+      "branch": "",
       "cmd": [
         "pip install -e .",
         "pip install -r requirements.txt",

From 277fa6b9878b0d8f698e006f228a0d04494c3d20 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 14 Jan 2025 12:52:21 -0600
Subject: [PATCH 57/69] Revised qsv install process

---
 docker/ckan/Dockerfile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 9485020..7df88c9 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -278,10 +278,13 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
     # source the cargo env to setup the $PATH for rust
     . $HOME/.cargo/env && \
     # install qsvdp from source
-    cargo install qsv --locked --bin qsvdp -F datapusher_plus,luau && \
+    git clone https://github.com/dathere/qsv.git && \
+    cd qsv && \
+    export CARGO_BUILD_RUSTFLAGS='-C target-cpu=native' && \
+    cargo build --release --locked --bin qsvdp -F datapusher_plus,luau,polars && \
     # mv the qsvdp binary so that ckan user can run it
-    mv /root/.cargo/bin/qsvdp /usr/local/bin/
-
+    mv target/release/qsvdp /usr/local/bin/ && \
+    cargo clean
 
 # Remove wheels
 RUN rm -rf /srv/app/wheels

From 8281bb5c95edc6bac46775a8bad6c9d04b729378 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 15 Jan 2025 08:29:41 -0600
Subject: [PATCH 58/69] Set datapusher branch to druf-phase1

---
 docker/ckan/plugins/ckan_plugins.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 10a07f1..0fb5abd 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -32,7 +32,7 @@
       "owner": "TNRIS",
       "repo": "datapusher-plus",
       "tag": "",
-      "branch": "dev-v1.0",
+      "branch": "druf-phase1",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {

From cf77ec0dfcd881cafc55f766a29a9c9b4715f82a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 15 Jan 2025 08:53:01 -0600
Subject: [PATCH 59/69] Updated datapusher-plus cmd

---
 docker/ckan/plugins/ckan_plugins.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 0fb5abd..ef77bba 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -33,7 +33,7 @@
       "repo": "datapusher-plus",
       "tag": "",
       "branch": "druf-phase1",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
+      "cmd": ["pip install wheel", "pip install -e ."]
     },
     {
       "owner": "TNRIS",

From 269b02290d74853844966a7c78e66f745033167a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 15 Jan 2025 14:07:44 -0600
Subject: [PATCH 60/69] Updated datapusher-plus plugin

---
 docker/ckan/plugins/ckan_plugins.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index ef77bba..fcc1303 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -33,7 +33,7 @@
       "repo": "datapusher-plus",
       "tag": "",
       "branch": "druf-phase1",
-      "cmd": ["pip install wheel", "pip install -e ."]
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"],
     },
     {
       "owner": "TNRIS",

From abbeeb05d4e09e6a3935eaac4361ae045e2fcc6a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 15 Jan 2025 14:18:35 -0600
Subject: [PATCH 61/69] Fixed typo

---
 docker/ckan/plugins/ckan_plugins.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index fcc1303..0fb5abd 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -33,7 +33,7 @@
       "repo": "datapusher-plus",
       "tag": "",
       "branch": "druf-phase1",
-      "cmd": ["pip install -e .", "pip install -r requirements.txt"],
+      "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {
       "owner": "TNRIS",

From 23ab98cedb193f9ff712c54bf8af902c1f5bad51 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Wed, 15 Jan 2025 15:05:25 -0600
Subject: [PATCH 62/69] Fixed branch name

---
 docker/ckan/plugins/ckan_plugins.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/plugins/ckan_plugins.json b/docker/ckan/plugins/ckan_plugins.json
index 0fb5abd..e5837cc 100644
--- a/docker/ckan/plugins/ckan_plugins.json
+++ b/docker/ckan/plugins/ckan_plugins.json
@@ -32,7 +32,7 @@
       "owner": "TNRIS",
       "repo": "datapusher-plus",
       "tag": "",
-      "branch": "druf-phase1",
+      "branch": "druf-phase1-wip",
       "cmd": ["pip install -e .", "pip install -r requirements.txt"]
     },
     {

From fa1302272c8390ba4d89cb262803f7211608ef7a Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 16 Jan 2025 08:10:51 -0600
Subject: [PATCH 63/69] Turned on db UPGRADE

---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
index e96a087..f429c0c 100644
--- a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -1,11 +1,12 @@
 #!/bin/sh
 echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
 
-UPGRADE=0
+UPGRADE=1
 
 if [ "$UPGRADE" -ne 1 ]; then
     echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"
 else 
   ckan -c /srv/app/production.ini db upgrade 
+  ckan -c /srv/app/production.ini db pending-migrations --apply 
 fi
 

From 5a0cca8c19df43c10fc033f22ff05e47542ea213 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Thu, 16 Jan 2025 13:28:05 -0600
Subject: [PATCH 64/69] Turned UPDRADE off

---
 docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
index f429c0c..6d18592 100644
--- a/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
+++ b/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 echo "@@@@@@ UPGRADE CKAN DB @@@@@@"
 
-UPGRADE=1
+UPGRADE=0
 
 if [ "$UPGRADE" -ne 1 ]; then
     echo "Warning: UPGRADE not enabled. If you want db upgrade to run, set UPGRADE=1 in twdh_ckan_docker/docker/ckan/entrypoint.d/01_ckan_db_upgrade.sh"

From 550c7783b98499d4c91860452aca36bfb1253434 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 17 Jan 2025 08:30:31 -0600
Subject: [PATCH 65/69] Reverted to non-DRUF dataset.yaml

---
 docker/ckan/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/ckan/Dockerfile b/docker/ckan/Dockerfile
index 7df88c9..a1b1171 100644
--- a/docker/ckan/Dockerfile
+++ b/docker/ckan/Dockerfile
@@ -254,7 +254,7 @@ RUN ckan config-tool ${APP_DIR}/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
     # upgrade ckan_max_resource_size
     ckan config-tool ${APP_DIR}/production.ini "ckan.max_resource_size = ${CKAN__MAX_RESOURCE_SIZE}" && \
     # add scheming config
-    ckan config-tool ${APP_DIR}/production.ini "scheming.dataset_schemas = ckanext.twdh_schema:schemas/dataset-druf.yaml ckanext.twdh_schema:schemas/application.yaml" && \
+    ckan config-tool ${APP_DIR}/production.ini "scheming.dataset_schemas = ckanext.twdh_schema:schemas/dataset.yaml ckanext.twdh_schema:schemas/application.yaml" && \
     ckan config-tool ${APP_DIR}/production.ini "scheming.organization_schemas = ckanext.twdh_schema:schemas/organization.yaml" && \
     ckan config-tool ${APP_DIR}/production.ini "scheming.presets = ckanext.scheming:presets.json ckanext.composite:presets.json ckanext.twdh_schema:schemas/presets.yaml" && \
     # add ckan.spatial.search_backend to enable solr spatial search

From 89029a3c23a23aa8e3030504fb3be9406c243b22 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Fri, 17 Jan 2025 21:50:52 +0000
Subject: [PATCH 66/69] Removed testing cruft

---
 docker/.env-ckan.env | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 2945f83..a68938f 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -21,9 +21,6 @@ CKAN_MAX_UPLOAD_SIZE_MB=${CKAN_MAX_UPLOAD_SIZE_MB}
 DATASTORE_READONLY_USER=datastore
 DATASTORE_READONLY_PASSWORD=datastore
 
-#CKAN_BEAKER_SESSION_SECRET=RIeHgKbiMu99qo98-i2kyjQxA
-#CKAN_DATAPUSHER_API_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJOTmNVRGpvVHpkTU9CT3NId3c0NWpWcDFHWlFfc19DWGVLdkhrZW0tY1ZnIiwiaWF0IjoxNzMzOTU1OTc3fQ.Izx7VqBNHV5OAAhseqRa20EPUjxVNgBua1kuIxZ_rpk
-
 #DB CONNECTION STRINGS
 CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@db/datastore
 CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@db/datastore
@@ -101,7 +98,6 @@ CKAN___BEAKER__SESSION__TYPE=ext:redis
 CKAN___BEAKER__SESSION__URL=redis://redis:6379/8
 CKAN___BEAKER__SESSION__COOKIE_EXPIRES=true
 CKAN___BEAKER__SESSION__COOKIE_DOMAIN=0.0.0.0
-#CKAN___BEAKER__SESSION__SECRET=${CKAN_BEAKER_SESSION_SECRET}
 
 CKANEXT__SECURITY__DOMAIN=0.0.0.0
 CKANEXT__SECURITY__REDIS__HOST=redis
@@ -113,4 +109,3 @@ CKANEXT__SECURITY__BRUTE_FORCE_KEY=user_name
 CKANEXT__SECURITY__DISABLE_PASSWORD_RESET_OVERRIDE=true
 CKANEXT__SECURITY__MFA_HELP_LINK=http://0.0.0.0:5000/pages/mfa
 
-#CKAN__DATAPUSHER__API_TOKEN=${CKAN_DATAPUSHER_API_TOKEN}

From 75afff707aa220f5cba9facf005a53316a5d803b Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 21 Jan 2025 15:48:02 -0600
Subject: [PATCH 67/69] Cleanup

---
 docker/.env-ckan.env             | 2 --
 docker/ckan/merge.production.ini | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/docker/.env-ckan.env b/docker/.env-ckan.env
index 2945f83..05340f7 100644
--- a/docker/.env-ckan.env
+++ b/docker/.env-ckan.env
@@ -41,7 +41,6 @@ CKAN___DEBUG=false
 CKAN__AUTH__CREATE_USER_VIA_WEB=false
 CKAN__AUTH__ALLOW_DATASET_COLLABORATORS=false
 
-
 # Tracking Settings
 CKAN__TRACKING_ENABLED=true
 
@@ -77,7 +76,6 @@ DATAPUSHER_SSL_VERIFY=False
 DATAPUSHER_REWRITE_RESOURCES=True
 DATAPUSHER_REWRITE_URL=http://0.0.0.0:5000
 
-
 # Harvest settings
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
diff --git a/docker/ckan/merge.production.ini b/docker/ckan/merge.production.ini
index f4903aa..cc90e60 100644
--- a/docker/ckan/merge.production.ini
+++ b/docker/ckan/merge.production.ini
@@ -34,3 +34,4 @@ ckanext.security.disable_password_reset_override = true
 # Provide a help page to allow 2fa users to contact support or get more information
 # Shows up as 'Need help?' on the 2fa entry form beside the submit button. Does not display a link if none provided
 ckanext.security.mfa_help_link = https://data.govt.nz/catalogue-guide/releasing-data-on-data-govt-nz/how-do-i-set-up-two-factor-authentication/
+

From 4d53da9cc453194c01d87c321c553471cdaac00b Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 21 Jan 2025 15:49:36 -0600
Subject: [PATCH 68/69] Cleanup

---
 docker/ckan/merge.production.ini | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/ckan/merge.production.ini b/docker/ckan/merge.production.ini
index cc90e60..f4903aa 100644
--- a/docker/ckan/merge.production.ini
+++ b/docker/ckan/merge.production.ini
@@ -34,4 +34,3 @@ ckanext.security.disable_password_reset_override = true
 # Provide a help page to allow 2fa users to contact support or get more information
 # Shows up as 'Need help?' on the 2fa entry form beside the submit button. Does not display a link if none provided
 ckanext.security.mfa_help_link = https://data.govt.nz/catalogue-guide/releasing-data-on-data-govt-nz/how-do-i-set-up-two-factor-authentication/
-

From 9aaf51ac36da2d2fa29fe5da788bfa5cc59fb982 Mon Sep 17 00:00:00 2001
From: Ben Bright <ben.bright@twdb.texas.gov>
Date: Tue, 21 Jan 2025 15:52:55 -0600
Subject: [PATCH 69/69] Refactor of print() statement for git command

---
 docker/ckan/plugins/plugins.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker/ckan/plugins/plugins.py b/docker/ckan/plugins/plugins.py
index 0437ada..561476f 100644
--- a/docker/ckan/plugins/plugins.py
+++ b/docker/ckan/plugins/plugins.py
@@ -22,13 +22,13 @@ def install(self):
         git = f"git clone"
         if(self.branch != ""):
             git = f"git clone -b {self.branch}"
-            print(git)
         if(self.tag != ""):
             git = f"git clone -b {self.tag}"
-            print(git)
         
         git = git + f" --depth 1 https://{GH_TOKEN}@github.com/{self.owner}/{self.repo} ./{self.repo}"
 
+        print(git)
+
         cmds = " && ".join(self.cmd)
         
         out = run(f'''