Update dbt version and use duckdb adapter

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 executors:
   standard:
     docker:
-      - image: circleci/python:3.8
+      - image: circleci/python:3.10
     working_directory: "~/lib"
 
 # -----------------
@@ -33,7 +33,7 @@ commands:
             - virtualenv-{{ .Environment.CACHE_VERSION }}-
       - run:
           name: Install Python packages
-          command: pipenv install --dev --system
+          command: pipenv sync --dev --system
       - save_cache:
           key: virtualenv-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ checksum "Pipfile.lock" }}
           paths:
@@ -64,7 +64,7 @@ jobs:
     executor: standard
     environment:
       SALT: himalayan
-      DBT_PROFILES_DIR: ~/lib/testing/
+      DBT_PROFILES_DIR: testing
       TESTING_PROFILE_NAME: jaffle_shop
     steps:
       - checkout

.gitignore

@@ -5,7 +5,10 @@ logs/
 
 **/.DS_Store
 .env
+.venv
 
 # Python temp files
 *.pyc
 __pycache__
+
+*.duckdb

Makefile

@@ -1,6 +1,6 @@
 init:
-	pipenv update
-	pipenv run dbt deps
+	pipenv install
+	pipenv run dbt deps --project-dir jaffle_shop
 
 clean-env:
 	:> .env
@@ -11,21 +11,19 @@ env-development-salt:
 env-target:
 	echo DATABRICKS_TARGET=$$(git symbolic-ref --short HEAD | tr /- _) >> .env
 
-package-project:
-	for PACKAGE in dbt_project_evaluator ; do \
-		cp package_projects/$$PACKAGE.yml dbt_packages/$$PACKAGE/dbt_project.yml ; \
-	done
-
-build-env: clean-env env-development-salt env-target package-project
+build-env: clean-env env-development-salt env-target
 
 dbt-deps:
-	pipenv run dbt deps
+	pipenv run dbt deps --project-dir jaffle_shop
 
 dbt-build: build-env
-	pipenv run dbt build --selector jaffle_shop
+	pipenv run dbt build --selector jaffle_shop --project-dir jaffle_shop
 
 run-dbt-project-evaluator: dbt-deps build-env
-	pipenv run dbt --warn-error build --select package:dbt_project_evaluator dbt_project_evaluator_exceptions
+	pipenv run dbt --warn-error build \
+		--select package:dbt_project_evaluator dbt_project_evaluator_exceptions \
+		--select jaffle_shop \
+		--project-dir jaffle_shop
 
 lint: build-env
 	pipenv run sqlfluff lint
@@ -34,7 +32,7 @@ format: build-env
 	pipenv run sqlfluff fix
 
 run-python-tests:
-	pipenv run pytest --quiet --show-capture=no --tb=no
+	pipenv run pytest testing --quiet --show-capture=no --tb=no
 
 run-python-tests-detailed:
-	pipenv run pytest
+	pipenv run pytest testing

Pipfile

@@ -4,14 +4,14 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-dbt-core = "==1.3.0"
-dbt-databricks = {extras = ["odbc"]}
-pytest-mock = "*"
+duckdb = "*"
+dbt-core = "==1.7.17"
+dbt-duckdb = "*"
 glob2 = "*"
 
 [dev-packages]
 pytest = "*"
 pytest-mock = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.10"

jaffle_shop/dbt_project.yml

@@ -0,0 +1,62 @@
+name: 'jaffle_shop'
+
+config-version: 2
+version: '0.1'
+
+profile: 'jaffle_shop'
+
+macro-paths: ["macros"]
+model-paths: ["models"]
+seed-paths: ["seeds"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"
+clean-targets:
+    - "target"
+    - "dbt_modules"
+    - "logs"
+
+require-dbt-version: [">=1.0.0", "<2.0.0"]
+
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['dbt_project_evaluator', 'spark_utils', 'dbt_utils']
+  # - macro_namespace: dbt
+  #   search_order: ['dbt_project_evaluator', 'dbt']
+
+models:
+  +file_format: delta
+
+  jaffle_shop:
+    +pk_required: true
+    +desc_required: true
+    +no_hardcoded_dbs: true
+
+    staging:
+      +materialized: view
+
+      src_seed:
+        +materialized: view
+
+    warehouse:
+      +materialized: table
+
+    final:
+      +materialized: table
+
+seeds:
+  dbt_project_evaluator:
+    dbt_project_evaluator_exceptions:
+      +enabled: false
+
+vars:
+  dbt_project_evaluator:
+    model_types: ['staging', 'intermediate', 'warehouse', 'final', 'metrics']
+    staging_folder_name: 'staging'
+    staging_prefixes: ['stg_']
+    intermediate_folder_name: 'intermediate'
+    intermediate_prefixes: ['int_', 'inter_']
+    warehouse_folder_name: 'warehouse'
+    warehouse_prefixes: ['wh_']
+    final_folder_name: 'final'
+    final_prefixes: ['fnl_']

jaffle_shop/macros/sensitive/hash_of_column.sql

@@ -1,13 +1,12 @@
 {% macro hash_of_column(column) %}
 
-    SHA2(
+    SHA256(
         TRIM(
             LOWER(
                 CAST({{ column|lower }} AS {{ dbt_utils.type_string() }})
                 || '{{ jaffle_shop.get_salt(column|lower) }}'
             )
-        ),
-        256
+        )
     ) AS {{column|lower}}_hash,
 
 {% endmacro %}

jaffle_shop/package-lock.yml

@@ -0,0 +1,10 @@
+packages:
+- package: dbt-labs/dbt_utils
+  version: 1.3.0
+- package: dbt-labs/spark_utils
+  version: 0.3.0
+- package: dbt-labs/codegen
+  version: 0.12.1
+- package: dbt-labs/dbt_project_evaluator
+  version: 0.14.3
+sha1_hash: 59adf0a667e307048209cbea5ef70e871d6ec49a

packages.yml → jaffle_shop/packages.yml

@@ -1,9 +1,9 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: 0.8.6
+    version: '>=0.8.6'
   - package: dbt-labs/spark_utils
-    version: 0.3.0
+    version: '>=0.3.0'
   - package: dbt-labs/codegen
-    version: 0.7.0
+    version: '>=0.7.0'
   - package: dbt-labs/dbt_project_evaluator
-    version: 0.1.3
+    version: '>=0.1.3'

jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv

@@ -0,0 +1,2 @@
+fct_name,column_name,id_to_exclude,comment
+fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.

testing/profiles.yml

@@ -2,9 +2,6 @@ jaffle_shop:
   target: jaffle_shop
   outputs:
     jaffle_shop:
-      type: databricks
-      host: "{{ env_var('DATABRICKS_HOST') }}"
-      token: "{{ env_var('DATABRICKS_TOKEN') }}"
-      schema: "{{ env_var('DATABRICKS_TARGET') }}"
-      http_path: "/sql/1.0/endpoints/{{ env_var('DATABRICKS_ENDPOINT') }}"
-      threads: 300
+      type: duckdb
+      path: dbt.duckdb
+      threads: 10