From 715d647ef553af18731faf9668eb92771fb5d0fa Mon Sep 17 00:00:00 2001 From: Will Carpenter <65402309+WillC3@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:11:23 +0100 Subject: [PATCH] Fix repo --- jaffle_shop/models/final/finance/_models.yml | 23 +++++++++++++++++++ .../fnl_finance_orders_per_customer.sql | 4 ++++ jaffle_shop/models/final/sales/_models.yml | 23 +++++++++++++++++++ .../final/sales/fnl_sales_newcustomers.sql | 6 +++++ .../{schema.yml => src_seed/_models.yml} | 18 +++++++++++++++ .../models/staging/src_seed/stg_customers.sql | 4 ++++ .../stg_customers_pii.sql} | 0 .../staging/{ => src_seed}/stg_orders.sql | 0 .../staging/{ => src_seed}/stg_payments.sql | 0 .../models/{docs.md => warehouse/_docs.md} | 0 .../{schema.yml => warehouse/_models.yml} | 20 +++++++--------- .../wh_customers.sql} | 1 - .../{orders.sql => warehouse/wh_orders.sql} | 0 .../dbt_project_evaluator_exceptions.csv | 2 ++ 14 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 jaffle_shop/models/final/finance/_models.yml create mode 100644 jaffle_shop/models/final/finance/fnl_finance_orders_per_customer.sql create mode 100644 jaffle_shop/models/final/sales/_models.yml create mode 100644 jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql rename jaffle_shop/models/staging/{schema.yml => src_seed/_models.yml} (60%) create mode 100644 jaffle_shop/models/staging/src_seed/stg_customers.sql rename jaffle_shop/models/staging/{stg_customers.sql => src_seed/stg_customers_pii.sql} (100%) rename jaffle_shop/models/staging/{ => src_seed}/stg_orders.sql (100%) rename jaffle_shop/models/staging/{ => src_seed}/stg_payments.sql (100%) rename jaffle_shop/models/{docs.md => warehouse/_docs.md} (100%) rename jaffle_shop/models/{schema.yml => warehouse/_models.yml} (87%) rename jaffle_shop/models/{customers.sql => warehouse/wh_customers.sql} (99%) rename jaffle_shop/models/{orders.sql => warehouse/wh_orders.sql} (100%) create mode 100644 jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv diff --git a/jaffle_shop/models/final/finance/_models.yml b/jaffle_shop/models/final/finance/_models.yml new file mode 100644 index 000000000..03bd02972 --- /dev/null +++ b/jaffle_shop/models/final/finance/_models.yml @@ -0,0 +1,23 @@ +version: 2 + +exposures: + - name: fnl_finance_orders_per_customer + description: Used in Inksacio app + type: application + url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/ + owner: + email: will.carpenter@octoenergy.com + depends_on: + - ref('wh_customers') + +models: + - name: fnl_finance_orders_per_customer + description: This table contains the total value of orders per customer + columns: + - name: customer_id + description: This is a unique identifier for a customer + tests: + - unique + - not_null + - name: customer_lifetime_value + description: This is the total value of orders for a customer diff --git a/jaffle_shop/models/final/finance/fnl_finance_orders_per_customer.sql b/jaffle_shop/models/final/finance/fnl_finance_orders_per_customer.sql new file mode 100644 index 000000000..38d3eec04 --- /dev/null +++ b/jaffle_shop/models/final/finance/fnl_finance_orders_per_customer.sql @@ -0,0 +1,4 @@ +SELECT + customers.customer_id + , customers.customer_lifetime_value +FROM {{ ref('wh_customers') }} AS customers diff --git a/jaffle_shop/models/final/sales/_models.yml b/jaffle_shop/models/final/sales/_models.yml new file mode 100644 index 000000000..35603a646 --- /dev/null +++ b/jaffle_shop/models/final/sales/_models.yml @@ -0,0 +1,23 @@ +version: 2 + +exposures: + - name: fnl_sales_newcustomers + description: Used in Inksacio app + type: application + url: https://inksacio.eks.octopus.engineering/my_certification_dashboard/ + owner: + email: will.carpenter@octoenergy.com + depends_on: + - ref('wh_customers') + +models: + - name: fnl_sales_newcustomers + description: This table contains the number of new customers per month + columns: + - name: month_start_date + description: This is a date representing the start of a month + tests: + - unique + - not_null + - name: new_customer_count + description: This is the number of new customers in a given month diff --git a/jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql b/jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql new file mode 100644 index 000000000..8c706010f --- /dev/null +++ b/jaffle_shop/models/final/sales/fnl_sales_newcustomers.sql @@ -0,0 +1,6 @@ +SELECT + DATE_TRUNC('MONTH', customers.first_order) AS month_start_date + , COUNT(customers.customer_id) AS new_customer_count +FROM {{ ref('wh_customers') }} AS customers +GROUP BY DATE_TRUNC('MONTH', customers.first_order) +ORDER BY DATE_TRUNC('MONTH', customers.first_order) DESC diff --git a/jaffle_shop/models/staging/schema.yml b/jaffle_shop/models/staging/src_seed/_models.yml similarity index 60% rename from jaffle_shop/models/staging/schema.yml rename to jaffle_shop/models/staging/src_seed/_models.yml index c207e4cf5..c30078ff1 100644 --- a/jaffle_shop/models/staging/schema.yml +++ b/jaffle_shop/models/staging/src_seed/_models.yml @@ -1,12 +1,30 @@ version: 2 models: + - name: stg_customers_pii + columns: + - name: customer_id + tests: + - unique + - not_null + - name: first_name + meta: + sensitive: true + - name: last_name + meta: + sensitive: true - name: stg_customers columns: - name: customer_id tests: - unique - not_null + - name: first_name_hash + tests: + - dbt_expectations.expect_column_to_exist + - name: last_name_hash + tests: + - dbt_expectations.expect_column_to_exist - name: stg_orders columns: diff --git a/jaffle_shop/models/staging/src_seed/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers.sql new file mode 100644 index 000000000..e02720807 --- /dev/null +++ b/jaffle_shop/models/staging/src_seed/stg_customers.sql @@ -0,0 +1,4 @@ +SELECT + {{ hash_sensitive_columns('stg_customers_pii') }} +FROM {{ ref('stg_customers_pii') }} + diff --git a/jaffle_shop/models/staging/stg_customers.sql b/jaffle_shop/models/staging/src_seed/stg_customers_pii.sql similarity index 100% rename from jaffle_shop/models/staging/stg_customers.sql rename to jaffle_shop/models/staging/src_seed/stg_customers_pii.sql diff --git a/jaffle_shop/models/staging/stg_orders.sql b/jaffle_shop/models/staging/src_seed/stg_orders.sql similarity index 100% rename from jaffle_shop/models/staging/stg_orders.sql rename to jaffle_shop/models/staging/src_seed/stg_orders.sql diff --git a/jaffle_shop/models/staging/stg_payments.sql b/jaffle_shop/models/staging/src_seed/stg_payments.sql similarity index 100% rename from jaffle_shop/models/staging/stg_payments.sql rename to jaffle_shop/models/staging/src_seed/stg_payments.sql diff --git a/jaffle_shop/models/docs.md b/jaffle_shop/models/warehouse/_docs.md similarity index 100% rename from jaffle_shop/models/docs.md rename to jaffle_shop/models/warehouse/_docs.md diff --git a/jaffle_shop/models/schema.yml b/jaffle_shop/models/warehouse/_models.yml similarity index 87% rename from jaffle_shop/models/schema.yml rename to jaffle_shop/models/warehouse/_models.yml index 381349cfd..6d1187969 100644 --- a/jaffle_shop/models/schema.yml +++ b/jaffle_shop/models/warehouse/_models.yml @@ -1,9 +1,8 @@ version: 2 models: - - name: customers - description: This table has basic information about a customer, as well as some derived facts based on a customer's orders - + - name: wh_customers + description: This table has basic information about a customer, as well as some derived facts based on a customer's orders. columns: - name: customer_id description: This is a unique identifier for a customer @@ -11,15 +10,12 @@ models: - unique - not_null - - name: first_name - description: Customer's first name. PII. - - - name: last_name - description: Customer's last name. PII. - + - name: first_name_hash + description: Customer's first name. + - name: last_name_hash + description: Customer's last name. - name: first_order description: Date (UTC) of a customer's first order - - name: most_recent_order description: Date (UTC) of a customer's most recent order @@ -29,7 +25,7 @@ models: - name: total_order_amount description: Total value (AUD) of a customer's orders - - name: orders + - name: wh_orders description: This table has basic information about orders, as well as some derived facts based on payments columns: @@ -44,7 +40,7 @@ models: tests: - not_null - relationships: - to: ref('customers') + to: ref('stg_customers') field: customer_id - name: order_date diff --git a/jaffle_shop/models/customers.sql b/jaffle_shop/models/warehouse/wh_customers.sql similarity index 99% rename from jaffle_shop/models/customers.sql rename to jaffle_shop/models/warehouse/wh_customers.sql index 016a004fe..99000f9db 100644 --- a/jaffle_shop/models/customers.sql +++ b/jaffle_shop/models/warehouse/wh_customers.sql @@ -20,7 +20,6 @@ customer_orders as ( select customer_id, - min(order_date) as first_order, max(order_date) as most_recent_order, count(order_id) as number_of_orders diff --git a/jaffle_shop/models/orders.sql b/jaffle_shop/models/warehouse/wh_orders.sql similarity index 100% rename from jaffle_shop/models/orders.sql rename to jaffle_shop/models/warehouse/wh_orders.sql diff --git a/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv new file mode 100644 index 000000000..fa604d678 --- /dev/null +++ b/jaffle_shop/seeds/dbt_project_evaluator_exceptions.csv @@ -0,0 +1,2 @@ +fct_name,column_name,id_to_exclude,comment +fct_staging_dependent_on_staging,parent,stg_customers_pii,Scrubbing pii permitted in staging layer.