Merge pull request #13 from techindicium/main

merge main in branch to tests

integration_tests/.gitignore

@@ -1 +1,5 @@
+venv/
+target/
+dbt_packages/
+logs/
 .env

integration_tests/requirements.txt

@@ -1,2 +1,3 @@
 dbt-snowflake>=1.3.0,<2.0.0
-dbt-databricks==1.7.1
+dbt-databricks==1.6.0
+databricks-sql-connector==2.7.0

models/calendar/dim_dates.sql

@@ -14,11 +14,10 @@ with
         select
             cast(date_day as date) as date_day
             , dayofweek(date_day) as day_of_week
-            , extract(month from date_day) as month
-            , extract(quarter from date_day) as quarter
-            , extract(dayofyear from date_day) as day_of_year
-            , extract(year from date_day) as year
-            , to_char(date_day, 'DD-MM') AS month_day
+            , month(date_day) as month
+            , quarter(date_day) as quarter
+            , dayofyear(date_day) as day_of_year
+            , year(date_day) as year
         from dates_raw
     )
 
@@ -27,13 +26,13 @@ with
         select
             *
             , case
-                when day_of_week = 1 then 'Segunda-feira'
-                when day_of_week = 2 then 'Terça-feira'
-                when day_of_week = 3 then 'Quarta-feira'
-                when day_of_week = 4 then 'Quinta-feira'
-                when day_of_week = 5 then 'Sexta-feira'
-                when day_of_week = 6 then 'Sábado'
-                when day_of_week = 0 then 'Domingo'
+                when day_of_week = 1 then 'Domingo'
+                when day_of_week = 2 then 'Segunda-feira'
+                when day_of_week = 3 then 'Terça-feira'
+                when day_of_week = 4 then 'Quarta-feira'
+                when day_of_week = 5 then 'Quinta-feira'
+                when day_of_week = 6 then 'Sexta-feira'
+                when day_of_week = 7 then 'Sábado'
             end as nome_do_dia
             , case
                 when month = 1 then 'Janeiro'
@@ -78,13 +77,13 @@ with
                 else '2º Semestre'
             end as nome_semestre
             , case
-                when day_of_week = 1 then 'Monday'
-                when day_of_week = 2 then 'Tuesday'
-                when day_of_week = 3 then 'Wednesday'
-                when day_of_week = 4 then 'Thursday'
-                when day_of_week = 5 then 'Friday'
-                when day_of_week = 6 then 'Saturday'
-                when day_of_week = 0 then 'Sunday'
+                when day_of_week = 2 then 'Monday'
+                when day_of_week = 3 then 'Tuesday'
+                when day_of_week = 4 then 'Wednesday'
+                when day_of_week = 5 then 'Thursday'
+                when day_of_week = 6 then 'Friday'
+                when day_of_week = 7 then 'Saturday'
+                when day_of_week = 1 then 'Sunday'
             end as name_of_the_day
             , case
                 when month = 1 then 'January'
@@ -131,37 +130,6 @@ with
         from days_info
     )
 
-    , flags_cte as (
-        /*flags de feriado e dia util*/
-        select
-            *
-            , case
-                when month_day = '01-01' then true
-                when month_day = '21-04' then true
-                when month_day = '01-05' then true
-                when month_day = '07-09' then true
-                when month_day = '12-10' then true
-                when month_day = '02-11' then true
-                when month_day = '15-11' then true
-                when month_day = '25-12' then true
-                else false
-            end as fl_holiday
-            , case
-                when day_of_week in(6, 0) then false
-                when month_day = '01-01' then false
-                when month_day = '21-04' then false
-                when month_day = '01-05' then false
-                when month_day = '07-09' then false
-                when month_day = '12-10' then false
-                when month_day = '02-11' then false
-                when month_day = '15-11' then false
-                when month_day = '25-12' then false
-                else true
-            end as fl_business_day
-            , coalesce(day_of_week in(6, 0), false) as fl_weekend
-        from days_named
-    )
-
 /* reorganizando as colunas */
     , final_cte as (
         select
@@ -181,12 +149,9 @@ with
             , semester
             , nome_semestre
             , semester_name
-            , fl_holiday
-            , fl_business_day
-            , fl_weekend
             , day_of_year
             , year
-        from flags_cte
+        from days_named
     )
 
 select *

models/marts/fct_databricks_analytics_billing.sql

@@ -68,6 +68,17 @@ with
     )
 
     , dedup as (
+        select
+            *
+            , row_number() over (
+                partition by date_day, end_time, node_type 
+                order by extracted_at desc
+              ) as row_num
+        from joined
+        qualify row_num = 1
+    )
+
+    , deduplicated as (
         select
             cluster_fk
             , sku_fk
@@ -85,13 +96,9 @@ with
             , total_price
             , machine_hours
             , extracted_at
-            , row_number() over (
-                partition by date_day, end_time, node_type 
-                order by extracted_at desc
-              ) as row_num
-        from joined
-        qualify row_num = 1
+        from dedup
+        where row_num = 1
     )
 
 select *
-from dedup
+from deduplicated

models/staging/stg_databricks_analytics_billing.sql

@@ -6,7 +6,7 @@ with
             , clusterowneruserid owner_id
             , cast(clustername as string) as cluster_name
             , cast(timestamp as date) as end_date
-            , cast(timestamp as time) as end_time
+            , substring(timestamp, 12, 8) as end_time
             , cast(clusternodetype as string) as node_type
             , cast(clustercustomtags as string) as custom_tags
             , cast(sku as string) as sku