Merge pull request #12 from techindicium/main

merge main in branch
techindicium · Apr 11, 2024 · ea48925 · ea48925
2 parents cce0d93 + 5ecad88
commit ea48925
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 3 deletions.
diff --git a/models/marts/fct_databricks_analytics_billing.sql b/models/marts/fct_databricks_analytics_billing.sql
@@ -67,5 +67,31 @@ with
         left join dim_dates on stg_billing.end_date = dim_dates.date_day
     )
 
+    , dedup as (
+        select
+            cluster_fk
+            , sku_fk
+            , workspace_id
+            , cluster_id
+            , owner_id
+            , cluster_name
+            , date_day
+            , end_time
+            , node_type
+            , custom_tags
+            , sku
+            , dbus
+            , dbus_unit_price
+            , total_price
+            , machine_hours
+            , extracted_at
+            , row_number() over (
+                partition by date_day, end_time, node_type 
+                order by extracted_at desc
+              ) as row_num
+        from joined
+        qualify row_num = 1
+    )
+
 select *
-from joined
+from dedup
diff --git a/models/staging/stg_databricks_analytics_billing.sql b/models/staging/stg_databricks_analytics_billing.sql
@@ -12,7 +12,7 @@ with
             , cast(sku as string) as sku
             , round(dbus,2) as dbus
             , round(machinehours,2) as machine_hours
-            , '_SDC_EXTRACTED_AT' as extracted_at
+            , _SDC_EXTRACTED_AT as extracted_at
         from {{ source('raw_databricks', 'billing') }}
     )
 
@@ -41,7 +41,7 @@ with
         from renamed
     )
 
-        , calculate as (
+    , calculate as (
         select
             workspace_id
             , cluster_id