Skip to content

Commit

Permalink
feat: move page engagement to an mv
Browse files Browse the repository at this point in the history
fix: use macro on int_pages_per_subsection
  • Loading branch information
Ian2012 committed May 27, 2024
1 parent b043d49 commit 668f81c
Show file tree
Hide file tree
Showing 6 changed files with 288 additions and 117 deletions.
2 changes: 1 addition & 1 deletion models/navigation/fact_navigation_completion.sql
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ select
pages.section_with_name as section_with_name,
pages.subsection_with_name as subsection_with_name,
pages.course_order as course_order,
pages.page_count as page_count,
pages.item_count as page_count,
visits.actor_id as actor_id,
visits.block_id as block_id,
users.username as username,
Expand Down
99 changes: 26 additions & 73 deletions models/navigation/fact_pageview_engagement.sql
Original file line number Diff line number Diff line change
@@ -1,97 +1,50 @@
with
subsection_counts as (
subsection_engagement as (
select
org,
course_key,
course_run,
section_with_name,
subsection_with_name,
actor_id,
page_count,
count(distinct block_id) as pages_visited,
case
when pages_visited = 0
then 'No pages viewed yet'
when pages_visited = page_count
then 'All pages viewed'
else 'At least one page viewed'
end as engagement_level
from {{ ref("fact_navigation_completion") }}
group by
org,
course_key,
course_run,
section_with_name,
subsection_with_name,
'subsection' as content_level,
actor_id,
page_count
subsection_block_id as block_id,
engagement_level as section_subsection_page_engagement
from {{ ref("subsection_page_engagement") }}
),
section_counts as (
section_engagement as (
select
org,
course_key,
course_run,
section_with_name,
'' as subsection_with_name,
'section' as content_level,
actor_id,
sum(page_count) as page_count,
sum(pages_visited) as pages_visited,
case
when pages_visited = 0
then 'No pages viewed yet'
when pages_visited = page_count
then 'All pages viewed'
else 'At least one page viewed'
end as engagement_level
from subsection_counts
group by
org,
course_key,
course_run,
section_with_name,
subsection_with_name,
actor_id
),
pageview_counts as (
select
org,
course_key,
course_run,
section_with_name as section_with_name,
subsection_with_name as subsection_with_name,
subsection_with_name as section_subsection_name,
'subsection' as content_level,
actor_id as actor_id,
section_block_id as block_id,
engagement_level as section_subsection_page_engagement
from subsection_counts
from {{ ref("section_page_engagement") }}
),
page_engagement as (
select *
from subsection_engagement
union all
select
org,
course_key,
course_run,
section_with_name as section_with_name,
subsection_with_name as subsection_with_name,
section_with_name as section_subsection_name,
'section' as content_level,
actor_id as actor_id,
engagement_level as section_subsection_page_engagement
from section_counts

select *
from section_engagement
)

select
pv.org as org,
pv.course_key as course_key,
pv.course_run as course_run,
pv.section_with_name as section_with_name,
pv.subsection_with_name as subsection_with_name,
pv.section_subsection_name as section_subsection_name,
course_blocks.course_run as course_run,
course_blocks.display_name_with_location as section_subsection_name,
pv.content_level as content_level,
pv.actor_id as actor_id,
pv.section_subsection_page_engagement as section_subsection_page_engagement,
users.username as username,
users.name as name,
users.email as email
from pageview_counts pv
from page_engagement pv
join
{{ ref("dim_course_blocks") }} course_blocks
on (
pv.org = course_blocks.org
and pv.course_key = course_blocks.course_key
and pv.block_id = course_blocks.block_id
)
left outer join
{{ ref("dim_user_pii") }} users on toUUID(actor_id) = users.external_user_id
{{ ref("dim_user_pii") }} users on toUUID(pv.actor_id) = users.external_user_id
41 changes: 1 addition & 40 deletions models/navigation/int_pages_per_subsection.sql
Original file line number Diff line number Diff line change
@@ -1,40 +1 @@
with
pages_per_subsection as (
select
org,
course_key,
section_number,
subsection_number,
course_order,
count(*) as page_count
from {{ ref("dim_course_blocks") }}
where block_id like '%@vertical+block@%'
group by org, course_key, section_number, subsection_number, course_order
)

select
pps.org as org,
pps.course_key as course_key,
pps.section_number as section_number,
section_blocks.display_name_with_location as section_with_name,
pps.subsection_number as subsection_number,
subsection_blocks.display_name_with_location as subsection_with_name,
pps.course_order as course_order,
pps.page_count as page_count
from pages_per_subsection pps
left join
{{ ref("dim_course_blocks") }} section_blocks
on (
pps.section_number = section_blocks.hierarchy_location
and pps.org = section_blocks.org
and pps.course_key = section_blocks.course_key
and section_blocks.block_id like '%@chapter+block@%'
)
left join
{{ ref("dim_course_blocks") }} subsection_blocks
on (
pps.subsection_number = subsection_blocks.hierarchy_location
and pps.org = subsection_blocks.org
and pps.course_key = subsection_blocks.course_key
and subsection_blocks.block_id like '%@sequential+block@%'
)
select * from ({{ items_per_subsection("%@vertical+block@%") }})
65 changes: 62 additions & 3 deletions models/navigation/schema.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,12 +194,21 @@ models:
- name: subsection_with_name
data_type: string
description: The subsection number and name
- name: page_count
data_type: uint64
description: The number of pages in the associated subsection
- name: course_order
data_type: Int32
description: "The sort order of this block in the course across all course blocks"
- name: graded
data_type: boolean
description: "Whether the unit is graded"
- name: item_count
data_type: uint64
description: The number of pages in the associated subsection
- name: subsection_block_id
data_type: string
description: The unique identifier for the subsection block
- name: section_block_id
data_type: string
description: The unique identifier for the section block

- name: fact_pageview_engagement
description: "A view for analyzing the number of page views per learner per section and subsection"
Expand Down Expand Up @@ -259,3 +268,53 @@ models:
- name: emission_time
data_type: datetime
description: "Timestamp, to the second, of when this event was emitted"

- name: section_page_engagement
description: "A record per course per section per learner with their engagement level"
columns:
- name: org
data_type: string
description: "The organization that the course belongs to"
- name: course_key
data_type: string
description: "The course key for the course"
- name: actor_id
data_type: string
description: "The xAPI actor identifier"
- name: section_block_id
data_type: string
description: "The unique identifier for the section block"
- name: engagement_level
data_type: string
description: "The engagement level of the learner with the section"
tests:
- accepted_values:
values:
- 'No problems attempted yet'
- 'All problems attempted'
- 'At least one problem attempted'

- name: subsection_page_engagement
description: "A record per course per subsection per learner with their engagement level"
columns:
- name: org
data_type: string
description: "The organization that the course belongs to"
- name: course_key
data_type: string
description: "The course key for the course"
- name: actor_id
data_type: string
description: "The xAPI actor identifier"
- name: subsection_block_id
data_type: string
description: "The unique identifier for the subsection block"
- name: engagement_level
data_type: string
description: "The engagement level of the learner with the subsection"
tests:
- accepted_values:
values:
- 'No pages viewed yet'
- 'All pages viewed'
- 'At least one page viewed'
108 changes: 108 additions & 0 deletions models/navigation/section_page_engagement.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
{{
config(
materialized="materialized_view",
schema=env_var("ASPECTS_XAPI_DATABASE", "xapi"),
engine=get_engine("ReplacingMergeTree()"),
primary_key="(org, course_key)",
order_by="(org, course_key, section_block_id, actor_id)",
)
}}

with
fact_navigation as (
select
navigation.emission_time as emission_time,
navigation.org as org,
navigation.course_key as course_key,
navigation.actor_id as actor_id,
navigation.block_id as block_id,
blocks.display_name_with_location as block_name_with_location
from {{ ref("navigation_events") }} navigation
join
{{ ref("dim_course_blocks") }} blocks
on (
navigation.course_key = blocks.course_key
and navigation.block_id = blocks.block_id
)
),
visited_subsection_pages as (
select distinct
date(emission_time) as visited_on,
org,
course_key,
{{ section_from_display("block_name_with_location") }} as section_number,
{{ subsection_from_display("block_name_with_location") }}
as subsection_number,
actor_id,
block_id
from fact_navigation
),
fact_navigation_completion as (
select
visits.visited_on as visited_on,
visits.org as org,
visits.course_key as course_key,
pages.section_with_name as section_with_name,
pages.subsection_with_name as subsection_with_name,
pages.item_count as item_count,
visits.actor_id as actor_id,
visits.block_id as block_id,
pages.section_block_id as section_block_id
from visited_subsection_pages visits
join
{{ ref("int_pages_per_subsection") }} pages
on (
visits.org = pages.org
and visits.course_key = pages.course_key
and visits.section_number = pages.section_number
and visits.subsection_number = pages.subsection_number
)
),
subsection_counts as (
select
org,
course_key,
section_with_name,
subsection_with_name,
actor_id,
item_count,
count(distinct block_id) as pages_visited,
case
when pages_visited = 0
then 'No pages viewed yet'
when pages_visited = item_count
then 'All pages viewed'
else 'At least one page viewed'
end as engagement_level,
section_block_id
from fact_navigation_completion
group by
org,
course_key,
section_with_name,
subsection_with_name,
actor_id,
item_count,
section_block_id
),
section_counts as (
select
org,
course_key,
actor_id,
sum(item_count) as item_count,
sum(pages_visited) as pages_visited,
case
when pages_visited = 0
then 'No pages viewed yet'
when pages_visited = item_count
then 'All pages viewed'
else 'At least one page viewed'
end as engagement_level,
section_block_id
from subsection_counts
group by org, course_key, section_block_id, actor_id
)

select org, course_key, actor_id, section_block_id, engagement_level
from section_counts
Loading

0 comments on commit 668f81c

Please sign in to comment.