Adjusted verification query for polling trigger (#24)

* fix and check last_modified

* fix condition

* fix partitions

* update dataform sources

* referencing tables

* lint

* format

* descriptive asserts

* simplified trigger query

* explanatory message

definitions/sources/chrome-ux-report.js

@@ -0,0 +1,38 @@
+const database = 'chrome-ux-report'
+
+const pastMonth = constants.fnPastMonth(constants.currentMonth)
+const pastMonthYYYYMM = pastMonth.substring(0, 7).replace('-', '')
+
+declare({
+  database,
+  schema: 'materialized',
+  name: 'country_summary'
+})
+
+assert('country_summary_not_empty').query(ctx => `
+FROM ${ctx.ref(database, 'materialized', 'country_summary')}
+|> WHERE yyyymm = ${pastMonthYYYYMM}
+|> AGGREGATE COUNT(DISTINCT country_code) AS cnt_countries
+|> WHERE cnt_countries != 238
+|> SELECT 'Table data doesn't match 238 countries' AS error_message;
+`)
+
+declare({
+  database,
+  schema: 'materialized',
+  name: 'device_summary'
+})
+
+assert('device_summary_not_empty').query(ctx => `
+FROM ${ctx.ref(database, 'materialized', 'device_summary')}
+|> WHERE date = ''${pastMonth}''
+|> AGGREGATE COUNT(DISTINCT device) AS cnt_devices, COUNT(DISTINCT rank) AS cnt_ranks
+|> WHERE cnt_devices != 3 OR cnt_ranks != 10
+|> SELECT 'Table data doesn't match 3 unique devices and 10 ranks' AS error_message;
+`)
+
+declare({
+  database,
+  schema: 'experimental',
+  name: 'global'
+})

definitions/sources/httparchive.js

@@ -0,0 +1,7 @@
+const stagingTables = ['pages', 'requests', 'parsed_css']
+for (const table of stagingTables) {
+  declare({
+    schema: 'crawl_staging',
+    name: table
+  })
+}

src/index.js

@@ -1,27 +1,30 @@
 const functions = require('@google-cloud/functions-framework')
 
-const currentDate = new Date().toISOString().substring(0, 10)
 const TRIGGERS = {
   cwv_tech_report: {
     type: 'poller',
     query: `
-SELECT LOGICAL_AND(condition)
-FROM (
-  SELECT TOTAL_ROWS > 0 AS condition
-  FROM \`chrome-ux-report.materialized.INFORMATION_SCHEMA.PARTITIONS\`
-  WHERE TABLE_NAME = 'device_summary'
-    AND PARTITION_ID = FORMAT_DATE('%Y%m%d', DATE_SUB(DATE_TRUNC(DATE '${currentDate}', MONTH), INTERVAL 1 MONTH))
-  UNION ALL
-  SELECT TOTAL_ROWS > 0 AS condition
-  FROM \`chrome-ux-report.materialized.INFORMATION_SCHEMA.PARTITIONS\`
-  WHERE TABLE_NAME = 'device_summary'
-    AND PARTITION_ID = FORMAT_DATE('%Y%m%d', DATE_SUB(DATE_TRUNC(DATE '${currentDate}', MONTH), INTERVAL 1 MONTH))
-  UNION ALL
-  SELECT NOT TOTAL_ROWS > 0 AS condition
-  FROM \`httparchive.core_web_vitals.INFORMATION_SCHEMA.PARTITIONS\`
-  WHERE TABLE_NAME = 'technologies'
-    AND PARTITION_ID = FORMAT_DATE('%Y%m%d', DATE_SUB(DATE_TRUNC(DATE '${currentDate}', MONTH), INTERVAL 1 MONTH))
-);
+DECLARE previousMonth STRING DEFAULT FORMAT_DATE('%Y%m%d', DATE_SUB(DATE_TRUNC(CURRENT_DATE(), MONTH), INTERVAL 1 MONTH));
+DECLARE previousMonth_YYYYMM STRING DEFAULT SUBSTR(previousMonth, 1, 6);
+
+WITH crux AS (
+  SELECT
+    LOGICAL_AND(total_rows > 0) AS rows_available,
+    LOGICAL_AND(TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), last_modified_time, HOUR) < 7) AS recent_last_modified
+  FROM chrome-ux-report.materialized.INFORMATION_SCHEMA.PARTITIONS
+  WHERE table_name IN ('device_summary', 'country_summary')
+    AND partition_id IN (previousMonth, previousMonth_YYYYMM)
+), report AS (
+  SELECT TOTAL_ROWS > 0 AS report_exists
+  FROM httparchive.core_web_vitals.INFORMATION_SCHEMA.PARTITIONS
+  WHERE table_name = 'technologies'
+    AND partition_id = previousMonth
+)
+
+SELECT
+  (rows_available AND NOT report_exists)
+    OR (rows_available AND recent_last_modified) AS condition
+FROM crux, report;
     `,
     action: 'runDataformRepo',
     actionArgs: {