Add support for selecting time granularity on MAT (#818)

* Add support for selecting time granularity on MAT

* Fix week granularity scale

* Hour formatting, conditional dropdown options

* Fix missing messages

components/Chart.js

@@ -22,10 +22,12 @@ const Chart = React.memo(function Chart({testName, testGroup = null, title, quer
   }, [queryParams])
 
   const { data, error } = useSWR(
-    testGroup ? { query: apiQuery,
-      testNames: testGroup.tests,
-      groupKey: name
-    } : apiQuery,
+    testGroup ? 
+      { query: apiQuery, 
+        testNames: testGroup.tests, 
+        groupKey: name
+      } : 
+      apiQuery,
     testGroup ? MATMultipleFetcher : MATFetcher,
     swrOptions
   )

components/aggregation/mat/Form.js

@@ -51,6 +51,22 @@ const messages = defineMessages({
     id: 'MAT.Form.Label.AxisOption.probe_asn',
     defaultMessage: ''
   },
+  'hour': {
+    id: 'MAT.Form.TimeGrainOption.hour',
+    defaultMessage: ''
+  },
+  'day': {
+    id: 'MAT.Form.TimeGrainOption.day',
+    defaultMessage: ''
+  },
+  'week': {
+    id: 'MAT.Form.TimeGrainOption.week',
+    defaultMessage: ''
+  },
+  'month': {
+    id: 'MAT.Form.TimeGrainOption.month',
+    defaultMessage: ''
+  }
 })
 
 
@@ -103,7 +119,8 @@ const defaultDefaultValues = {
   since: lastMonthToday,
   until: tomorrow,
   axis_x: 'measurement_start_day',
-  axis_y: ''
+  axis_y: '',
+  time_grain: 'day'
 }
 
 export const Form = ({ onSubmit, testNames, query }) => {
@@ -185,6 +202,26 @@ export const Form = ({ onSubmit, testNames, query }) => {
     if (!yAxisOptionsFiltered.includes(getValues('axis_y'))) setValue('axis_y', '')
   }, [setValue, getValues, yAxisOptionsFiltered])
 
+  const since = watch('since')
+  const until = watch('until')
+  const timeGrainOptions = useMemo(() => {
+    if (!since || !until) return ['hour', 'day', 'week', 'month']
+    const diff = dayjs(until).diff(dayjs(since), 'day')
+    if (diff < 8) {
+      const availableValues = ['hour', 'day']
+      if (!availableValues.includes(getValues('time_grain'))) setValue('time_grain', 'hour')
+      return availableValues
+    } else if (diff >= 8 && diff < 31) {
+      const availableValues = ['day', 'week']
+      if (!availableValues.includes(getValues('time_grain'))) setValue('time_grain', 'day')
+      return availableValues
+    } else if (diff >= 31 ) {
+      const availableValues = ['day', 'week', 'month']
+      if (!availableValues.includes(getValues('time_grain'))) setValue('time_grain', 'day')
+      return availableValues
+    }
+  }, [setValue, getValues, since, until])
+
   return (
     <form onSubmit={handleSubmit(onSubmit)}>
       <ConfirmationModal show={showConfirmation} onConfirm={onConfirm} onCancel={onCancel} />
@@ -265,6 +302,22 @@ export const Form = ({ onSubmit, testNames, query }) => {
             />
           }
         </Box>
+        <Box width={[1, 2/12]} mx={[0, 2]}>
+          <StyledLabel>
+            <FormattedMessage id='MAT.Form.Label.TimeGrain' />
+          </StyledLabel>
+          <Controller
+            name='time_grain'
+            control={control}
+            render={({field}) => (
+              <Select {...field} width={1}>
+                {timeGrainOptions.map((option, idx) => (
+                  <option key={idx} value={option}>{intl.formatMessage(messages[option])}</option>
+                ))}
+              </Select>
+            )}
+          />
+        </Box>
         <Box width={[1, 2/12]} mx={[0, 2]}>
           <StyledLabel>
             <FormattedMessage id='MAT.Form.Label.XAxis' />
@@ -390,5 +443,6 @@ Form.propTypes = {
     input: PropTypes.string,
     probe_cc: PropTypes.string,
     category_code: PropTypes.string,
+    time_grain: PropTypes.string,
   })
 }

components/aggregation/mat/RowChart.js

@@ -28,7 +28,18 @@ const colorFunc = (d) => colorMap[d.id] || '#ccc'
 const barLayers = ['grid', 'axes', 'bars']
 export const chartMargins = { top: 4, right: 50, bottom: 4, left: 0 }
 
-const chartProps1D = {
+const formatXAxisValues = (value, query, intl) => {
+  if (query.axis_x === 'measurement_start_day' && Date.parse(value)) {
+    if (query.time_grain === 'hour') {
+      const dateTime = new Date(value)
+      return new Intl.DateTimeFormat(intl.locale, { dateStyle: 'short', timeStyle: 'short', timeZone: 'UTC', hourCycle: 'h23' }).format(dateTime)
+    }
+  } else {
+    return value
+  }
+}
+
+const chartProps1D = (query, intl) => ({
   colors: colorFunc,
   indexScale: {
     type: 'band',
@@ -49,7 +60,10 @@ const chartProps1D = {
     tickPadding: 5,
     tickRotation: 45,
     legendPosition: 'middle',
-    legendOffset: 60
+    legendOffset: 70,
+    tickValues: getXAxisTicks(query),
+    legend: query.axis_x ? intl.formatMessage({id: `MAT.Form.Label.AxisOption.${query.axis_x}`, defaultMessage: '' }) : '',
+    format: (values) => formatXAxisValues(values, query, intl),
   },
   axisLeft: {
     tickSize: 5,
@@ -64,9 +78,9 @@ const chartProps1D = {
   animate: true,
   motionStiffness: 90,
   motionDamping: 15,
-}
+})
 
-const chartProps2D = {
+const chartProps2D = (query) => ({
   // NOTE: These dimensions are linked to accuracy of the custom axes rendered in
   // <GridChart />
   margin: chartMargins,
@@ -98,7 +112,7 @@ const chartProps2D = {
   animate: false,
   isInteractive: true,
   layers: barLayers,
-}
+})
 
 const RowChart = ({ data, indexBy, label, height, rowIndex /* width, first, last */}) => {
   const intl = useIntl()
@@ -128,9 +142,6 @@ const RowChart = ({ data, indexBy, label, height, rowIndex /* width, first, last
   // react-spring from working on the actual data during
   // first render. This forces an update after 1ms with
   // real data, which appears quick enough with animation disabled
-  // const [chartData, setChartData] = useState([])
-  // useEffect(() => {
-  //   let animation = setTimeout(() => setChartData(data), 1)
   const [chartData, setChartData] = useState([])
   useEffect(() => {
     let animation = setTimeout(() => setChartData(data), 1)
@@ -140,11 +151,9 @@ const RowChart = ({ data, indexBy, label, height, rowIndex /* width, first, last
     }
   }, [data])
 
+
   const chartProps = useMemo(() => {
-    const xAxisTicks = getXAxisTicks(query)
-    chartProps1D.axisBottom.tickValues = xAxisTicks
-    chartProps1D.axisBottom.legend = query.axis_x ? intl.formatMessage({id: `MAT.Form.Label.AxisOption.${query.axis_x}`, defaultMessage: ''}) : ''
-      return label === undefined ? chartProps1D : chartProps2D
+    return label === undefined ? chartProps1D(query, intl) : chartProps2D(query)
   }, [intl, label, query])
 
   return (

components/aggregation/mat/computations.js

@@ -1,14 +1,33 @@
 import { getCategoryCodesMap } from '../../utils/categoryCodes'
 import { getLocalisedRegionName } from 'utils/i18nCountries'
+import dayjs from 'services/dayjs'
 
 const categoryCodesMap = getCategoryCodesMap()
 
-export function getDatesBetween(startDate, endDate) {
+export function getDatesBetween(startDate, endDate, timeGrain) {
   const dateSet = new Set()
   var currentDate = startDate
   while (currentDate < endDate) {
-    dateSet.add(currentDate.toISOString().slice(0, 10))
-    currentDate.setDate(currentDate.getDate() + 1)
+    if (timeGrain === 'hour') {
+      let startOfDay = dayjs(currentDate).utc().startOf('day')
+      const nextDay = startOfDay.add(1, 'day')
+      while (startOfDay.toDate() < nextDay.toDate()) {
+        dateSet.add(startOfDay.toISOString().split('.')[0] + 'Z')
+        startOfDay = startOfDay.utc().add(1, 'hours')
+      }
+      currentDate = dayjs(currentDate).utc().add(1, 'day')
+    } else if (timeGrain === 'month') {
+      const monthStart = dayjs(currentDate).utc().startOf('month')
+      dateSet.add(monthStart.toISOString().slice(0, 10))
+      currentDate = monthStart.add(1, 'month').toDate()
+    } else if (timeGrain === 'week') {
+      const weekStart = dayjs(currentDate).utc().startOf('week')
+      dateSet.add(weekStart.toISOString().slice(0, 10))
+      currentDate = weekStart.add(1, 'week').toDate()
+    } else if (timeGrain === 'day') {
+      dateSet.add(currentDate.toISOString().slice(0, 10))
+      currentDate.setDate(currentDate.getDate() + 1)
+    }
   }
   return dateSet
 }
@@ -20,7 +39,7 @@ export function fillRowHoles (data, query, locale) {
 
   switch(query.axis_x) {
     case 'measurement_start_day':
-      domain = getDatesBetween(new Date(query.since), new Date(query.until))
+      domain = getDatesBetween(new Date(query.since), new Date(query.until), query.time_grain)
       break
     case 'category_code':
       domain = [...getCategoryCodesMap().keys()]

components/aggregation/mat/timeScaleXAxis.js

@@ -1,19 +1,43 @@
 import { scaleUtc } from 'd3-scale'
 
 import { getDatesBetween } from './computations'
+import dayjs from 'services/dayjs'
 
 
 const defaultCount = 20
 
+const getIntervalTicks = (data, count = defaultCount) => {
+  if (!(data && data.length)) return []
+
+  const start = data[0]
+  const end = data[data.length - 1]
+  const intervalType = 'week'
+  const intervalCount = Math.floor(dayjs(end).diff(start, intervalType))
+  return data.reduce((accum, point, index) => {
+      const divisor = Math.ceil(intervalCount / count)
+      if (index % divisor === 0)
+          accum.push(point)
+      return accum
+  }, [])
+}
+
+
 export function getXAxisTicks (query, count = defaultCount) {
 
   if (query.axis_x === 'measurement_start_day') {
-    const dateDomain = [...getDatesBetween(new Date(query.since), new Date(query.until))].map(d => new Date(d))
+    const dateDomain = [...getDatesBetween(new Date(query.since), new Date(query.until), query.time_grain)].map(d => new Date(d))
     const xScale = scaleUtc().domain([dateDomain[0], dateDomain[dateDomain.length-1]])
-
     const xAxisTickValues = dateDomain.length < 30 ? dateDomain : [
       ...xScale.ticks(count),
     ]
+
+    if (query.time_grain === 'hour') {
+      return Array.from(xAxisTickValues).map(d => d.toISOString().split('.')[0] + 'Z')
+    } else if (query.time_grain === 'week') {
+      return dateDomain.length < 30 ? 
+        Array.from(dateDomain).map(d => d.toISOString().split('T')[0]) : 
+        getIntervalTicks(dateDomain.map((d) => d.toISOString().split('T')[0]), count)
+    }
 
     return Array.from(xAxisTickValues).map(d => d.toISOString().split('T')[0])
   }

pages/chart/mat.js

@@ -102,6 +102,7 @@ const MeasurementAggregationToolkit = ({ testNames }) => {
           axis_x: 'measurement_start_day',
           since: monthAgo.format('YYYY-MM-DD'),
           until: today.format('YYYY-MM-DD'),
+          time_grain: 'day',
         },
       }
       router.replace(href, undefined, { shallow: true })
@@ -112,9 +113,7 @@ const MeasurementAggregationToolkit = ({ testNames }) => {
   }, [])
 
   const shouldFetchData = router.pathname !== router.asPath
-  // THIS IS TEMPORARY - in the next iteration users will be
-  // able to set time_grain themselves
-  const query = {...router.query, time_grain: 'day'}
+  const query = {...router.query}
 
   const { data, error, isValidating } = useSWR(
     () => shouldFetchData ? [query] : null,

public/static/lang/en.json

@@ -464,12 +464,17 @@
   "MAT.CSVData": "CSV Data",
   "MAT.Form.Label.XAxis": "Columns",
   "MAT.Form.Label.YAxis": "Rows",
+  "MAT.Form.Label.TimeGrain": "Time Granularity",
   "MAT.Form.Label.AxisOption.domain": "Domain",
   "MAT.Form.Label.AxisOption.input": "Input",
   "MAT.Form.Label.AxisOption.measurement_start_day": "Measurement Day",
   "MAT.Form.Label.AxisOption.probe_cc": "Countries",
   "MAT.Form.Label.AxisOption.category_code": "Website Categories",
   "MAT.Form.Label.AxisOption.probe_asn": "ASN",
+  "MAT.Form.TimeGrainOption.hour": "Hour",
+  "MAT.Form.TimeGrainOption.day": "Day",
+  "MAT.Form.TimeGrainOption.week": "Week",
+  "MAT.Form.TimeGrainOption.month": "Month",
   "MAT.Form.ConfirmationModal.Title": "Are you sure?",
   "MAT.Form.ConfirmationModal.Message": "Duration too long. This can potentially slow down the page",
   "MAT.Form.ConfirmationModal.No": "No",