From 93c2cf4fc3d8f88ad8ed94b966b358638a74570c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= Date: Mon, 27 Jan 2025 16:42:54 +0100 Subject: [PATCH 1/2] Add data download handling for multi-dims --- functions/_common/downloadFunctions.ts | 24 +++++++++++++++-------- functions/_common/grapherRenderer.ts | 7 ++++++- functions/_common/grapherTools.ts | 23 ++++++++++++++++++---- functions/_common/metadataTools.ts | 5 +++-- functions/_common/readmeTools.ts | 22 ++++++++++++--------- functions/_common/urlTools.ts | 8 ++++++-- site/multiDim/MultiDimDataPageContent.tsx | 13 ++++++++---- 7 files changed, 72 insertions(+), 30 deletions(-) diff --git a/functions/_common/downloadFunctions.ts b/functions/_common/downloadFunctions.ts index 67d33b80984..6d877c33212 100644 --- a/functions/_common/downloadFunctions.ts +++ b/functions/_common/downloadFunctions.ts @@ -14,7 +14,7 @@ export async function fetchMetadataForGrapher( searchParams?: URLSearchParams ) { console.log("Initializing grapher") - const grapher = await initGrapher( + const { grapher, multiDimDimensions } = await initGrapher( identifier, TWITTER_OPTIONS, searchParams ?? new URLSearchParams(""), @@ -25,7 +25,8 @@ export async function fetchMetadataForGrapher( const fullMetadata = assembleMetadata( grapher, - searchParams ?? new URLSearchParams("") + searchParams ?? new URLSearchParams(""), + multiDimDimensions ) return Response.json(fullMetadata) @@ -37,7 +38,7 @@ export async function fetchZipForGrapher( searchParams?: URLSearchParams ) { console.log("preparing to generate zip file") - const grapher = await initGrapher( + const { grapher } = await initGrapher( identifier, TWITTER_OPTIONS, searchParams ?? new URLSearchParams(""), @@ -82,7 +83,7 @@ export async function fetchCsvForGrapher( env: Env, searchParams?: URLSearchParams ) { - const grapher = await initGrapher( + const { grapher } = await initGrapher( identifier, TWITTER_OPTIONS, searchParams ?? new URLSearchParams(""), @@ -118,7 +119,7 @@ export async function fetchReadmeForGrapher( searchParams?: URLSearchParams ) { console.log("Initializing grapher") - const grapher = await initGrapher( + const { grapher, multiDimDimensions } = await initGrapher( identifier, TWITTER_OPTIONS, searchParams ?? new URLSearchParams(""), @@ -127,17 +128,24 @@ export async function fetchReadmeForGrapher( await grapher.downloadLegacyDataFromOwidVariableIds() - const readme = assembleReadme(grapher, searchParams) + const readme = assembleReadme(grapher, searchParams, multiDimDimensions) return new Response(readme, { headers: { "Content-Type": "text/markdown; charset=utf-8", }, }) } + function assembleReadme( grapher: Grapher, - searchParams: URLSearchParams + searchParams: URLSearchParams, + multiDimDimensions?: string[] ): string { const metadataCols = getColumnsForMetadata(grapher) - return constructReadme(grapher, metadataCols, searchParams) + return constructReadme( + grapher, + metadataCols, + searchParams, + multiDimDimensions + ) } diff --git a/functions/_common/grapherRenderer.ts b/functions/_common/grapherRenderer.ts index 7dd979a085a..74f1ec4738e 100644 --- a/functions/_common/grapherRenderer.ts +++ b/functions/_common/grapherRenderer.ts @@ -62,7 +62,12 @@ async function fetchAndRenderGrapherToSvg( env: Env ) { const grapherLogger = new TimeLogger("grapher") - const grapher = await initGrapher(identifier, options, searchParams, env) + const { grapher } = await initGrapher( + identifier, + options, + searchParams, + env + ) grapherLogger.log("initGrapher") const promises = [] diff --git a/functions/_common/grapherTools.ts b/functions/_common/grapherTools.ts index 7e5b3160176..9313afd2ec1 100644 --- a/functions/_common/grapherTools.ts +++ b/functions/_common/grapherTools.ts @@ -16,6 +16,7 @@ import { ImageOptions } from "./imageOptions.js" interface FetchGrapherConfigResult { grapherConfig: GrapherInterface | null + multiDimDimensions?: string[] status: number etag: string | undefined } @@ -139,28 +140,39 @@ export async function fetchGrapherConfig({ const config = await fetchResponse.json() let grapherConfig: GrapherInterface + let multiDimDimensions: string[] if (identifier.type === "multi-dim-slug") { + const multiDimConfig = config as MultiDimDataPageConfigEnriched grapherConfig = await fetchMultiDimGrapherConfig( - config as MultiDimDataPageConfigEnriched, + multiDimConfig, searchParams, env ) + multiDimDimensions = multiDimConfig.dimensions.map((dim) => dim.slug) } else { grapherConfig = config } console.log("grapher title", grapherConfig.title) - return { + const result: FetchGrapherConfigResult = { grapherConfig, status: 200, etag: fetchResponse.headers.get("etag"), } + if (identifier.type === "multi-dim-slug") { + result.multiDimDimensions = multiDimDimensions + } + return result } + export async function initGrapher( identifier: GrapherIdentifier, options: ImageOptions, searchParams: URLSearchParams, env: Env -): Promise { +): Promise<{ + grapher: Grapher + multiDimDimensions?: string[] +}> { let grapherConfigResponse: FetchGrapherConfigResult try { grapherConfigResponse = await fetchGrapherConfig({ @@ -211,7 +223,10 @@ export async function initGrapher( grapher.isExportingToSvgOrPng = true grapher.shouldIncludeDetailsInStaticExport = options.details - return grapher + return { + grapher, + multiDimDimensions: grapherConfigResponse.multiDimDimensions, + } } /** diff --git a/functions/_common/metadataTools.ts b/functions/_common/metadataTools.ts index 83cf0bb02ce..c6f735b839b 100644 --- a/functions/_common/metadataTools.ts +++ b/functions/_common/metadataTools.ts @@ -55,7 +55,8 @@ export const getColumnsForMetadata = (grapher: Grapher) => { } export function assembleMetadata( grapher: Grapher, - searchParams: URLSearchParams + searchParams: URLSearchParams, + multiDimDimensions?: string[] ) { const useShortNames = searchParams.get("useColumnShortNames") === "true" @@ -192,7 +193,7 @@ export function assembleMetadata( dateDownloaded: dateDownloaded.toISOString().split("T")[0], // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be // inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters - activeFilters: getGrapherFilters(searchParams), + activeFilters: getGrapherFilters(searchParams, multiDimDimensions), } return fullMetadata diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index e2620b21d1a..d7a7e4773b9 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -230,10 +230,13 @@ function* columnReadmeText(col: CoreColumn) { yield "" } -function* activeFilterSettings(searchParams: URLSearchParams) { +function* activeFilterSettings( + searchParams: URLSearchParams, + multiDimDimensions?: string[] +) { // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be - // inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters - const filterSettings = getGrapherFilters(searchParams) + // inlucded here, add them to the whitelist inside getGrapherFilters + const filterSettings = getGrapherFilters(searchParams, multiDimDimensions) if (filterSettings) { yield "" yield `### Active Filters` @@ -251,7 +254,8 @@ function* activeFilterSettings(searchParams: URLSearchParams) { export function constructReadme( grapher: Grapher, columns: CoreColumn[], - searchParams: URLSearchParams + searchParams: URLSearchParams, + multiDimDimensions?: string[] ): string { const isSingleColumn = columns.length === 1 // Some computed columns have neither a source nor origins - filter these away @@ -265,10 +269,10 @@ export function constructReadme( const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024" if (isSingleColumn) - readme = `# ${grapher.title} - Data package + readme = `# ${grapher.displayTitle} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${urlWithFilters}) on the Our World in Data website. It was downloaded on ${downloadDate}. -${[...activeFilterSettings(searchParams)].join("\n")} +This data package contains the data that powers the chart ["${grapher.displayTitle}"](${urlWithFilters}) on the Our World in Data website. It was downloaded on ${downloadDate}. +${[...activeFilterSettings(searchParams, multiDimDimensions)].join("\n")} ## CSV Structure The high level structure of the CSV file is that each row is an observation for an entity (usually a country or region) and a timepoint (usually a year). @@ -293,9 +297,9 @@ ${sources.join("\n")} ` else - readme = `# ${grapher.title} - Data package + readme = `# ${grapher.displayTitle} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${urlWithFilters}) on the Our World in Data website. +This data package contains the data that powers the chart ["${grapher.displayTitle}"](${urlWithFilters}) on the Our World in Data website. ## CSV Structure diff --git a/functions/_common/urlTools.ts b/functions/_common/urlTools.ts index 0fc6419ea2b..952d0a63dab 100644 --- a/functions/_common/urlTools.ts +++ b/functions/_common/urlTools.ts @@ -2,7 +2,8 @@ import { pick } from "@ourworldindata/utils" import { GRAPHER_QUERY_PARAM_KEYS } from "@ourworldindata/types" export function getGrapherFilters( - searchParams: URLSearchParams + searchParams: URLSearchParams, + multiDimDimensions?: string[] ): Record | undefined { const params = searchParams.size ? Object.fromEntries(searchParams) @@ -12,5 +13,8 @@ export function getGrapherFilters( delete params.v1 delete params.csvType delete params.useColumnShortNames - return pick(params, GRAPHER_QUERY_PARAM_KEYS) + return pick(params, [ + ...GRAPHER_QUERY_PARAM_KEYS, + ...(multiDimDimensions ?? []), + ]) } diff --git a/site/multiDim/MultiDimDataPageContent.tsx b/site/multiDim/MultiDimDataPageContent.tsx index 8d202ecd34e..e5cd817b594 100644 --- a/site/multiDim/MultiDimDataPageContent.tsx +++ b/site/multiDim/MultiDimDataPageContent.tsx @@ -26,7 +26,11 @@ import { fetchWithRetry, } from "@ourworldindata/utils" import cx from "classnames" -import { ADMIN_BASE_URL, DATA_API_URL } from "../../settings/clientSettings.js" +import { + ADMIN_BASE_URL, + BAKED_GRAPHER_URL, + DATA_API_URL, +} from "../../settings/clientSettings.js" import { DataPageRelatedResearch, FaqEntryKeyedByGdocIdAndFragmentId, @@ -295,6 +299,9 @@ export const MultiDimDataPageContent = ({ const grapherConfigComputed = useMemo(() => { const baseConfig: GrapherProgrammaticInterface = { bounds, + bakedGrapherURL: BAKED_GRAPHER_URL, + adminBaseUrl: ADMIN_BASE_URL, + dataApiUrl: DATA_API_URL, manager: {}, // Don't resize while data is loading. } @@ -307,12 +314,11 @@ export const MultiDimDataPageContent = ({ return { ...varGrapherConfig, ...baseConfig, - dataApiUrl: DATA_API_URL, manager: { canonicalUrl, editUrl, }, - } as GrapherProgrammaticInterface + } }, [ varGrapherConfig, grapherConfigIsReady, @@ -399,7 +405,6 @@ export const MultiDimDataPageContent = ({ {...grapherConfigComputed} queryStr={queryStr} getGrapherInstance={setGrapherInst} - adminBaseUrl={ADMIN_BASE_URL} /> From 41f7052e597508d8205252c5ff556a9b32a4a47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= Date: Tue, 28 Jan 2025 12:15:55 +0100 Subject: [PATCH 2/2] Rename multiDimDimensions to multiDimAvailableDimensions --- functions/_common/downloadFunctions.ts | 16 ++++++++++------ functions/_common/grapherTools.ts | 15 +++++++++------ functions/_common/metadataTools.ts | 7 +++++-- functions/_common/readmeTools.ts | 11 +++++++---- functions/_common/urlTools.ts | 4 ++-- 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/functions/_common/downloadFunctions.ts b/functions/_common/downloadFunctions.ts index 6d877c33212..0edb5f36f84 100644 --- a/functions/_common/downloadFunctions.ts +++ b/functions/_common/downloadFunctions.ts @@ -14,7 +14,7 @@ export async function fetchMetadataForGrapher( searchParams?: URLSearchParams ) { console.log("Initializing grapher") - const { grapher, multiDimDimensions } = await initGrapher( + const { grapher, multiDimAvailableDimensions } = await initGrapher( identifier, TWITTER_OPTIONS, searchParams ?? new URLSearchParams(""), @@ -26,7 +26,7 @@ export async function fetchMetadataForGrapher( const fullMetadata = assembleMetadata( grapher, searchParams ?? new URLSearchParams(""), - multiDimDimensions + multiDimAvailableDimensions ) return Response.json(fullMetadata) @@ -119,7 +119,7 @@ export async function fetchReadmeForGrapher( searchParams?: URLSearchParams ) { console.log("Initializing grapher") - const { grapher, multiDimDimensions } = await initGrapher( + const { grapher, multiDimAvailableDimensions } = await initGrapher( identifier, TWITTER_OPTIONS, searchParams ?? new URLSearchParams(""), @@ -128,7 +128,11 @@ export async function fetchReadmeForGrapher( await grapher.downloadLegacyDataFromOwidVariableIds() - const readme = assembleReadme(grapher, searchParams, multiDimDimensions) + const readme = assembleReadme( + grapher, + searchParams, + multiDimAvailableDimensions + ) return new Response(readme, { headers: { "Content-Type": "text/markdown; charset=utf-8", @@ -139,13 +143,13 @@ export async function fetchReadmeForGrapher( function assembleReadme( grapher: Grapher, searchParams: URLSearchParams, - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] ): string { const metadataCols = getColumnsForMetadata(grapher) return constructReadme( grapher, metadataCols, searchParams, - multiDimDimensions + multiDimAvailableDimensions ) } diff --git a/functions/_common/grapherTools.ts b/functions/_common/grapherTools.ts index 9313afd2ec1..4f2553a0945 100644 --- a/functions/_common/grapherTools.ts +++ b/functions/_common/grapherTools.ts @@ -16,7 +16,7 @@ import { ImageOptions } from "./imageOptions.js" interface FetchGrapherConfigResult { grapherConfig: GrapherInterface | null - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] status: number etag: string | undefined } @@ -140,7 +140,7 @@ export async function fetchGrapherConfig({ const config = await fetchResponse.json() let grapherConfig: GrapherInterface - let multiDimDimensions: string[] + let multiDimAvailableDimensions: string[] if (identifier.type === "multi-dim-slug") { const multiDimConfig = config as MultiDimDataPageConfigEnriched grapherConfig = await fetchMultiDimGrapherConfig( @@ -148,7 +148,9 @@ export async function fetchGrapherConfig({ searchParams, env ) - multiDimDimensions = multiDimConfig.dimensions.map((dim) => dim.slug) + multiDimAvailableDimensions = multiDimConfig.dimensions.map( + (dim) => dim.slug + ) } else { grapherConfig = config } @@ -159,7 +161,7 @@ export async function fetchGrapherConfig({ etag: fetchResponse.headers.get("etag"), } if (identifier.type === "multi-dim-slug") { - result.multiDimDimensions = multiDimDimensions + result.multiDimAvailableDimensions = multiDimAvailableDimensions } return result } @@ -171,7 +173,7 @@ export async function initGrapher( env: Env ): Promise<{ grapher: Grapher - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] }> { let grapherConfigResponse: FetchGrapherConfigResult try { @@ -225,7 +227,8 @@ export async function initGrapher( return { grapher, - multiDimDimensions: grapherConfigResponse.multiDimDimensions, + multiDimAvailableDimensions: + grapherConfigResponse.multiDimAvailableDimensions, } } diff --git a/functions/_common/metadataTools.ts b/functions/_common/metadataTools.ts index c6f735b839b..9dcd074494b 100644 --- a/functions/_common/metadataTools.ts +++ b/functions/_common/metadataTools.ts @@ -56,7 +56,7 @@ export const getColumnsForMetadata = (grapher: Grapher) => { export function assembleMetadata( grapher: Grapher, searchParams: URLSearchParams, - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] ) { const useShortNames = searchParams.get("useColumnShortNames") === "true" @@ -193,7 +193,10 @@ export function assembleMetadata( dateDownloaded: dateDownloaded.toISOString().split("T")[0], // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be // inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters - activeFilters: getGrapherFilters(searchParams, multiDimDimensions), + activeFilters: getGrapherFilters( + searchParams, + multiDimAvailableDimensions + ), } return fullMetadata diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index d7a7e4773b9..3d1e3e20062 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -232,11 +232,14 @@ function* columnReadmeText(col: CoreColumn) { function* activeFilterSettings( searchParams: URLSearchParams, - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] ) { // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be // inlucded here, add them to the whitelist inside getGrapherFilters - const filterSettings = getGrapherFilters(searchParams, multiDimDimensions) + const filterSettings = getGrapherFilters( + searchParams, + multiDimAvailableDimensions + ) if (filterSettings) { yield "" yield `### Active Filters` @@ -255,7 +258,7 @@ export function constructReadme( grapher: Grapher, columns: CoreColumn[], searchParams: URLSearchParams, - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] ): string { const isSingleColumn = columns.length === 1 // Some computed columns have neither a source nor origins - filter these away @@ -272,7 +275,7 @@ export function constructReadme( readme = `# ${grapher.displayTitle} - Data package This data package contains the data that powers the chart ["${grapher.displayTitle}"](${urlWithFilters}) on the Our World in Data website. It was downloaded on ${downloadDate}. -${[...activeFilterSettings(searchParams, multiDimDimensions)].join("\n")} +${[...activeFilterSettings(searchParams, multiDimAvailableDimensions)].join("\n")} ## CSV Structure The high level structure of the CSV file is that each row is an observation for an entity (usually a country or region) and a timepoint (usually a year). diff --git a/functions/_common/urlTools.ts b/functions/_common/urlTools.ts index 952d0a63dab..3a5f65067f2 100644 --- a/functions/_common/urlTools.ts +++ b/functions/_common/urlTools.ts @@ -3,7 +3,7 @@ import { GRAPHER_QUERY_PARAM_KEYS } from "@ourworldindata/types" export function getGrapherFilters( searchParams: URLSearchParams, - multiDimDimensions?: string[] + multiDimAvailableDimensions?: string[] ): Record | undefined { const params = searchParams.size ? Object.fromEntries(searchParams) @@ -15,6 +15,6 @@ export function getGrapherFilters( delete params.useColumnShortNames return pick(params, [ ...GRAPHER_QUERY_PARAM_KEYS, - ...(multiDimDimensions ?? []), + ...(multiDimAvailableDimensions ?? []), ]) }