diff --git a/.vscode/settings.json b/.vscode/settings.json index 75a3f5f169..d81610df90 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,6 @@ }, "Prettier-SQL.keywordCase": "upper", "Prettier-SQL.SQLFlavourOverride": "mysql", - "Prettier-SQL.expressionWidth": 80 + "Prettier-SQL.expressionWidth": 80, + "prettier.semi": false } diff --git a/adminSiteClient/GrapherConfigGridEditor.tsx b/adminSiteClient/GrapherConfigGridEditor.tsx index b064688980..3652b23f7e 100644 --- a/adminSiteClient/GrapherConfigGridEditor.tsx +++ b/adminSiteClient/GrapherConfigGridEditor.tsx @@ -332,7 +332,8 @@ export class GrapherConfigGridEditor extends React.Component } diff --git a/baker/GrapherImageBaker.tsx b/baker/GrapherImageBaker.tsx index 51362d1caf..ea5a01cad1 100644 --- a/baker/GrapherImageBaker.tsx +++ b/baker/GrapherImageBaker.tsx @@ -34,7 +34,7 @@ export async function bakeGrapherToSvgAndPng( optimizeSvgs = false ) { const grapher = initGrapherForSvgExport(jsonConfig) - grapher.receiveOwidData(vardata) + // grapher.receiveOwidData(vardata) const outPath = path.join(outDir, grapher.slug as string) let svgCode = grapher.staticSVG @@ -124,8 +124,8 @@ export async function bakeGrapherToSvg( if (fs.existsSync(outPath) && !overwriteExisting) return const variableIds = grapher.dimensions.map((d) => d.variableId) - const vardata = await getDataForMultipleVariables(variableIds) - grapher.receiveOwidData(vardata) + const _vardata = await getDataForMultipleVariables(variableIds) + // grapher.receiveOwidData(vardata) let svgCode = grapher.staticSVG if (optimizeSvgs) svgCode = await optimizeSvg(svgCode) @@ -234,6 +234,6 @@ export async function grapherToSVG( const grapher = new Grapher({ ...jsonConfig, manuallyProvideData: true }) grapher.isExportingToSvgOrPng = true grapher.shouldIncludeDetailsInStaticExport = false - grapher.receiveOwidData(vardata) + // grapher.receiveOwidData(vardata) return grapher.staticSVG } diff --git a/baker/updateChartEntities.ts b/baker/updateChartEntities.ts index cbaf6a7410..1f5cfab9ec 100644 --- a/baker/updateChartEntities.ts +++ b/baker/updateChartEntities.ts @@ -93,7 +93,8 @@ const obtainAvailableEntitiesForGrapherConfig = async ( await getVariableDataUsingCache(variableId), ]) ) - grapher.receiveOwidData(variableData) + // TODO: make sure that data is loaded here + // grapher.receiveOwidData(variableData) // If the grapher has a chart tab, then the available entities there are the "most interesting" ones to us if (grapher.hasChartTab) { diff --git a/devTools/svgTester/utils.ts b/devTools/svgTester/utils.ts index 12d20a1c1d..537667bc29 100644 --- a/devTools/svgTester/utils.ts +++ b/devTools/svgTester/utils.ts @@ -419,7 +419,7 @@ export async function renderSvg( { shouldHashQueryStr: false, separator: "?" } ) - grapher.receiveOwidData(configAndData.variableData) + // grapher.receiveOwidData(configAndData.variableData) const durationReceiveData = Date.now() - timeStart const svg = grapher.staticSVG diff --git a/packages/@ourworldindata/explorer/src/Explorer.tsx b/packages/@ourworldindata/explorer/src/Explorer.tsx index 8a9b232f4b..991d2be99f 100644 --- a/packages/@ourworldindata/explorer/src/Explorer.tsx +++ b/packages/@ourworldindata/explorer/src/Explorer.tsx @@ -578,7 +578,7 @@ export class Explorer grapher.setAuthoredVersion(config) grapher.reset() grapher.updateFromObject(config) - grapher.downloadData() + // grapher.downloadData() } @action.bound async updateGrapherFromExplorerUsingVariableIds() { @@ -697,7 +697,7 @@ export class Explorer config.dimensions = dimensions if (ySlugs && yVariableIds) config.ySlugs = ySlugs + " " + yVariableIds - const inputTableTransformer = (table: OwidTable) => { + const _inputTableTransformer = (table: OwidTable) => { // add transformed (and intermediate) columns to the grapher table if (uniqueSlugsInGrapherRow.length) { const allColumnSlugs = uniq( @@ -745,11 +745,11 @@ export class Explorer if (dimensions.length === 0) { // If dimensions are empty, explicitly set the table to an empty table // so we don't end up confusingly showing stale data from a previous chart - grapher.receiveOwidData(new Map()) + // grapher.receiveOwidData(new Map()) } else { - await grapher.downloadLegacyDataFromOwidVariableIds( - inputTableTransformer - ) + // await grapher.downloadLegacyDataFromOwidVariableIds( + // inputTableTransformer + // ) } } diff --git a/packages/@ourworldindata/grapher/src/core/FetchingGrapher.tsx b/packages/@ourworldindata/grapher/src/core/FetchingGrapher.tsx new file mode 100644 index 0000000000..7b669a1c1b --- /dev/null +++ b/packages/@ourworldindata/grapher/src/core/FetchingGrapher.tsx @@ -0,0 +1,170 @@ +import { + GrapherInterface, + MultipleOwidVariableDataDimensionsMap, + OwidVariableDataMetadataDimensions, +} from "@ourworldindata/types" +import React from "react" +import { Grapher } from "./Grapher.js" +import { loadVariableDataAndMetadata } from "./loadVariable.js" +import { + legacyToOwidTableAndDimensions, + legacyToOwidTableAndDimensionsWithMandatorySlug, +} from "./LegacyToOwidTable.js" +import { OwidTable } from "@ourworldindata/core-table" + +export interface FetchingGrapherProps { + config?: GrapherInterface + configUrl?: string + queryString?: string + dataApiUrl: string + adminBaseUrl: string + bakedGrapherURL: string +} +export function FetchingGrapher( + props: FetchingGrapherProps +): JSX.Element | null { + // if config is not provided, fetch it from configUrl + + const [config, setConfig] = React.useState( + props.config + ) + + const [inputTable, setInputTable] = React.useState( + undefined + ) + + React.useEffect(() => { + async function fetchConfigAndLoadData(): Promise { + if (!config && props.configUrl) { + const fetchedConfig = await fetch(props.configUrl).then((res) => + res.json() + ) + setConfig(fetchedConfig) + } + if (!config) return + const dimensions = config.dimensions || [] + if (dimensions.length === 0) return + const variables = dimensions.map((d) => d.variableId) + const variablesDataMap = await loadVariablesDataSite( + variables, + props.dataApiUrl + ) + const inputTable = legacyToOwidTableAndDimensionsWithMandatorySlug( + variablesDataMap, + dimensions, + config.selectedEntityColors + ) + setInputTable(inputTable) + } + void fetchConfigAndLoadData() + }, [props.configUrl, config, props.dataApiUrl]) + + if (!config) return null + if (!inputTable) return null + return ( + + ) +} + +// async function loadVariablesDataAdmin( +// variableFetchBaseUrl: string | undefined, +// variableIds: number[] +// ): Promise { +// const dataFetchPath = (variableId: number): string => +// variableFetchBaseUrl +// ? `${variableFetchBaseUrl}/v1/variableById/data/${variableId}` +// : `/api/data/variables/data/${variableId}.json` +// const metadataFetchPath = (variableId: number): string => +// variableFetchBaseUrl +// ? `${variableFetchBaseUrl}/v1/variableById/metadata/${variableId}` +// : `/api/data/variables/metadata/${variableId}.json` + +// const loadVariableDataPromises = variableIds.map(async (variableId) => { +// const dataPromise = window.admin.getJSON( +// dataFetchPath(variableId) +// ) as Promise +// const metadataPromise = window.admin.getJSON( +// metadataFetchPath(variableId) +// ) as Promise +// const [data, metadata] = await Promise.all([ +// dataPromise, +// metadataPromise, +// ]) +// return { data, metadata: { ...metadata, id: variableId } } +// }) +// const variablesData: OwidVariableDataMetadataDimensions[] = +// await Promise.all(loadVariableDataPromises) +// const variablesDataMap = new Map( +// variablesData.map((data) => [data.metadata.id, data]) +// ) +// return variablesDataMap +// } + +async function loadVariablesDataSite( + variableIds: number[], + dataApiUrl: string +): Promise { + const loadVariableDataPromises = variableIds.map((variableId) => + loadVariableDataAndMetadata(variableId, dataApiUrl) + ) + const variablesData: OwidVariableDataMetadataDimensions[] = + await Promise.all(loadVariableDataPromises) + const variablesDataMap = new Map( + variablesData.map((data) => [data.metadata.id, data]) + ) + return variablesDataMap +} + +// function downloadData(): void { +// if (this.manuallyProvideData) { +// } else if (this.owidDataset) { +// this._receiveOwidDataAndApplySelection(this.owidDataset) +// } else void this.downloadLegacyDataFromOwidVariableIds() +// } + +// async function downloadLegacyDataFromOwidVariableIds( +// inputTableTransformer?: ChartTableTransformer +// ): Promise { +// if (this.variableIds.length === 0) +// // No data to download +// return + +// try { +// let variablesDataMap: MultipleOwidVariableDataDimensionsMap + +// const startMark = performance.now() +// if (this.useAdminAPI) { +// // TODO grapher model: switch this to downloading multiple data and metadata files +// variablesDataMap = await loadVariablesDataAdmin( +// this.dataApiUrlForAdmin, +// this.variableIds +// ) +// } else { +// variablesDataMap = await loadVariablesDataSite( +// this.variableIds, +// this.dataApiUrl +// ) +// } +// this.createPerformanceMeasurement("downloadVariablesData", startMark) + +// this._receiveOwidDataAndApplySelection( +// variablesDataMap, +// inputTableTransformer +// ) +// } catch (err) { +// // eslint-disable-next-line no-console +// console.log(`Error fetching '${err}'`) +// console.error(err) +// Bugsnag?.notify(`Error fetching variables: ${err}`, (event) => { +// event.addMetadata("context", { +// variableIds: this.variableIds, +// }) +// }) +// } +// } diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index bf0bef66f9..4f482f7bae 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -27,9 +27,6 @@ import { differenceObj, QueryParams, MultipleOwidVariableDataDimensionsMap, - OwidVariableDataMetadataDimensions, - OwidVariableMixedData, - OwidVariableWithSourceAndDimension, Bounds, DEFAULT_BOUNDS, minTimeBoundFromJSONOrNegativeInfinity, @@ -137,7 +134,6 @@ import { latestGrapherConfigSchema, GRAPHER_SQUARE_SIZE, } from "../core/GrapherConstants" -import { loadVariableDataAndMetadata } from "./loadVariable" import Cookies from "js-cookie" import { ChartDimension, @@ -205,7 +201,7 @@ import { import classnames from "classnames" import { GrapherAnalytics } from "./GrapherAnalytics" import { legacyToCurrentGrapherQueryParams } from "./GrapherUrlMigrations" -import { ChartInterface, ChartTableTransformer } from "../chart/ChartInterface" +import { ChartInterface } from "../chart/ChartInterface" import { MarimekkoChartManager } from "../stackedCharts/MarimekkoChartConstants" import Bugsnag from "@bugsnag/js" import { FacetChartManager } from "../facetChart/FacetChartConstants" @@ -238,55 +234,6 @@ declare global { } } -async function loadVariablesDataAdmin( - variableFetchBaseUrl: string | undefined, - variableIds: number[] -): Promise { - const dataFetchPath = (variableId: number): string => - variableFetchBaseUrl - ? `${variableFetchBaseUrl}/v1/variableById/data/${variableId}` - : `/api/data/variables/data/${variableId}.json` - const metadataFetchPath = (variableId: number): string => - variableFetchBaseUrl - ? `${variableFetchBaseUrl}/v1/variableById/metadata/${variableId}` - : `/api/data/variables/metadata/${variableId}.json` - - const loadVariableDataPromises = variableIds.map(async (variableId) => { - const dataPromise = window.admin.getJSON( - dataFetchPath(variableId) - ) as Promise - const metadataPromise = window.admin.getJSON( - metadataFetchPath(variableId) - ) as Promise - const [data, metadata] = await Promise.all([ - dataPromise, - metadataPromise, - ]) - return { data, metadata: { ...metadata, id: variableId } } - }) - const variablesData: OwidVariableDataMetadataDimensions[] = - await Promise.all(loadVariableDataPromises) - const variablesDataMap = new Map( - variablesData.map((data) => [data.metadata.id, data]) - ) - return variablesDataMap -} - -async function loadVariablesDataSite( - variableIds: number[], - dataApiUrl: string -): Promise { - const loadVariableDataPromises = variableIds.map((variableId) => - loadVariableDataAndMetadata(variableId, dataApiUrl) - ) - const variablesData: OwidVariableDataMetadataDimensions[] = - await Promise.all(loadVariableDataPromises) - const variablesDataMap = new Map( - variablesData.map((data) => [data.metadata.id, data]) - ) - return variablesDataMap -} - const DEFAULT_MS_PER_TICK = 100 // Exactly the same as GrapherInterface, but contains options that developers want but authors won't be touching. @@ -569,8 +516,6 @@ export class Grapher this.updateFromObject(props) } - if (!props.table) this.downloadData() - this.populateFromQueryParams( legacyToCurrentGrapherQueryParams(props.queryStr ?? "") ) @@ -616,14 +561,6 @@ export class Grapher return obj } - @action.bound downloadData(): void { - if (this.manuallyProvideData) { - // ignore - } else if (this.owidDataset) { - this._receiveOwidDataAndApplySelection(this.owidDataset) - } else void this.downloadLegacyDataFromOwidVariableIds() - } - @action.bound updateFromObject(obj?: GrapherProgrammaticInterface): void { if (!obj) return @@ -1098,63 +1035,9 @@ export class Grapher // In old browsers, the above may throw an error - just ignore it } } - - @action.bound - async downloadLegacyDataFromOwidVariableIds( - inputTableTransformer?: ChartTableTransformer - ): Promise { - if (this.variableIds.length === 0) - // No data to download - return - - try { - let variablesDataMap: MultipleOwidVariableDataDimensionsMap - - const startMark = performance.now() - if (this.useAdminAPI) { - // TODO grapher model: switch this to downloading multiple data and metadata files - variablesDataMap = await loadVariablesDataAdmin( - this.dataApiUrlForAdmin, - this.variableIds - ) - } else { - variablesDataMap = await loadVariablesDataSite( - this.variableIds, - this.dataApiUrl - ) - } - this.createPerformanceMeasurement( - "downloadVariablesData", - startMark - ) - - this._receiveOwidDataAndApplySelection( - variablesDataMap, - inputTableTransformer - ) - } catch (err) { - // eslint-disable-next-line no-console - console.log(`Error fetching '${err}'`) - console.error(err) - Bugsnag?.notify(`Error fetching variables: ${err}`, (event) => { - event.addMetadata("context", { - variableIds: this.variableIds, - }) - }) - } - } - - @action.bound receiveOwidData( - json: MultipleOwidVariableDataDimensionsMap - ): void { - // TODO grapher model: switch this to downloading multiple data and metadata files - this._receiveOwidDataAndApplySelection(json) - } - @action.bound private _setInputTable( json: MultipleOwidVariableDataDimensionsMap, - legacyConfig: Partial, - inputTableTransformer?: ChartTableTransformer + legacyConfig: Partial ): void { // TODO grapher model: switch this to downloading multiple data and metadata files @@ -1178,9 +1061,7 @@ export class Grapher startMark ) - if (inputTableTransformer) - this.inputTable = inputTableTransformer(tableWithColors) - else this.inputTable = tableWithColors + this.inputTable = tableWithColors this.appendNewEntitySelectionOptions() @@ -1191,30 +1072,17 @@ export class Grapher } else this.applyOriginalSelectionAsAuthored() } - @action rebuildInputOwidTable( - inputTableTransformer?: ChartTableTransformer - ): void { + @action rebuildInputOwidTable(): void { // TODO grapher model: switch this to downloading multiple data and metadata files if (!this.legacyVariableDataJson) return this._setInputTable( this.legacyVariableDataJson, - this.legacyConfigAsAuthored, - inputTableTransformer + this.legacyConfigAsAuthored ) } @observable private legacyVariableDataJson?: MultipleOwidVariableDataDimensionsMap - - @action.bound private _receiveOwidDataAndApplySelection( - json: MultipleOwidVariableDataDimensionsMap, - inputTableTransformer?: ChartTableTransformer - ): void { - this.legacyVariableDataJson = json - - this.rebuildInputOwidTable(inputTableTransformer) - } - @action.bound appendNewEntitySelectionOptions(): void { const { selection } = this const currentEntities = selection.availableEntityNameSet @@ -1443,12 +1311,6 @@ export class Grapher // todo: can we remove this? // I believe these states can only occur during editing. @action.bound private ensureValidConfigWhenEditing(): void { - this.disposers.push( - reaction( - () => this.variableIds, - () => this.downloadLegacyDataFromOwidVariableIds() - ) - ) const disposers = [ autorun(() => { if (!this.availableTabs.includes(this.activeTab)) diff --git a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.test.ts b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.test.ts index a6bd3265a0..1f950c2a22 100755 --- a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.test.ts +++ b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.test.ts @@ -6,40 +6,17 @@ import { OwidTableSlugs, StandardOwidColumnDefs, LegacyGrapherInterface, - OwidChartDimensionInterface, } from "@ourworldindata/types" +import { ColumnTypeMap, ErrorValueTypes } from "@ourworldindata/core-table" import { - ColumnTypeMap, - ErrorValueTypes, - OwidTable, -} from "@ourworldindata/core-table" -import { legacyToOwidTableAndDimensions } from "./LegacyToOwidTable" + legacyToOwidTableAndDimensions, + legacyToOwidTableAndDimensionsWithMandatorySlug, +} from "./LegacyToOwidTable" import { MultipleOwidVariableDataDimensionsMap, OwidVariableDataMetadataDimensions, DimensionProperty, } from "@ourworldindata/utils" -import { getDimensionColumnSlug } from "../chart/ChartDimension.js" - -export const legacyToOwidTableAndDimensionsWithMandatorySlug = ( - json: MultipleOwidVariableDataDimensionsMap, - dimensions: OwidChartDimensionInterface[], - selectedEntityColors: - | { [entityName: string]: string | undefined } - | undefined -): OwidTable => { - const dimensionsWithSlug = dimensions?.map((dimension) => ({ - ...dimension, - slug: - dimension.slug ?? - getDimensionColumnSlug(dimension.variableId, dimension.targetYear), - })) - return legacyToOwidTableAndDimensions( - json, - dimensionsWithSlug, - selectedEntityColors - ) -} describe(legacyToOwidTableAndDimensions, () => { const legacyVariableEntry: OwidVariableDataMetadataDimensions = { diff --git a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts index 20836a136f..0ca6200802 100644 --- a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts +++ b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts @@ -11,6 +11,7 @@ import { EntityId, ErrorValue, OwidChartDimensionInterfaceWithMandatorySlug, + OwidChartDimensionInterface, } from "@ourworldindata/types" import { OwidTable, @@ -40,6 +41,27 @@ import { isEmpty, } from "@ourworldindata/utils" import { isContinentsVariableId } from "./GrapherConstants" +import { getDimensionColumnSlug } from "../chart/ChartDimension.js" + +export const legacyToOwidTableAndDimensionsWithMandatorySlug = ( + json: MultipleOwidVariableDataDimensionsMap, + dimensions: OwidChartDimensionInterface[], + selectedEntityColors: + | { [entityName: string]: string | undefined } + | undefined +): OwidTable => { + const dimensionsWithSlug = dimensions?.map((dimension) => ({ + ...dimension, + slug: + dimension.slug ?? + getDimensionColumnSlug(dimension.variableId, dimension.targetYear), + })) + return legacyToOwidTableAndDimensions( + json, + dimensionsWithSlug, + selectedEntityColors + ) +} export const legacyToOwidTableAndDimensions = ( json: MultipleOwidVariableDataDimensionsMap, @@ -90,7 +112,8 @@ export const legacyToOwidTableAndDimensions = ( const valueColumnColor = dimension.display?.color // Ensure the column slug is unique by copying it from the dimensions // (there can be two columns of the same variable with different targetTimes) - valueColumnDef.slug = dimension.slug + if (dimension.slug) valueColumnDef.slug = dimension.slug + else throw new Error("Dimension slug was undefined") // Because database columns can contain mixed types, we want to avoid // parsing for Grapher data until we fix that. valueColumnDef.skipParsing = true diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index 13ca26a69c..ef77a13b1e 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -7,6 +7,7 @@ export { type ColorScaleBin, } from "./color/ColorScaleBin" export { ChartDimension } from "./chart/ChartDimension" +export { FetchingGrapher } from "./core/FetchingGrapher" export { GRAPHER_EMBEDDED_FIGURE_ATTR, GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR, diff --git a/site/DataPageV2Content.tsx b/site/DataPageV2Content.tsx index 673dbc7a39..1f3c936059 100644 --- a/site/DataPageV2Content.tsx +++ b/site/DataPageV2Content.tsx @@ -1,10 +1,12 @@ -import { useState, useEffect, useMemo } from "react" -import { Grapher, GrapherProgrammaticInterface } from "@ourworldindata/grapher" +import { useMemo } from "react" +import { + FetchingGrapher, + GrapherProgrammaticInterface, +} from "@ourworldindata/grapher" import { REUSE_THIS_WORK_SECTION_ID, DATAPAGE_SOURCES_AND_PROCESSING_SECTION_ID, } from "@ourworldindata/components" -import { GrapherWithFallback } from "./GrapherWithFallback.js" import { RelatedCharts } from "./blocks/RelatedCharts.js" import { DataPageV2ContentFields, @@ -18,7 +20,12 @@ import { import { DocumentContext } from "./gdocs/DocumentContext.js" import { AttachmentsContext } from "./gdocs/AttachmentsContext.js" import StickyNav from "./blocks/StickyNav.js" -import { BAKED_BASE_URL } from "../settings/clientSettings.js" +import { + ADMIN_BASE_URL, + BAKED_BASE_URL, + BAKED_GRAPHER_URL, + DATA_API_URL, +} from "../settings/clientSettings.js" import Image from "./gdocs/components/Image.js" import AboutThisData from "./AboutThisData.js" import MetadataSection from "./MetadataSection.js" @@ -72,8 +79,6 @@ export const DataPageV2Content = ({ grapherConfig: GrapherInterface imageMetadata: Record }) => { - const [grapher, setGrapher] = useState(undefined) - const titleFragments = joinTitleFragments( datapageData.attributionShort, datapageData.titleVariant @@ -89,10 +94,6 @@ export const DataPageV2Content = ({ [grapherConfig] ) - useEffect(() => { - setGrapher(new Grapher(mergedGrapherConfig)) - }, [mergedGrapherConfig]) - const stickyNavLinks = [ { text: "Explore the Data", @@ -147,9 +148,11 @@ export const DataPageV2Content = ({ >
-
@@ -181,14 +184,11 @@ export const DataPageV2Content = ({
-
-
- -
+
{grapher.slug && (