diff --git a/adminSiteClient/AdminSidebar.tsx b/adminSiteClient/AdminSidebar.tsx index 3b7ad4c2d0c..ec76340cbe7 100644 --- a/adminSiteClient/AdminSidebar.tsx +++ b/adminSiteClient/AdminSidebar.tsx @@ -37,7 +37,7 @@ export const AdminSidebar = (): React.ReactElement => ( {chartViewsFeatureEnabled && (
  • - Narrative views + Narrative charts
  • )} diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index 1a4747fbb39..839efe0794a 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -13,7 +13,6 @@ import { getParentVariableIdFromChartConfig, mergeGrapherConfigs, isEmpty, - slugify, omit, CHART_VIEW_PROPS_TO_OMIT, } from "@ourworldindata/utils" @@ -200,23 +199,13 @@ export class ChartEditor extends AbstractChartEditor { ) } - async saveAsNarrativeView(): Promise { + async saveAsChartView( + name: string + ): Promise<{ success: boolean; errorMsg?: string }> { const { patchConfig, grapher } = this const chartJson = omit(patchConfig, CHART_VIEW_PROPS_TO_OMIT) - const suggestedName = grapher.title ? slugify(grapher.title) : undefined - - const name = prompt( - "Please enter a programmatic name for the narrative view. Note that this name cannot be changed later.", - suggestedName - ) - - if (name === null) return - - // Need to open intermediary tab before AJAX to avoid popup blockers - const w = window.open("/", "_blank") as Window - const body = { name, parentChartId: grapher.id, @@ -228,11 +217,14 @@ export class ChartEditor extends AbstractChartEditor { body, "POST" ) - - if (json.success) - w.location.assign( + if (json.success) { + window.open( this.manager.admin.url(`chartViews/${json.chartViewId}/edit`) ) + return { success: true } + } else { + return { success: false, errorMsg: json.errorMsg } + } } publishGrapher(): void { diff --git a/adminSiteClient/ChartViewEditor.ts b/adminSiteClient/ChartViewEditor.ts index 982f967b83a..578ec7028c3 100644 --- a/adminSiteClient/ChartViewEditor.ts +++ b/adminSiteClient/ChartViewEditor.ts @@ -5,7 +5,7 @@ import { References, type EditorTab, } from "./AbstractChartEditor.js" -import { ENV } from "../settings/clientSettings.js" +import { BAKED_BASE_URL, ENV } from "../settings/clientSettings.js" import { CHART_VIEW_PROPS_TO_OMIT, CHART_VIEW_PROPS_TO_PERSIST, @@ -16,7 +16,8 @@ import { diffGrapherConfigs, omit, pick } from "@ourworldindata/utils" // Don't yet show chart views in the admin interface // This is low-stakes - if it shows up anyhow (e.g. on staging servers), it's not a big deal. // TODO: Remove this flag once we're launching this feature -export const chartViewsFeatureEnabled = ENV === "development" +export const chartViewsFeatureEnabled = + ENV === "development" || BAKED_BASE_URL.includes("narrative-") export interface Chart { id: number diff --git a/adminSiteClient/ChartViewIndexPage.tsx b/adminSiteClient/ChartViewIndexPage.tsx index 4211ba1d15b..c0b5d64d3b1 100644 --- a/adminSiteClient/ChartViewIndexPage.tsx +++ b/adminSiteClient/ChartViewIndexPage.tsx @@ -7,7 +7,7 @@ import { AdminAppContext } from "./AdminAppContext.js" import { Timeago } from "./Forms.js" import { ColumnsType } from "antd/es/table/InternalTable.js" import { ApiChartViewOverview } from "../adminShared/AdminTypes.js" -import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js" +import { GRAPHER_DYNAMIC_THUMBNAIL_URL } from "../settings/clientSettings.js" import { Link } from "./Link.js" import { buildSearchWordsFromSearchString, @@ -28,7 +28,7 @@ function createColumns( width: 200, render: (chartConfigId) => ( ), @@ -135,7 +135,7 @@ export function ChartViewIndexPage() { }, [admin]) return ( - +
    -

    Narrative views based on this chart

    +

    Narrative charts based on this chart

      {props.references.chartViews.map((chartView) => (
    • diff --git a/adminSiteClient/EditorTextTab.tsx b/adminSiteClient/EditorTextTab.tsx index b259827bdd7..a8106b6e962 100644 --- a/adminSiteClient/EditorTextTab.tsx +++ b/adminSiteClient/EditorTextTab.tsx @@ -19,6 +19,7 @@ import { } from "./Forms.js" import { AbstractChartEditor } from "./AbstractChartEditor.js" import { ErrorMessages } from "./ChartEditorTypes.js" +import { isChartViewEditorInstance } from "./ChartViewEditor.js" @observer export class EditorTextTab< @@ -74,6 +75,10 @@ export class EditorTextTab< return this.props.errorMessages } + @computed get showChartSlug() { + return !isChartViewEditorInstance(this.props.editor) + } + @computed get showAnyAnnotationFieldInTitleToggle() { const { features } = this.props.editor return ( @@ -139,19 +144,21 @@ export class EditorTextTab< /> )} {this.showAnyAnnotationFieldInTitleToggle &&
      } - - (grapher.slug = - grapher.slug === undefined - ? grapher.displaySlug - : undefined) - } - helpText="Human-friendly URL for this chart" - /> + {this.showChartSlug && ( + + (grapher.slug = + grapher.slug === undefined + ? grapher.displaySlug + : undefined) + } + helpText="Human-friendly URL for this chart" + /> + )} grapher.currentSubtitle} diff --git a/adminSiteClient/ImagesIndexPage.tsx b/adminSiteClient/ImagesIndexPage.tsx index 1de6114e21b..09a4400c863 100644 --- a/adminSiteClient/ImagesIndexPage.tsx +++ b/adminSiteClient/ImagesIndexPage.tsx @@ -281,12 +281,13 @@ function createColumns({ title: "Filename", dataIndex: "filename", key: "filename", - width: 300, + width: 200, }, { title: "Alt text", dataIndex: "defaultAlt", key: "defaultAlt", + width: "auto", sorter: (a, b) => a.defaultAlt && b.defaultAlt ? a.defaultAlt.localeCompare(b.defaultAlt) @@ -309,7 +310,7 @@ function createColumns({ a.originalWidth && b.originalWidth ? a.originalWidth - b.originalWidth : 0, - width: 100, + width: 50, }, { title: "Height", @@ -319,13 +320,13 @@ function createColumns({ a.originalHeight && b.originalHeight ? a.originalHeight - b.originalHeight : 0, - width: 100, + width: 50, }, { title: "Last updated", dataIndex: "updatedAt", key: "updatedAt", - width: 150, + width: 50, defaultSortOrder: "descend", sorter: (a, b) => a.updatedAt && b.updatedAt ? a.updatedAt - b.updatedAt : 0, @@ -334,7 +335,7 @@ function createColumns({ { title: "Owner", key: "userId", - width: 200, + width: 100, filters: [ { text: "Unassigned", @@ -375,7 +376,7 @@ function createColumns({ { title: "Action", key: "action", - width: 100, + width: 50, render: (_, image) => { const isDeleteDisabled = !!(usage && usage[image.id]?.length) return ( @@ -657,7 +658,12 @@ export function ImageIndexPage() { /> - +
      x.id} + /> diff --git a/adminSiteClient/NarrativeChartNameModal.tsx b/adminSiteClient/NarrativeChartNameModal.tsx new file mode 100644 index 00000000000..81d61c02bd8 --- /dev/null +++ b/adminSiteClient/NarrativeChartNameModal.tsx @@ -0,0 +1,64 @@ +import { useEffect, useMemo, useRef, useState } from "react" +import { Form, Input, InputRef, Modal, Spin } from "antd" + +export const NarrativeChartNameModal = (props: { + initialName: string + open: "open" | "open-loading" | "closed" + errorMsg?: string + onSubmit: (name: string) => void + onCancel?: () => void +}) => { + const [name, setName] = useState(props.initialName) + const inputField = useRef(null) + const isLoading = useMemo(() => props.open === "open-loading", [props.open]) + const isOpen = useMemo(() => props.open !== "closed", [props.open]) + + useEffect(() => setName(props.initialName), [props.initialName]) + + useEffect(() => { + if (isOpen) { + inputField.current?.focus({ cursor: "all" }) + } + }, [isOpen]) + + return ( + props.onSubmit(name)} + onCancel={props.onCancel} + onClose={props.onCancel} + okButtonProps={{ disabled: !name || isLoading }} + cancelButtonProps={{ disabled: isLoading }} + > +
      +

      + This will create a new narrative chart that is linked to + this chart. Any currently pending changes will be applied to + the narrative chart. +

      +

      + Please enter a programmatic name for the narrative chart.{" "} + Note that this name cannot be changed later. +

      + + setName(e.target.value)} + value={name} + disabled={isLoading} + /> + + {isLoading && } + {props.errorMsg && ( +
      + {props.errorMsg} +
      + )} +
      +
      + ) +} diff --git a/adminSiteClient/SaveButtons.tsx b/adminSiteClient/SaveButtons.tsx index ce63127a917..0bdbafca275 100644 --- a/adminSiteClient/SaveButtons.tsx +++ b/adminSiteClient/SaveButtons.tsx @@ -1,8 +1,8 @@ import { Component } from "react" import { ChartEditor, isChartEditorInstance } from "./ChartEditor.js" -import { action, computed } from "mobx" +import { action, computed, observable } from "mobx" import { observer } from "mobx-react" -import { excludeUndefined, omit } from "@ourworldindata/utils" +import { excludeUndefined, omit, slugify } from "@ourworldindata/utils" import { IndicatorChartEditor, isIndicatorChartEditorInstance, @@ -17,6 +17,7 @@ import { chartViewsFeatureEnabled, isChartViewEditorInstance, } from "./ChartViewEditor.js" +import { NarrativeChartNameModal } from "./NarrativeChartNameModal.js" @observer export class SaveButtons extends Component<{ @@ -61,10 +62,6 @@ class SaveButtonsForChart extends Component<{ void this.props.editor.saveAsNewGrapher() } - @action.bound onSaveAsNarrativeView() { - void this.props.editor.saveAsNarrativeView() - } - @action.bound onPublishToggle() { if (this.props.editor.grapher.isPublished) this.props.editor.unpublishGrapher() @@ -79,6 +76,29 @@ class SaveButtonsForChart extends Component<{ ]) } + @computed get initialNarrativeChartName(): string { + return slugify(this.props.editor.grapher.title ?? "") + } + + @observable narrativeChartNameModalOpen: + | "open" + | "open-loading" + | "closed" = "closed" + @observable narrativeChartNameModalError: string | undefined = undefined + + @action.bound async onSubmitNarrativeChartButton(name: string) { + const { editor } = this.props + + this.narrativeChartNameModalOpen = "open-loading" + const res = await editor.saveAsChartView(name) + if (res.success) { + this.narrativeChartNameModalOpen = "closed" + } else { + this.narrativeChartNameModalOpen = "open" + this.narrativeChartNameModalError = res.errorMsg + } + } + render() { const { editingErrors } = this const { editor } = this.props @@ -117,15 +137,30 @@ class SaveButtonsForChart extends Component<{ {chartViewsFeatureEnabled && ( -
      - -
      + <> +
      + +
      + + (this.narrativeChartNameModalOpen = "closed") + } + /> + )} {editingErrors.map((error, i) => (
      diff --git a/adminSiteClient/admin.scss b/adminSiteClient/admin.scss index 966b05e0a57..7fc8df085ef 100644 --- a/adminSiteClient/admin.scss +++ b/adminSiteClient/admin.scss @@ -1224,6 +1224,14 @@ main:not(.ChartEditorPage):not(.GdocsEditPage) { } .ImageIndexPage { + @media (max-width: 1300px) { + padding-left: 0 !important; + padding-right: 0 !important; + .ant-table-cell { + padding-left: 4px !important; + padding-right: 4px !important; + } + } .ImageIndexPage__delete-user-button { border-radius: 50%; margin-left: 8px; diff --git a/adminSiteServer/FunctionalRouter.ts b/adminSiteServer/FunctionalRouter.ts deleted file mode 100644 index 28275eb8d03..00000000000 --- a/adminSiteServer/FunctionalRouter.ts +++ /dev/null @@ -1,60 +0,0 @@ -import express, { NextFunction, Router } from "express" -import { Request, Response } from "./authentication.js" - -// Little wrapper to automatically send returned objects as JSON, makes -// the API code a bit cleaner -export class FunctionalRouter { - router: Router - constructor() { - this.router = Router() - this.router.use(express.urlencoded({ extended: true })) - // Parse incoming requests with JSON payloads http://expressjs.com/en/api.html - this.router.use(express.json({ limit: "50mb" })) - } - - wrap(callback: (req: Request, res: Response) => Promise) { - return async (req: Request, res: Response, next: NextFunction) => { - try { - res.send(await callback(req, res)) - } catch (e) { - console.error(e) - next(e) - } - } - } - - get( - targetPath: string, - callback: (req: Request, res: Response) => Promise - ) { - this.router.get(targetPath, this.wrap(callback)) - } - - post( - targetPath: string, - callback: (req: Request, res: Response) => Promise - ) { - this.router.post(targetPath, this.wrap(callback)) - } - - patch( - targetPath: string, - callback: (req: Request, res: Response) => Promise - ) { - this.router.patch(targetPath, this.wrap(callback)) - } - - put( - targetPath: string, - callback: (req: Request, res: Response) => Promise - ) { - this.router.put(targetPath, this.wrap(callback)) - } - - delete( - targetPath: string, - callback: (req: Request, res: Response) => Promise - ) { - this.router.delete(targetPath, this.wrap(callback)) - } -} diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 5cf0e042bc3..5d97a523223 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -1,3871 +1,437 @@ /* eslint @typescript-eslint/no-unused-vars: [ "warn", { argsIgnorePattern: "^(res|req)$" } ] */ -import * as lodash from "lodash" -import * as db from "../db/db.js" -import { - UNCATEGORIZED_TAG_ID, - BAKE_ON_CHANGE, - BAKED_BASE_URL, - ADMIN_BASE_URL, - DATA_API_URL, - FEATURE_FLAGS, -} from "../settings/serverSettings.js" +import { TaggableType } from "@ourworldindata/types" +import { DeployQueueServer } from "../baker/DeployQueueServer.js" import { - CLOUDFLARE_IMAGES_URL, - FeatureFlagFeature, -} from "../settings/clientSettings.js" -import { expectInt, isValidSlug } from "../serverUtils/serverUtil.js" + updateVariableAnnotations, + getChartBulkUpdate, + updateBulkChartConfigs, + getVariableAnnotations, +} from "./apiRoutes/bulkUpdates.js" import { - OldChartFieldList, - assignTagsForCharts, - getChartConfigById, - getChartSlugById, - getGptTopicSuggestions, - getRedirectsByChartId, - oldChartFieldList, - setChartTags, - getParentByChartConfig, - getPatchConfigByChartId, - isInheritanceEnabledForChart, - getParentByChartId, -} from "../db/model/Chart.js" -import { Request } from "./authentication.js" + getChartViews, + getChartViewById, + createChartView, + updateChartView, + deleteChartView, +} from "./apiRoutes/chartViews.js" import { - getMergedGrapherConfigForVariable, - fetchS3MetadataByPath, - fetchS3DataValuesByPath, - searchVariables, - getGrapherConfigsForVariable, - updateGrapherConfigAdminOfVariable, - updateGrapherConfigETLOfVariable, - updateAllChartsThatInheritFromIndicator, - updateAllMultiDimViewsThatInheritFromIndicator, - getAllChartsForIndicator, -} from "../db/model/Variable.js" -import { updateExistingFullConfig } from "../db/model/ChartConfigs.js" -import { getCanonicalUrl } from "@ourworldindata/components" + getDatasets, + getDataset, + updateDataset, + setArchived, + setTags, + deleteDataset, + republishCharts, +} from "./apiRoutes/datasets.js" +import { addExplorerTags, deleteExplorerTags } from "./apiRoutes/explorer.js" import { - GDOCS_BASE_URL, - camelCaseProperties, - GdocsContentSource, - isEmpty, - JsonError, - OwidGdocPostInterface, - parseIntOrUndefined, - DbRawPostWithGdocPublishStatus, - OwidVariableWithSource, - TaggableType, - DbChartTagJoin, - pick, - Json, - checkIsGdocPostExcludingFragments, - checkIsPlainObjectWithGuard, - mergeGrapherConfigs, - diffGrapherConfigs, - omitUndefinedValues, - getParentVariableIdFromChartConfig, - omit, - gdocUrlRegex, -} from "@ourworldindata/utils" -import { applyPatch } from "../adminShared/patchHelper.js" + getAllGdocIndexItems, + getIndividualGdoc, + createOrUpdateGdoc, + deleteGdoc, + setGdocTags, +} from "./apiRoutes/gdocs.js" import { - OperationContext, - parseToOperation, -} from "../adminShared/SqlFilterSExpression.js" + getImagesHandler, + postImageHandler, + putImageHandler, + patchImageHandler, + deleteImageHandler, + getImageUsageHandler, +} from "./apiRoutes/images.js" +import { handleMultiDimDataPageRequest } from "./apiRoutes/mdims.js" import { - BulkChartEditResponseRow, - BulkGrapherConfigResponse, - chartBulkUpdateAllowedColumnNamesAndTypes, - GrapherConfigPatch, - variableAnnotationAllowedColumnNamesAndTypes, - VariableAnnotationsResponseRow, -} from "../adminShared/AdminSessionTypes.js" + fetchAllWork, + fetchNamespaces, + fetchSourceById, +} from "./apiRoutes/misc.js" import { - DbPlainDatasetTag, - GrapherInterface, - OwidGdocType, - DbPlainUser, - UsersTableName, - DbPlainTag, - DbRawVariable, - parseOriginsRow, - PostsTableName, - DbRawPost, - DbPlainChartSlugRedirect, - DbPlainChart, - DbInsertChartRevision, - serializeChartConfig, - DbRawOrigin, - DbRawPostGdoc, - PostsGdocsXImagesTableName, - PostsGdocsLinksTableName, - PostsGdocsTableName, - DbPlainDataset, - DbInsertUser, - FlatTagGraph, - DbRawChartConfig, - parseChartConfig, - MultiDimDataPageConfigRaw, - R2GrapherConfigDirectory, - ChartConfigsTableName, - Base64String, - DbPlainChartView, - ChartViewsTableName, - DbInsertChartView, - PostsGdocsComponentsTableName, - CHART_VIEW_PROPS_TO_PERSIST, - CHART_VIEW_PROPS_TO_OMIT, - DbEnrichedImage, - JsonString, -} from "@ourworldindata/types" -import { uuidv7 } from "uuidv7" + handleGetPostsJson, + handleSetTagsForPost, + handleGetPostById, + handleCreateGdoc, + handleUnlinkGdoc, +} from "./apiRoutes/posts.js" import { - migrateGrapherConfigToLatestVersion, - getVariableDataRoute, - getVariableMetadataRoute, - defaultGrapherConfig, - grapherConfigToQueryParams, -} from "@ourworldindata/grapher" -import { getDatasetById, setTagsForDataset } from "../db/model/Dataset.js" -import { getUserById, insertUser, updateUser } from "../db/model/User.js" -import { GdocPost } from "../db/model/Gdoc/GdocPost.js" + handleGetSiteRedirects, + handlePostNewSiteRedirect, + handleDeleteSiteRedirect, + handleGetRedirects, + handlePostNewChartRedirect, + handleDeleteChartRedirect, +} from "./apiRoutes/redirects.js" +import { triggerStaticBuild } from "./apiRoutes/routeUtils.js" +import { suggestGptTopics, suggestGptAltText } from "./apiRoutes/suggest.js" import { - syncDatasetToGitRepo, - removeDatasetFromGitRepo, -} from "./gitDataExport.js" -import { denormalizeLatestCountryData } from "../baker/countryProfiles.js" + handleGetFlatTagGraph, + handlePostTagGraph, +} from "./apiRoutes/tagGraph.js" import { - indexIndividualGdocPost, - removeIndividualGdocPostFromIndex, -} from "../baker/algolia/utils/pages.js" -import { ChartViewMinimalInformation } from "../adminSiteClient/ChartEditor.js" -import { DeployQueueServer } from "../baker/DeployQueueServer.js" -import { FunctionalRouter } from "./FunctionalRouter.js" -import Papa from "papaparse" + getTagById, + updateTag, + createTag, + getAllTags, + deleteTag, +} from "./apiRoutes/tags.js" import { - setTagsForPost, - getTagsByPostId, - getWordpressPostReferencesByChartId, - getGdocsPostReferencesByChartId, -} from "../db/model/Post.js" + getUsers, + getUserByIdHandler, + deleteUser, + updateUserHandler, + addUser, + addImageToUser, + removeUserImage, +} from "./apiRoutes/users.js" import { - checkHasChanges, - checkIsLightningUpdate, - GdocPublishingAction, - getPublishingAction, -} from "../adminSiteClient/gdocsDeploy.js" -import { createGdocAndInsertOwidGdocPostContent } from "../db/model/Gdoc/archieToGdoc.js" -import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" + getEditorVariablesJson, + getVariableDataJson, + getVariableMetadataJson, + getVariablesJson, + getVariablesUsagesJson, + getVariablesGrapherConfigETLPatchConfigJson, + getVariablesGrapherConfigAdminPatchConfigJson, + getVariablesMergedGrapherConfigJson, + getVariablesVariableIdJson, + putVariablesVariableIdGrapherConfigETL, + deleteVariablesVariableIdGrapherConfigETL, + putVariablesVariableIdGrapherConfigAdmin, + deleteVariablesVariableIdGrapherConfigAdmin, + getVariablesVariableIdChartsJson, +} from "./apiRoutes/variables.js" import { + patchRouteWithRWTransaction, getRouteWithROTransaction, - deleteRouteWithRWTransaction, - putRouteWithRWTransaction, postRouteWithRWTransaction, - patchRouteWithRWTransaction, + putRouteWithRWTransaction, + deleteRouteWithRWTransaction, getRouteNonIdempotentWithRWTransaction, } from "./functionalRouterHelpers.js" -import { getPublishedLinksTo } from "../db/model/Link.js" -import { - getChainedRedirect, - getRedirectById, - getRedirects, - redirectWithSourceExists, -} from "../db/model/Redirect.js" -import { getMinimalGdocPostsByIds } from "../db/model/Gdoc/GdocBase.js" -import { - GdocLinkUpdateMode, - createOrLoadGdocById, - gdocFromJSON, - getAllGdocIndexItemsOrderedByUpdatedAt, - getAndLoadGdocById, - getGdocBaseObjectById, - setLinksForGdoc, - setTagsForGdoc, - addImagesToContentGraph, - updateGdocContentOnly, - upsertGdoc, -} from "../db/model/Gdoc/GdocFactory.js" -import { match } from "ts-pattern" -import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js" -import { GdocHomepage } from "../db/model/Gdoc/GdocHomepage.js" -import { GdocAbout } from "../db/model/Gdoc/GdocAbout.js" -import { GdocAuthor } from "../db/model/Gdoc/GdocAuthor.js" -import path from "path" -import { - deleteGrapherConfigFromR2, - deleteGrapherConfigFromR2ByUUID, - saveGrapherConfigToR2ByUUID, -} from "./chartConfigR2Helpers.js" -import { createMultiDimConfig } from "./multiDim.js" -import { isMultiDimDataPagePublished } from "../db/model/MultiDimDataPage.js" import { - retrieveChartConfigFromDbAndSaveToR2, - saveNewChartConfigInDbAndR2, - updateChartConfigInDbAndR2, -} from "./chartConfigHelpers.js" -import { ApiChartViewOverview } from "../adminShared/AdminTypes.js" -import { References } from "../adminSiteClient/AbstractChartEditor.js" -import { - deleteFromCloudflare, - fetchGptGeneratedAltText, - processImageContent, - uploadToCloudflare, - validateImagePayload, -} from "./imagesHelpers.js" -import pMap from "p-map" - -const apiRouter = new FunctionalRouter() - -// Call this to trigger build and deployment of static charts on change -const triggerStaticBuild = async (user: DbPlainUser, commitMessage: string) => { - if (!BAKE_ON_CHANGE) { - console.log( - "Not triggering static build because BAKE_ON_CHANGE is false" - ) - return - } - - return new DeployQueueServer().enqueueChange({ - timeISOString: new Date().toISOString(), - authorName: user.fullName, - authorEmail: user.email, - message: commitMessage, - }) -} - -const enqueueLightningChange = async ( - user: DbPlainUser, - commitMessage: string, - slug: string -) => { - if (!BAKE_ON_CHANGE) { - console.log( - "Not triggering static build because BAKE_ON_CHANGE is false" - ) - return - } - - return new DeployQueueServer().enqueueChange({ - timeISOString: new Date().toISOString(), - authorName: user.fullName, - authorEmail: user.email, - message: commitMessage, - slug, - }) -} - -async function getLogsByChartId( - knex: db.KnexReadonlyTransaction, - chartId: number -): Promise< - { - userId: number - config: Json - userName: string - createdAt: Date - }[] -> { - const logs = await db.knexRaw<{ - userId: number - config: string - userName: string - createdAt: Date - }>( - knex, - `SELECT userId, config, fullName as userName, l.createdAt - FROM chart_revisions l - LEFT JOIN users u on u.id = userId - WHERE chartId = ? - ORDER BY l.id DESC - LIMIT 50`, - [chartId] - ) - return logs.map((log) => ({ - ...log, - config: JSON.parse(log.config), - })) -} - -const getReferencesByChartId = async ( - chartId: number, - knex: db.KnexReadonlyTransaction -): Promise => { - const postsWordpressPromise = getWordpressPostReferencesByChartId( - chartId, - knex - ) - const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex) - const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>( - knex, - `SELECT DISTINCT - explorerSlug - FROM - explorer_charts - WHERE - chartId = ?`, - [chartId] - ) - const chartViewsPromise = db.knexRaw( - knex, - `-- sql - SELECT cv.id, cv.name, cc.full ->> "$.title" AS title - FROM chart_views cv - JOIN chart_configs cc ON cc.id = cv.chartConfigId - WHERE cv.parentChartId = ?`, - [chartId] - ) - const [postsWordpress, postsGdocs, explorerSlugs, chartViews] = - await Promise.all([ - postsWordpressPromise, - postGdocsPromise, - explorerSlugsPromise, - chartViewsPromise, - ]) - - return { - postsGdocs, - postsWordpress, - explorers: explorerSlugs.map( - (row: { explorerSlug: string }) => row.explorerSlug - ), - chartViews, - } -} - -const expectChartById = async ( - knex: db.KnexReadonlyTransaction, - chartId: any -): Promise => { - const chart = await getChartConfigById(knex, expectInt(chartId)) - if (chart) return chart.config - - throw new JsonError(`No chart found for id ${chartId}`, 404) -} - -const expectPatchConfigByChartId = async ( - knex: db.KnexReadonlyTransaction, - chartId: any -): Promise => { - const patchConfig = await getPatchConfigByChartId(knex, expectInt(chartId)) - if (!patchConfig) { - throw new JsonError(`No chart found for id ${chartId}`, 404) - } - return patchConfig -} - -const saveNewChart = async ( - knex: db.KnexReadWriteTransaction, - { - config, - user, - // new charts inherit by default - shouldInherit = true, - }: { config: GrapherInterface; user: DbPlainUser; shouldInherit?: boolean } -): Promise<{ - chartConfigId: Base64String - patchConfig: GrapherInterface - fullConfig: GrapherInterface -}> => { - // grab the parent of the chart if inheritance should be enabled - const parent = shouldInherit - ? await getParentByChartConfig(knex, config) - : undefined - - // compute patch and full configs - const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) - const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) - - // insert patch & full configs into the chart_configs table - // We can't quite use `saveNewChartConfigInDbAndR2` here, because - // we need to update the chart id in the config after inserting it. - const chartConfigId = uuidv7() as Base64String - await db.knexRaw( - knex, - `-- sql - INSERT INTO chart_configs (id, patch, full) - VALUES (?, ?, ?) - `, - [ - chartConfigId, - serializeChartConfig(patchConfig), - serializeChartConfig(fullConfig), - ] - ) - - // add a new chart to the charts table - const result = await db.knexRawInsert( - knex, - `-- sql - INSERT INTO charts (configId, isInheritanceEnabled, lastEditedAt, lastEditedByUserId) - VALUES (?, ?, ?, ?) - `, - [chartConfigId, shouldInherit, new Date(), user.id] - ) - - // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true - const chartId = result.insertId - patchConfig.id = chartId - fullConfig.id = chartId - await db.knexRaw( - knex, - `-- sql - UPDATE chart_configs cc - JOIN charts c ON c.configId = cc.id - SET - cc.patch=JSON_SET(cc.patch, '$.id', ?), - cc.full=JSON_SET(cc.full, '$.id', ?) - WHERE c.id = ? - `, - [chartId, chartId, chartId] - ) - - await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId) - - return { chartConfigId, patchConfig, fullConfig } -} - -const updateExistingChart = async ( - knex: db.KnexReadWriteTransaction, - params: { - config: GrapherInterface - user: DbPlainUser - chartId: number - // if undefined, keep inheritance as is. - // if true or false, enable or disable inheritance - shouldInherit?: boolean - } -): Promise<{ - chartConfigId: Base64String - patchConfig: GrapherInterface - fullConfig: GrapherInterface -}> => { - const { config, user, chartId } = params - - // make sure that the id of the incoming config matches the chart id - config.id = chartId - - // if inheritance is enabled, grab the parent from its config - const shouldInherit = - params.shouldInherit ?? - (await isInheritanceEnabledForChart(knex, chartId)) - const parent = shouldInherit - ? await getParentByChartConfig(knex, config) - : undefined - - // compute patch and full configs - const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) - const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) - - const chartConfigIdRow = await db.knexRawFirst< - Pick - >(knex, `SELECT configId FROM charts WHERE id = ?`, [chartId]) - - if (!chartConfigIdRow) - throw new JsonError(`No chart config found for id ${chartId}`, 404) - - const now = new Date() - - const { chartConfigId } = await updateChartConfigInDbAndR2( - knex, - chartConfigIdRow.configId as Base64String, - patchConfig, - fullConfig - ) - - // update charts row - await db.knexRaw( - knex, - `-- sql - UPDATE charts - SET isInheritanceEnabled=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=? - WHERE id = ? - `, - [shouldInherit, now, now, user.id, chartId] - ) - - return { chartConfigId, patchConfig, fullConfig } -} - -const saveGrapher = async ( - knex: db.KnexReadWriteTransaction, - { - user, - newConfig, - existingConfig, - shouldInherit, - referencedVariablesMightChange = true, - }: { - user: DbPlainUser - newConfig: GrapherInterface - existingConfig?: GrapherInterface - // if undefined, keep inheritance as is. - // if true or false, enable or disable inheritance - shouldInherit?: boolean - // if the variables a chart uses can change then we need - // to update the latest country data which takes quite a long time (hundreds of ms) - referencedVariablesMightChange?: boolean - } -) => { - // Try to migrate the new config to the latest version - newConfig = migrateGrapherConfigToLatestVersion(newConfig) - - // Slugs need some special logic to ensure public urls remain consistent whenever possible - async function isSlugUsedInRedirect() { - const rows = await db.knexRaw( - knex, - `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`, - // -1 is a placeholder ID that will never exist; but we cannot use NULL because - // in that case we would always get back an empty resultset - [existingConfig ? existingConfig.id : -1, newConfig.slug] - ) - return rows.length > 0 - } - - async function isSlugUsedInOtherGrapher() { - const rows = await db.knexRaw>( - knex, - `-- sql - SELECT c.id - FROM charts c - JOIN chart_configs cc ON cc.id = c.configId - WHERE - c.id != ? - AND cc.full ->> "$.isPublished" = "true" - AND cc.slug = ? - `, - // -1 is a placeholder ID that will never exist; but we cannot use NULL because - // in that case we would always get back an empty resultset - [existingConfig ? existingConfig.id : -1, newConfig.slug] - ) - return rows.length > 0 - } - - // When a chart is published, check for conflicts - if (newConfig.isPublished) { - if (!isValidSlug(newConfig.slug)) - throw new JsonError(`Invalid chart slug ${newConfig.slug}`) - else if (await isSlugUsedInRedirect()) - throw new JsonError( - `This chart slug was previously used by another chart: ${newConfig.slug}` - ) - else if (await isSlugUsedInOtherGrapher()) - throw new JsonError( - `This chart slug is in use by another published chart: ${newConfig.slug}` - ) - else if ( - existingConfig && - existingConfig.isPublished && - existingConfig.slug !== newConfig.slug - ) { - // Changing slug of an existing chart, delete any old redirect and create new one - await db.knexRaw( - knex, - `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`, - [existingConfig.id, existingConfig.slug] - ) - await db.knexRaw( - knex, - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [existingConfig.id, existingConfig.slug] - ) - // When we rename grapher configs, make sure to delete the old one (the new one will be saved below) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${existingConfig.slug}.json` - ) - } - } - - if (existingConfig) - // Bump chart version, very important for cachebusting - newConfig.version = existingConfig.version! + 1 - else if (newConfig.version) - // If a chart is republished, we want to keep incrementing the old version number, - // otherwise it can lead to clients receiving cached versions of the old data. - newConfig.version += 1 - else newConfig.version = 1 - - // add the isPublished field if is missing - if (newConfig.isPublished === undefined) { - newConfig.isPublished = false - } - - // Execute the actual database update or creation - let chartId: number - let chartConfigId: Base64String - let patchConfig: GrapherInterface - let fullConfig: GrapherInterface - if (existingConfig) { - chartId = existingConfig.id! - const configs = await updateExistingChart(knex, { - config: newConfig, - user, - chartId, - shouldInherit, - }) - chartConfigId = configs.chartConfigId - patchConfig = configs.patchConfig - fullConfig = configs.fullConfig - } else { - const configs = await saveNewChart(knex, { - config: newConfig, - user, - shouldInherit, - }) - chartConfigId = configs.chartConfigId - patchConfig = configs.patchConfig - fullConfig = configs.fullConfig - chartId = fullConfig.id! - } - - // Record this change in version history - const chartRevisionLog = { - chartId: chartId as number, - userId: user.id, - config: serializeChartConfig(patchConfig), - createdAt: new Date(), - updatedAt: new Date(), - } satisfies DbInsertChartRevision - await db.knexRaw( - knex, - `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, - [ - chartRevisionLog.chartId, - chartRevisionLog.userId, - chartRevisionLog.config, - chartRevisionLog.createdAt, - chartRevisionLog.updatedAt, - ] - ) - - // Remove any old dimensions and store the new ones - // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json - await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [ - chartId, - ]) - - const newDimensions = fullConfig.dimensions ?? [] - for (const [i, dim] of newDimensions.entries()) { - await db.knexRaw( - knex, - `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?, ?, ?, ?)`, - [chartId, dim.variableId, dim.property, i] - ) - } - - // So we can generate country profiles including this chart data - if (fullConfig.isPublished && referencedVariablesMightChange) - // TODO: remove this ad hoc knex transaction context when we switch the function to knex - await denormalizeLatestCountryData( - knex, - newDimensions.map((d) => d.variableId) - ) - - if (fullConfig.isPublished) { - await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId, { - directory: R2GrapherConfigDirectory.publishedGrapherBySlug, - filename: `${fullConfig.slug}.json`, - }) - } - - if ( - fullConfig.isPublished && - (!existingConfig || !existingConfig.isPublished) - ) { - // Newly published, set publication info - await db.knexRaw( - knex, - `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `, - [new Date(), user.id, chartId] - ) - await triggerStaticBuild(user, `Publishing chart ${fullConfig.slug}`) - } else if ( - !fullConfig.isPublished && - existingConfig && - existingConfig.isPublished - ) { - // Unpublishing chart, delete any existing redirects to it - await db.knexRaw( - knex, - `DELETE FROM chart_slug_redirects WHERE chart_id = ?`, - [existingConfig.id] - ) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${existingConfig.slug}.json` - ) - await triggerStaticBuild(user, `Unpublishing chart ${fullConfig.slug}`) - } else if (fullConfig.isPublished) - await triggerStaticBuild(user, `Updating chart ${fullConfig.slug}`) - - return { - chartId, - savedPatch: patchConfig, - } -} - -async function updateGrapherConfigsInR2( - knex: db.KnexReadonlyTransaction, - updatedCharts: { chartConfigId: string; isPublished: boolean }[], - updatedMultiDimViews: { chartConfigId: string; isPublished: boolean }[] -) { - const idsToUpdate = [ - ...updatedCharts.filter(({ isPublished }) => isPublished), - ...updatedMultiDimViews, - ].map(({ chartConfigId }) => chartConfigId) - const builder = knex(ChartConfigsTableName) - .select("id", "full", "fullMd5") - .whereIn("id", idsToUpdate) - for await (const { id, full, fullMd5 } of builder.stream()) { - await saveGrapherConfigToR2ByUUID(id, full, fullMd5) - } -} - -getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 - const charts = await db.knexRaw( - trx, - `-- sql - SELECT ${oldChartFieldList}, - round(views_365d / 365, 1) as pageviewsPerDay - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN analytics_pageviews on (analytics_pageviews.url = CONCAT("https://ourworldindata.org/grapher/", chart_configs.slug) AND chart_configs.full ->> '$.isPublished' = "true" ) - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - ORDER BY charts.lastEditedAt DESC LIMIT ? - `, - [limit] - ) - - await assignTagsForCharts(trx, charts) - - return { charts } -}) - -getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 - - // note: this query is extended from OldChart.listFields. - const charts = await db.knexRaw( - trx, - `-- sql - SELECT - charts.id, - chart_configs.full->>"$.version" AS version, - CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url, - CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl, - chart_configs.full->>"$.slug" AS slug, - chart_configs.full->>"$.title" AS title, - chart_configs.full->>"$.subtitle" AS subtitle, - chart_configs.full->>"$.sourceDesc" AS sourceDesc, - chart_configs.full->>"$.note" AS note, - chart_configs.chartType AS type, - chart_configs.full->>"$.internalNotes" AS internalNotes, - chart_configs.full->>"$.variantName" AS variantName, - chart_configs.full->>"$.isPublished" AS isPublished, - chart_configs.full->>"$.tab" AS tab, - chart_configs.chartType IS NOT NULL AS hasChartTab, - JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab, - chart_configs.full->>"$.originUrl" AS originUrl, - charts.lastEditedAt, - charts.lastEditedByUserId, - lastEditedByUser.fullName AS lastEditedBy, - charts.publishedAt, - charts.publishedByUserId, - publishedByUser.fullName AS publishedBy - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - ORDER BY charts.lastEditedAt DESC - LIMIT ? - `, - [limit] - ) - // note: retrieving references is VERY slow. - // await Promise.all( - // charts.map(async (chart: any) => { - // const references = await getReferencesByChartId(chart.id) - // chart.references = references.length - // ? references.map((ref) => ref.url) - // : "" - // }) - // ) - // await Chart.assignTagsForCharts(charts) - res.setHeader("Content-disposition", "attachment; filename=charts.csv") - res.setHeader("content-type", "text/csv") - const csv = Papa.unparse(charts) - return csv -}) - + getChartsJson, + getChartsCsv, + getChartConfigJson, + getChartParentJson, + getChartPatchConfigJson, + getChartLogsJson, + getChartReferencesJson, + getChartRedirectsJson, + getChartPageviewsJson, + createChart, + setChartTagsHandler, + updateChart, + deleteChart, +} from "./apiRoutes/charts.js" +import e, { Router } from "express" + +const apiRouter = Router() + +// Bulk chart update routes +patchRouteWithRWTransaction( + apiRouter, + "/variable-annotations", + updateVariableAnnotations +) +getRouteWithROTransaction(apiRouter, "/chart-bulk-update", getChartBulkUpdate) +patchRouteWithRWTransaction( + apiRouter, + "/chart-bulk-update", + updateBulkChartConfigs +) getRouteWithROTransaction( apiRouter, - "/charts/:chartId.config.json", - async (req, res, trx) => expectChartById(trx, req.params.chartId) + "/variable-annotations", + getVariableAnnotations ) +// Chart routes +getRouteWithROTransaction(apiRouter, "/charts.json", getChartsJson) +getRouteWithROTransaction(apiRouter, "/charts.csv", getChartsCsv) getRouteWithROTransaction( apiRouter, - "/charts/:chartId.parent.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const parent = await getParentByChartId(trx, chartId) - const isInheritanceEnabled = await isInheritanceEnabledForChart( - trx, - chartId - ) - return omitUndefinedValues({ - variableId: parent?.variableId, - config: parent?.config, - isActive: isInheritanceEnabled, - }) - } + "/charts/:chartId.config.json", + getChartConfigJson ) - getRouteWithROTransaction( apiRouter, - "/charts/:chartId.patchConfig.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const config = await expectPatchConfigByChartId(trx, chartId) - return config - } + "/charts/:chartId.parent.json", + getChartParentJson ) - getRouteWithROTransaction( apiRouter, - "/editorData/namespaces.json", - async (req, res, trx) => { - const rows = await db.knexRaw<{ - name: string - description?: string - isArchived: boolean - }>( - trx, - `SELECT DISTINCT - namespace AS name, - namespaces.description AS description, - namespaces.isArchived AS isArchived - FROM active_datasets - JOIN namespaces ON namespaces.name = active_datasets.namespace` - ) - - return { - namespaces: lodash - .sortBy(rows, (row) => row.description) - .map((namespace) => ({ - ...namespace, - isArchived: !!namespace.isArchived, - })), - } - } + "/charts/:chartId.patchConfig.json", + getChartPatchConfigJson ) - getRouteWithROTransaction( apiRouter, "/charts/:chartId.logs.json", - async (req, res, trx) => ({ - logs: await getLogsByChartId( - trx, - parseInt(req.params.chartId as string) - ), - }) + getChartLogsJson ) - getRouteWithROTransaction( apiRouter, "/charts/:chartId.references.json", - async (req, res, trx) => { - const references = { - references: await getReferencesByChartId( - parseInt(req.params.chartId as string), - trx - ), - } - return references - } + getChartReferencesJson ) - getRouteWithROTransaction( apiRouter, "/charts/:chartId.redirects.json", - async (req, res, trx) => ({ - redirects: await getRedirectsByChartId( - trx, - parseInt(req.params.chartId as string) - ), - }) + getChartRedirectsJson ) - getRouteWithROTransaction( apiRouter, "/charts/:chartId.pageviews.json", - async (req, res, trx) => { - const slug = await getChartSlugById( - trx, - parseInt(req.params.chartId as string) - ) - if (!slug) return {} - - const pageviewsByUrl = await db.knexRawFirst( - trx, - `-- sql - SELECT * - FROM - analytics_pageviews - WHERE - url = ?`, - [`https://ourworldindata.org/grapher/${slug}`] - ) - - return { - pageviews: pageviewsByUrl ?? undefined, - } - } -) - -getRouteWithROTransaction( - apiRouter, - "/editorData/variables.json", - async (req, res, trx) => { - const datasets = [] - const rows = await db.knexRaw< - Pick & { - datasetId: number - datasetName: string - datasetVersion: string - } & Pick< - DbPlainDataset, - "namespace" | "isPrivate" | "nonRedistributable" - > - >( - trx, - `-- sql - SELECT - v.name, - v.id, - d.id as datasetId, - d.name as datasetName, - d.version as datasetVersion, - d.namespace, - d.isPrivate, - d.nonRedistributable - FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id - ORDER BY d.updatedAt DESC - ` - ) - - let dataset: - | { - id: number - name: string - version: string - namespace: string - isPrivate: boolean - nonRedistributable: boolean - variables: { id: number; name: string }[] - } - | undefined - for (const row of rows) { - if (!dataset || row.datasetName !== dataset.name) { - if (dataset) datasets.push(dataset) - - dataset = { - id: row.datasetId, - name: row.datasetName, - version: row.datasetVersion, - namespace: row.namespace, - isPrivate: !!row.isPrivate, - nonRedistributable: !!row.nonRedistributable, - variables: [], - } - } - - dataset.variables.push({ - id: row.id, - name: row.name ?? "", - }) - } - - if (dataset) datasets.push(dataset) - - return { datasets: datasets } - } -) - -apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { - const variableStr = req.params.variableStr as string - if (!variableStr) throw new JsonError("No variable id given") - if (variableStr.includes("+")) - throw new JsonError( - "Requesting multiple variables at the same time is no longer supported" - ) - const variableId = parseInt(variableStr) - if (isNaN(variableId)) throw new JsonError("Invalid variable id") - return await fetchS3DataValuesByPath( - getVariableDataRoute(DATA_API_URL, variableId) + "?nocache" - ) -}) - -apiRouter.get( - "/data/variables/metadata/:variableStr.json", - async (req, res) => { - const variableStr = req.params.variableStr as string - if (!variableStr) throw new JsonError("No variable id given") - if (variableStr.includes("+")) - throw new JsonError( - "Requesting multiple variables at the same time is no longer supported" - ) - const variableId = parseInt(variableStr) - if (isNaN(variableId)) throw new JsonError("Invalid variable id") - return await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" - ) - } + getChartPageviewsJson ) - -postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { - let shouldInherit: boolean | undefined - if (req.query.inheritance) { - shouldInherit = req.query.inheritance === "enable" - } - - try { - const { chartId } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - shouldInherit, - }) - - return { success: true, chartId: chartId } - } catch (err) { - return { success: false, error: String(err) } - } -}) - +postRouteWithRWTransaction(apiRouter, "/charts", createChart) postRouteWithRWTransaction( apiRouter, "/charts/:chartId/setTags", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - - await setChartTags(trx, chartId, req.body.tags) - - return { success: true } - } + setChartTagsHandler +) +putRouteWithRWTransaction(apiRouter, "/charts/:chartId", updateChart) +deleteRouteWithRWTransaction(apiRouter, "/charts/:chartId", deleteChart) + +// Chart view routes +getRouteWithROTransaction(apiRouter, "/chartViews", getChartViews) +getRouteWithROTransaction(apiRouter, "/chartViews/:id", getChartViewById) +postRouteWithRWTransaction(apiRouter, "/chartViews", createChartView) +putRouteWithRWTransaction(apiRouter, "/chartViews/:id", updateChartView) +deleteRouteWithRWTransaction(apiRouter, "/chartViews/:id", deleteChartView) + +// Dataset routes +getRouteWithROTransaction(apiRouter, "/datasets.json", getDatasets) +getRouteWithROTransaction(apiRouter, "/datasets/:datasetId.json", getDataset) +putRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", updateDataset) +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/setArchived", + setArchived ) - -putRouteWithRWTransaction( +postRouteWithRWTransaction(apiRouter, "/datasets/:datasetId/setTags", setTags) +deleteRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", deleteDataset) +postRouteWithRWTransaction( apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - let shouldInherit: boolean | undefined - if (req.query.inheritance) { - shouldInherit = req.query.inheritance === "enable" - } - - const existingConfig = await expectChartById(trx, req.params.chartId) - - try { - const { chartId, savedPatch } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - existingConfig, - shouldInherit, - }) - - const logs = await getLogsByChartId( - trx, - existingConfig.id as number - ) - return { - success: true, - chartId, - savedPatch, - newLog: logs[0], - } - } catch (err) { - return { - success: false, - error: String(err), - } - } - } + "/datasets/:datasetId/charts", + republishCharts ) +// explorer routes +postRouteWithRWTransaction(apiRouter, "/explorer/:slug/tags", addExplorerTags) deleteRouteWithRWTransaction( apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - const chart = await expectChartById(trx, req.params.chartId) - if (chart.slug) { - const links = await getPublishedLinksTo(trx, [chart.slug]) - if (links.length) { - const sources = links.map((link) => link.sourceSlug).join(", ") - throw new Error( - `Cannot delete chart in-use in the following published documents: ${sources}` - ) - } - } - - await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ - chart.id, - ]) - await db.knexRaw( - trx, - `DELETE FROM chart_slug_redirects WHERE chart_id=?`, - [chart.id] - ) - - const row = await db.knexRawFirst>( - trx, - `SELECT configId FROM charts WHERE id = ?`, - [chart.id] - ) - if (!row || !row.configId) - throw new JsonError(`No chart config found for id ${chart.id}`, 404) - if (row) { - await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) - await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ - row.configId, - ]) - } - - if (chart.isPublished) - await triggerStaticBuild( - res.locals.user, - `Deleting chart ${chart.slug}` - ) + "/explorer/:slug/tags", + deleteExplorerTags +) - await deleteGrapherConfigFromR2ByUUID(row.configId) - if (chart.isPublished) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${chart.slug}.json` - ) +// Gdoc routes +getRouteWithROTransaction(apiRouter, "/gdocs", getAllGdocIndexItems) +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/gdocs/:id", + getIndividualGdoc +) +putRouteWithRWTransaction(apiRouter, "/gdocs/:id", createOrUpdateGdoc) +deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", deleteGdoc) +postRouteWithRWTransaction(apiRouter, "/gdocs/:gdocId/setTags", setGdocTags) - return { success: true } - } +// Images routes +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/images.json", + getImagesHandler ) +postRouteWithRWTransaction(apiRouter, "/images", postImageHandler) +putRouteWithRWTransaction(apiRouter, "/images/:id", putImageHandler) +// Update alt text via patch +patchRouteWithRWTransaction(apiRouter, "/images/:id", patchImageHandler) +deleteRouteWithRWTransaction(apiRouter, "/images/:id", deleteImageHandler) +getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler) +// Mdim routes putRouteWithRWTransaction( apiRouter, "/multi-dim/:slug", - async (req, res, trx) => { - const { slug } = req.params - if (!isValidSlug(slug)) { - throw new JsonError(`Invalid multi-dim slug ${slug}`) - } - const rawConfig = req.body as MultiDimDataPageConfigRaw - const id = await createMultiDimConfig(trx, slug, rawConfig) - if ( - FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && - (await isMultiDimDataPagePublished(trx, slug)) - ) { - await triggerStaticBuild( - res.locals.user, - `Publishing multidimensional chart ${slug}` - ) - } - return { success: true, id } - } + handleMultiDimDataPageRequest ) -getRouteWithROTransaction(apiRouter, "/users.json", async (req, res, trx) => ({ - users: await trx - .select( - "id" satisfies keyof DbPlainUser, - "email" satisfies keyof DbPlainUser, - "fullName" satisfies keyof DbPlainUser, - "isActive" satisfies keyof DbPlainUser, - "isSuperuser" satisfies keyof DbPlainUser, - "createdAt" satisfies keyof DbPlainUser, - "updatedAt" satisfies keyof DbPlainUser, - "lastLogin" satisfies keyof DbPlainUser, - "lastSeen" satisfies keyof DbPlainUser - ) - .from(UsersTableName) - .orderBy("lastSeen", "desc"), -})) - +// Misc routes +getRouteWithROTransaction(apiRouter, "/all-work", fetchAllWork) getRouteWithROTransaction( apiRouter, - "/users/:userId.json", - async (req, res, trx) => { - const id = parseIntOrUndefined(req.params.userId) - if (!id) throw new JsonError("No user id given") - const user = await getUserById(trx, id) - return { user } - } + "/editorData/namespaces.json", + fetchNamespaces ) +getRouteWithROTransaction(apiRouter, "/sources/:sourceId.json", fetchSourceById) -deleteRouteWithRWTransaction( +// Wordpress posts routes +getRouteWithROTransaction(apiRouter, "/posts.json", handleGetPostsJson) +postRouteWithRWTransaction( apiRouter, - "/users/:userId", - async (req, res, trx) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = expectInt(req.params.userId) - await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) - - return { success: true } - } + "/posts/:postId/setTags", + handleSetTagsForPost ) - -putRouteWithRWTransaction( +getRouteWithROTransaction(apiRouter, "/posts/:postId.json", handleGetPostById) +postRouteWithRWTransaction( apiRouter, - "/users/:userId", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = parseIntOrUndefined(req.params.userId) - const user = - userId !== undefined ? await getUserById(trx, userId) : null - if (!user) throw new JsonError("No such user", 404) - - user.fullName = req.body.fullName - user.isActive = req.body.isActive - - await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) - - return { success: true } - } + "/posts/:postId/createGdoc", + handleCreateGdoc ) - postRouteWithRWTransaction( apiRouter, - "/users/add", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const { email, fullName } = req.body - - await insertUser(trx, { - email, - fullName, - }) - - return { success: true } - } + "/posts/:postId/unlinkGdoc", + handleUnlinkGdoc ) +// Redirects routes +getRouteWithROTransaction( + apiRouter, + "/site-redirects.json", + handleGetSiteRedirects +) postRouteWithRWTransaction( apiRouter, - "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images").where({ id: imageId }).update({ userId }) - return { success: true } - } + "/site-redirects/new", + handlePostNewSiteRedirect ) - deleteRouteWithRWTransaction( apiRouter, - "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images") - .where({ id: imageId, userId }) - .update({ userId: null }) - return { success: true } - } + "/site-redirects/:id", + handleDeleteSiteRedirect +) +getRouteWithROTransaction(apiRouter, "/redirects.json", handleGetRedirects) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/redirects/new", + handlePostNewChartRedirect +) +deleteRouteWithRWTransaction( + apiRouter, + "/redirects/:id", + handleDeleteChartRedirect ) +// GPT routes getRouteWithROTransaction( apiRouter, - "/variables.json", - async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 - const query = req.query.search as string - return await searchVariables(query, limit, trx) - } + `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, + suggestGptTopics ) - getRouteWithROTransaction( apiRouter, - "/chart-bulk-update", - async ( - req, - res, - trx - ): Promise> => { - const context: OperationContext = { - grapherConfigFieldName: "chart_configs.full", - whitelistedColumnNamesAndTypes: - chartBulkUpdateAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined - - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql - SELECT - charts.id as id, - chart_configs.full as config, - charts.createdAt as createdAt, - charts.updatedAt as updatedAt, - charts.lastEditedAt as lastEditedAt, - charts.publishedAt as publishedAt, - lastEditedByUser.fullName as lastEditedByUser, - publishedByUser.fullName as publishedByUser - FROM charts - LEFT JOIN chart_configs ON chart_configs.id = charts.configId - LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId - WHERE ${whereClause} - ORDER BY charts.id DESC - LIMIT 50 - OFFSET ${offset.toString()} - ` - ) - - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql - SELECT count(*) as count - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - WHERE ${whereClause} - ` - ) - return { rows: results, numTotalRows: resultCount[0].count } - } + `/gpt/suggest-alt-text/:imageId`, + suggestGptAltText ) -patchRouteWithRWTransaction( +// Tag graph routes +getRouteWithROTransaction( apiRouter, - "/chart-bulk-update", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const chartIds = new Set(patchesList.map((patch) => patch.id)) - - const configsAndIds = await db.knexRaw< - Pick & { config: DbRawChartConfig["full"] } - >( - trx, - `-- sql - SELECT c.id, cc.full as config - FROM charts c - JOIN chart_configs cc ON cc.id = c.configId - WHERE c.id IN (?) - `, - [[...chartIds.values()]] - ) - const configMap = new Map( - configsAndIds.map((item: any) => [ - item.id, - // make sure that the id is set, otherwise the update behaviour is weird - // TODO: discuss if this has unintended side effects - item.config ? { ...JSON.parse(item.config), id: item.id } : {}, - ]) - ) - const oldValuesConfigMap = new Map(configMap) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } - - for (const [id, newConfig] of configMap.entries()) { - await saveGrapher(trx, { - user: res.locals.user, - newConfig, - existingConfig: oldValuesConfigMap.get(id), - referencedVariablesMightChange: false, - }) - } - - return { success: true } - } + "/flatTagGraph.json", + handleGetFlatTagGraph +) +postRouteWithRWTransaction(apiRouter, "/tagGraph", handlePostTagGraph) +getRouteWithROTransaction(apiRouter, "/tags/:tagId.json", getTagById) +putRouteWithRWTransaction(apiRouter, "/tags/:tagId", updateTag) +postRouteWithRWTransaction(apiRouter, "/tags/new", createTag) +getRouteWithROTransaction(apiRouter, "/tags.json", getAllTags) +deleteRouteWithRWTransaction(apiRouter, "/tags/:tagId/delete", deleteTag) + +// User routes +getRouteWithROTransaction(apiRouter, "/users.json", getUsers) +getRouteWithROTransaction(apiRouter, "/users/:userId.json", getUserByIdHandler) +deleteRouteWithRWTransaction(apiRouter, "/users/:userId", deleteUser) +putRouteWithRWTransaction(apiRouter, "/users/:userId", updateUserHandler) +postRouteWithRWTransaction(apiRouter, "/users/add", addUser) +postRouteWithRWTransaction( + apiRouter, + "/users/:userId/images/:imageId", + addImageToUser +) +deleteRouteWithRWTransaction( + apiRouter, + "/users/:userId/images/:imageId", + removeUserImage ) +// Variable routes getRouteWithROTransaction( apiRouter, - "/variable-annotations", - async ( - req, - res, - trx - ): Promise> => { - const context: OperationContext = { - grapherConfigFieldName: "grapherConfigAdmin", - whitelistedColumnNamesAndTypes: - variableAnnotationAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined - - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql - SELECT - variables.id as id, - variables.name as name, - chart_configs.patch as config, - d.name as datasetname, - namespaces.name as namespacename, - variables.createdAt as createdAt, - variables.updatedAt as updatedAt, - variables.description as description - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ORDER BY variables.id DESC - LIMIT 50 - OFFSET ${offset.toString()} - ` - ) - - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql - SELECT count(*) as count - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ` - ) - return { rows: results, numTotalRows: resultCount[0].count } - } + "/editorData/variables.json", + getEditorVariablesJson ) - -patchRouteWithRWTransaction( +getRouteWithROTransaction( apiRouter, - "/variable-annotations", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const variableIds = new Set(patchesList.map((patch) => patch.id)) - - const configsAndIds = await db.knexRaw< - Pick & { - grapherConfigAdmin: DbRawChartConfig["patch"] - } - >( - trx, - `-- sql - SELECT v.id, cc.patch AS grapherConfigAdmin - FROM variables v - LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id - WHERE v.id IN (?) - `, - [[...variableIds.values()]] - ) - const configMap = new Map( - configsAndIds.map((item: any) => [ - item.id, - item.grapherConfigAdmin - ? JSON.parse(item.grapherConfigAdmin) - : {}, - ]) - ) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } - - for (const [variableId, newConfig] of configMap.entries()) { - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) continue - await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) - } - - return { success: true } - } + "/data/variables/data/:variableStr.json", + getVariableDataJson ) - +getRouteWithROTransaction( + apiRouter, + "/data/variables/metadata/:variableStr.json", + getVariableMetadataJson +) +getRouteWithROTransaction(apiRouter, "/variables.json", getVariablesJson) getRouteWithROTransaction( apiRouter, "/variables.usages.json", - async (req, res, trx) => { - const query = `-- sql - SELECT - variableId, - COUNT(DISTINCT chartId) AS usageCount - FROM - chart_dimensions - GROUP BY - variableId - ORDER BY - usageCount DESC` - - const rows = await db.knexRaw(trx, query) - - return rows - } + getVariablesUsagesJson ) - getRouteWithROTransaction( apiRouter, "/variables/grapherConfigETL/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.etl?.patchConfig ?? {} - } + getVariablesGrapherConfigETLPatchConfigJson ) - getRouteWithROTransaction( apiRouter, "/variables/grapherConfigAdmin/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.admin?.patchConfig ?? {} - } + getVariablesGrapherConfigAdminPatchConfigJson ) - getRouteWithROTransaction( apiRouter, "/variables/mergedGrapherConfig/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const config = await getMergedGrapherConfigForVariable(trx, variableId) - return config ?? {} - } + getVariablesMergedGrapherConfigJson ) - // Used in VariableEditPage getRouteWithROTransaction( apiRouter, "/variables/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" - ) - - // XXX: Patch shortName onto the end of catalogPath when it's missing, - // a temporary hack since our S3 metadata is out of date with our DB. - // See: https://github.com/owid/etl/issues/2135 - if (variable.catalogPath && !variable.catalogPath.includes("#")) { - variable.catalogPath += `#${variable.shortName}` - } - - const rawCharts = await db.knexRaw< - OldChartFieldList & { - isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] - config: DbRawChartConfig["full"] - } - >( - trx, - `-- sql - SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - JOIN chart_dimensions cd ON cd.chartId = charts.id - WHERE cd.variableId = ? - GROUP BY charts.id - `, - [variableId] - ) - - // check for parent indicators - const charts = rawCharts.map((chart) => { - const parentIndicatorId = getParentVariableIdFromChartConfig( - parseChartConfig(chart.config) - ) - const hasParentIndicator = parentIndicatorId !== undefined - return omit({ ...chart, hasParentIndicator }, "config") - }) - - await assignTagsForCharts(trx, charts) - - const variableWithConfigs = await getGrapherConfigsForVariable( - trx, - variableId - ) - const grapherConfigETL = variableWithConfigs?.etl?.patchConfig - const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig - const mergedGrapherConfig = - variableWithConfigs?.admin?.fullConfig ?? - variableWithConfigs?.etl?.fullConfig - - // add the variable's display field to the merged grapher config - if (mergedGrapherConfig) { - const [varDims, otherDims] = lodash.partition( - mergedGrapherConfig.dimensions ?? [], - (dim) => dim.variableId === variableId - ) - const varDimsWithDisplay = varDims.map((dim) => ({ - display: variable.display, - ...dim, - })) - mergedGrapherConfig.dimensions = [ - ...varDimsWithDisplay, - ...otherDims, - ] - } - - const variableWithCharts: OwidVariableWithSource & { - charts: Record - grapherConfig: GrapherInterface | undefined - grapherConfigETL: GrapherInterface | undefined - grapherConfigAdmin: GrapherInterface | undefined - } = { - ...variable, - charts, - grapherConfig: mergedGrapherConfig, - grapherConfigETL, - grapherConfigAdmin, - } - - return { - variable: variableWithCharts, - } /*, vardata: await getVariableData([variableId]) }*/ - } + getVariablesVariableIdJson ) - // inserts a new config or updates an existing one putRouteWithRWTransaction( apiRouter, "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigETLOfVariable(trx, variable, validConfig) - - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } - - return { success: true, savedPatch } - } + putVariablesVariableIdGrapherConfigETL ) - deleteRouteWithRWTransaction( apiRouter, "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - // no-op if the variable doesn't have an ETL config - if (!variable.etl) return { success: true } - - const now = new Date() - - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql - UPDATE variables - SET grapherConfigIdETL = NULL - WHERE id = ? - `, - [variableId] - ) - - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql - DELETE FROM chart_configs - WHERE id = ? - `, - [variable.etl.configId] - ) - - // update admin config if there is one - if (variable.admin) { - await updateExistingFullConfig(trx, { - configId: variable.admin.configId, - config: variable.admin.patchConfig, - updatedAt: now, - }) - } - - const updates = { - patchConfigAdmin: variable.admin?.patchConfig, - updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( - trx, - variableId, - updates - ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } - - return { success: true } - } + deleteVariablesVariableIdGrapherConfigETL ) - // inserts a new config or updates an existing one putRouteWithRWTransaction( apiRouter, "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) - - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } - - return { success: true, savedPatch } - } + putVariablesVariableIdGrapherConfigAdmin ) - deleteRouteWithRWTransaction( apiRouter, "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - // no-op if the variable doesn't have an admin-authored config - if (!variable.admin) return { success: true } - - const now = new Date() - - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql - UPDATE variables - SET grapherConfigIdAdmin = NULL - WHERE id = ? - `, - [variableId] - ) - - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql - DELETE FROM chart_configs - WHERE id = ? - `, - [variable.admin.configId] - ) - - const updates = { - patchConfigETL: variable.etl?.patchConfig, - updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( - trx, - variableId, - updates - ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } - - return { success: true } - } + deleteVariablesVariableIdGrapherConfigAdmin ) - getRouteWithROTransaction( apiRouter, "/variables/:variableId/charts.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const charts = await getAllChartsForIndicator(trx, variableId) - return charts.map((chart) => ({ - id: chart.chartId, - title: chart.config.title, - variantName: chart.config.variantName, - isChild: chart.isChild, - isInheritanceEnabled: chart.isInheritanceEnabled, - isPublished: chart.isPublished, - })) - } -) - -getRouteWithROTransaction( - apiRouter, - "/datasets.json", - async (req, res, trx) => { - const datasets = await db.knexRaw>( - trx, - `-- sql - WITH variable_counts AS ( - SELECT - v.datasetId, - COUNT(DISTINCT cd.chartId) as numCharts - FROM chart_dimensions cd - JOIN variables v ON cd.variableId = v.id - GROUP BY v.datasetId - ) - SELECT - ad.id, - ad.namespace, - ad.name, - d.shortName, - ad.description, - ad.dataEditedAt, - du.fullName AS dataEditedByUserName, - ad.metadataEditedAt, - mu.fullName AS metadataEditedByUserName, - ad.isPrivate, - ad.nonRedistributable, - d.version, - vc.numCharts - FROM active_datasets ad - LEFT JOIN variable_counts vc ON ad.id = vc.datasetId - JOIN users du ON du.id=ad.dataEditedByUserId - JOIN users mu ON mu.id=ad.metadataEditedByUserId - JOIN datasets d ON d.id=ad.id - ORDER BY ad.dataEditedAt DESC - ` - ) - - const tags = await db.knexRaw< - Pick & - Pick - >( - trx, - `-- sql - SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt - JOIN tags t ON dt.tagId = t.id - ` - ) - const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) - for (const dataset of datasets) { - dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => - lodash.omit(t, "datasetId") - ) - } - /*LEFT JOIN variables AS v ON v.datasetId=d.id - GROUP BY d.id*/ - - return { datasets: datasets } - } -) - -getRouteWithROTransaction( - apiRouter, - "/datasets/:datasetId.json", - async (req: Request, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await db.knexRawFirst>( - trx, - `-- sql - SELECT d.id, - d.namespace, - d.name, - d.shortName, - d.version, - d.description, - d.updatedAt, - d.dataEditedAt, - d.dataEditedByUserId, - du.fullName AS dataEditedByUserName, - d.metadataEditedAt, - d.metadataEditedByUserId, - mu.fullName AS metadataEditedByUserName, - d.isPrivate, - d.isArchived, - d.nonRedistributable, - d.updatePeriodDays - FROM datasets AS d - JOIN users du ON du.id=d.dataEditedByUserId - JOIN users mu ON mu.id=d.metadataEditedByUserId - WHERE d.id = ? - `, - [datasetId] - ) - - if (!dataset) - throw new JsonError(`No dataset by id '${datasetId}'`, 404) - - const zipFile = await db.knexRawFirst<{ filename: string }>( - trx, - `SELECT filename FROM dataset_files WHERE datasetId=?`, - [datasetId] - ) - if (zipFile) dataset.zipFile = zipFile - - const variables = await db.knexRaw< - Pick< - DbRawVariable, - "id" | "name" | "description" | "display" | "catalogPath" - > - >( - trx, - `-- sql - SELECT - v.id, - v.name, - v.description, - v.display, - v.catalogPath - FROM - variables AS v - WHERE - v.datasetId = ? - `, - [datasetId] - ) - - for (const v of variables) { - v.display = JSON.parse(v.display) - } - - dataset.variables = variables - - // add all origins - const origins: DbRawOrigin[] = await db.knexRaw( - trx, - `-- sql - SELECT DISTINCT - o.* - FROM - origins_variables AS ov - JOIN origins AS o ON ov.originId = o.id - JOIN variables AS v ON ov.variableId = v.id - WHERE - v.datasetId = ? - `, - [datasetId] - ) - - const parsedOrigins = origins.map(parseOriginsRow) - - dataset.origins = parsedOrigins - - const sources = await db.knexRaw<{ - id: number - name: string - description: string - }>( - trx, - ` - SELECT s.id, s.name, s.description - FROM sources AS s - WHERE s.datasetId = ? - ORDER BY s.id ASC - `, - [datasetId] - ) - - // expand description of sources and add to dataset as variableSources - dataset.variableSources = sources.map((s: any) => { - return { - id: s.id, - name: s.name, - ...JSON.parse(s.description), - } - }) - - const charts = await db.knexRaw( - trx, - `-- sql - SELECT ${oldChartFieldList} - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN chart_dimensions AS cd ON cd.chartId = charts.id - JOIN variables AS v ON cd.variableId = v.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE v.datasetId = ? - GROUP BY charts.id - `, - [datasetId] - ) - - dataset.charts = charts - - await assignTagsForCharts(trx, charts) - - const tags = await db.knexRaw<{ id: number; name: string }>( - trx, - ` - SELECT t.id, t.name - FROM tags t - JOIN dataset_tags dt ON dt.tagId = t.id - WHERE dt.datasetId = ? - `, - [datasetId] - ) - dataset.tags = tags - - const availableTags = await db.knexRaw<{ - id: number - name: string - parentName: string - }>( - trx, - ` - SELECT t.id, t.name, p.name AS parentName - FROM tags AS t - JOIN tags AS p ON t.parentId=p.id - ` - ) - dataset.availableTags = availableTags - - return { dataset: dataset } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - // Only updates `nonRedistributable` and `tags`, other fields come from ETL - // and are not editable - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - const newDataset = (req.body as { dataset: any }).dataset - await db.knexRaw( - trx, - ` - UPDATE datasets - SET - nonRedistributable=?, - metadataEditedAt=?, - metadataEditedByUserId=? - WHERE id=? - `, - [ - newDataset.nonRedistributable, - new Date(), - res.locals.user.id, - datasetId, - ] - ) - - const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) - await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ - datasetId, - ]) - if (tagRows.length) - for (const tagRow of tagRows) { - await db.knexRaw( - trx, - `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, - tagRow - ) - } - - try { - await syncDatasetToGitRepo(trx, datasetId, { - oldDatasetName: dataset.name, - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue - } - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setArchived", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ - datasetId, - ]) - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setTags", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - await setTagsForDataset(trx, datasetId, req.body.tagIds) - - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - await db.knexRaw( - trx, - `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, - [datasetId] - ) - await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) - - try { - await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err: any) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue - } - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/charts", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - if (req.body.republish) { - await db.knexRaw( - trx, - `-- sql - UPDATE chart_configs cc - JOIN charts c ON c.configId = cc.id - SET - cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), - cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) - WHERE c.id IN ( - SELECT DISTINCT chart_dimensions.chartId - FROM chart_dimensions - JOIN variables ON variables.id = chart_dimensions.variableId - WHERE variables.datasetId = ? - )`, - [datasetId] - ) - } - - await triggerStaticBuild( - res.locals.user, - `Republishing all charts in dataset ${dataset.name} (${dataset.id})` - ) - - return { success: true } - } -) - -// Get a list of redirects that map old slugs to charts -getRouteWithROTransaction( - apiRouter, - "/redirects.json", - async (req, res, trx) => ({ - redirects: await db.knexRaw( - trx, - `-- sql - SELECT - r.id, - r.slug, - r.chart_id as chartId, - chart_configs.slug AS chartSlug - FROM chart_slug_redirects AS r - JOIN charts ON charts.id = r.chart_id - JOIN chart_configs ON chart_configs.id = charts.configId - ORDER BY r.id DESC - ` - ), - }) -) - -getRouteWithROTransaction( - apiRouter, - "/site-redirects.json", - async (req, res, trx) => ({ redirects: await getRedirects(trx) }) -) - -postRouteWithRWTransaction( - apiRouter, - "/site-redirects/new", - async (req: Request, res, trx) => { - const { source, target } = req.body - const sourceAsUrl = new URL(source, "https://ourworldindata.org") - if (sourceAsUrl.pathname === "/") - throw new JsonError("Cannot redirect from /", 400) - if (await redirectWithSourceExists(trx, source)) { - throw new JsonError( - `Redirect with source ${source} already exists`, - 400 - ) - } - const chainedRedirect = await getChainedRedirect(trx, source, target) - if (chainedRedirect) { - throw new JsonError( - "Creating this redirect would create a chain, redirect from " + - `${chainedRedirect.source} to ${chainedRedirect.target} ` + - "already exists. " + - (target === chainedRedirect.source - ? `Please create the redirect from ${source} to ` + - `${chainedRedirect.target} directly instead.` - : `Please delete the existing redirect and create a ` + - `new redirect from ${chainedRedirect.source} to ` + - `${target} instead.`), - 400 - ) - } - const { insertId: id } = await db.knexRawInsert( - trx, - `INSERT INTO redirects (source, target) VALUES (?, ?)`, - [source, target] - ) - await triggerStaticBuild( - res.locals.user, - `Creating redirect id=${id} source=${source} target=${target}` - ) - return { success: true, redirect: { id, source, target } } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/site-redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const redirect = await getRedirectById(trx, id) - if (!redirect) { - throw new JsonError(`No redirect found for id ${id}`, 404) - } - await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` - ) - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/tags/:tagId.json", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) as number | null - - // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff - // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag - // every time we create a new chart etcs - const uncategorized = tagId === UNCATEGORIZED_TAG_ID - - // TODO: when we have types for our endpoints, make tag of that type instead of any - const tag: any = await db.knexRawFirst< - Pick< - DbPlainTag, - | "id" - | "name" - | "specialType" - | "updatedAt" - | "parentId" - | "slug" - > - >( - trx, - `-- sql - SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug - FROM tags t LEFT JOIN tags p ON t.parentId=p.id - WHERE t.id = ? - `, - [tagId] - ) - - // Datasets tagged with this tag - const datasets = await db.knexRaw< - Pick< - DbPlainDataset, - | "id" - | "namespace" - | "name" - | "description" - | "createdAt" - | "updatedAt" - | "dataEditedAt" - | "isPrivate" - | "nonRedistributable" - > & { dataEditedByUserName: string } - >( - trx, - `-- sql - SELECT - d.id, - d.namespace, - d.name, - d.description, - d.createdAt, - d.updatedAt, - d.dataEditedAt, - du.fullName AS dataEditedByUserName, - d.isPrivate, - d.nonRedistributable - FROM active_datasets d - JOIN users du ON du.id=d.dataEditedByUserId - LEFT JOIN dataset_tags dt ON dt.datasetId = d.id - WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} - ORDER BY d.dataEditedAt DESC - `, - uncategorized ? [] : [tagId] - ) - tag.datasets = datasets - - // The other tags for those datasets - if (tag.datasets.length) { - if (uncategorized) { - for (const dataset of tag.datasets) dataset.tags = [] - } else { - const datasetTags = await db.knexRaw<{ - datasetId: number - id: number - name: string - }>( - trx, - `-- sql - SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt - JOIN tags t ON dt.tagId = t.id - WHERE dt.datasetId IN (?) - `, - [tag.datasets.map((d: any) => d.id)] - ) - const tagsByDatasetId = lodash.groupBy( - datasetTags, - (t) => t.datasetId - ) - for (const dataset of tag.datasets) { - dataset.tags = tagsByDatasetId[dataset.id].map((t) => - lodash.omit(t, "datasetId") - ) - } - } - } - - // Charts using datasets under this tag - const charts = await db.knexRaw( - trx, - `-- sql - SELECT ${oldChartFieldList} FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - LEFT JOIN chart_tags ct ON ct.chartId=charts.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"} - GROUP BY charts.id - ORDER BY charts.updatedAt DESC - `, - uncategorized ? [] : [tagId] - ) - tag.charts = charts - - await assignTagsForCharts(trx, charts) - - // Subcategories - const children = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql - SELECT t.id, t.name FROM tags t - WHERE t.parentId = ? - `, - [tag.id] - ) - tag.children = children - - // Possible parents to choose from - const possibleParents = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql - SELECT t.id, t.name FROM tags t - WHERE t.parentId IS NULL - ` - ) - tag.possibleParents = possibleParents - - return { - tag, - } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/tags/:tagId", - async (req: Request, res, trx) => { - const tagId = expectInt(req.params.tagId) - const tag = (req.body as { tag: any }).tag - await db.knexRaw( - trx, - `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, - [tag.name, new Date(), tag.slug, tagId] - ) - if (tag.slug) { - // See if there's a published gdoc with a matching slug. - // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, - // where the page for the topic is just an article. - const gdoc = await db.knexRaw>( - trx, - `-- sql - SELECT slug FROM posts_gdocs pg - WHERE EXISTS ( - SELECT 1 - FROM posts_gdocs_x_tags gt - WHERE pg.id = gt.gdocId AND gt.tagId = ? - ) AND pg.published = TRUE AND pg.slug = ?`, - [tagId, tag.slug] - ) - if (!gdoc.length) { - return { - success: true, - tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. - -Are you sure you haven't made a typo?`, - } - } - } - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/tags/new", - async (req: Request, res, trx) => { - const tag = req.body - function validateTag( - tag: unknown - ): tag is { name: string; slug: string | null } { - return ( - checkIsPlainObjectWithGuard(tag) && - typeof tag.name === "string" && - (tag.slug === null || - (typeof tag.slug === "string" && tag.slug !== "")) - ) - } - if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) - - const conflictingTag = await db.knexRawFirst<{ - name: string - slug: string | null - }>( - trx, - `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, - [tag.name, tag.slug] - ) - if (conflictingTag) - throw new JsonError( - conflictingTag.name === tag.name - ? `Tag with name ${tag.name} already exists` - : `Tag with slug ${tag.slug} already exists`, - 400 - ) - - const now = new Date() - const result = await db.knexRawInsert( - trx, - `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, - // parentId will be deprecated soon once we migrate fully to the tag graph - [tag.name, tag.slug, now, now] - ) - return { success: true, tagId: result.insertId } - } -) - -getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { - return { tags: await db.getMinimalTagsWithIsTopic(trx) } -}) - -deleteRouteWithRWTransaction( - apiRouter, - "/tags/:tagId/delete", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) - - await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/redirects/new", - async (req: Request, res, trx) => { - const chartId = expectInt(req.params.chartId) - const fields = req.body as { slug: string } - const result = await db.knexRawInsert( - trx, - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [chartId, fields.slug] - ) - const redirectId = result.insertId - const redirect = await db.knexRaw( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [redirectId] - ) - return { success: true, redirect: redirect } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - const redirect = await db.knexRawFirst( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [id] - ) - - if (!redirect) - throw new JsonError(`No redirect found for id ${id}`, 404) - - await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [ - id, - ]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect from ${redirect.slug}` - ) - - return { success: true } - } -) - -getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { - const raw_rows = await db.knexRaw( - trx, - `-- sql - WITH - posts_tags_aggregated AS ( - SELECT - post_id, - IF( - COUNT(tags.id) = 0, - JSON_ARRAY(), - JSON_ARRAYAGG(JSON_OBJECT("id", tags.id, "name", tags.name)) - ) AS tags - FROM - post_tags - LEFT JOIN tags ON tags.id = post_tags.tag_id - GROUP BY - post_id - ), - post_gdoc_slug_successors AS ( - SELECT - posts.id, - IF( - COUNT(gdocSlugSuccessor.id) = 0, - JSON_ARRAY(), - JSON_ARRAYAGG( - JSON_OBJECT("id", gdocSlugSuccessor.id, "published", gdocSlugSuccessor.published) - ) - ) AS gdocSlugSuccessors - FROM - posts - LEFT JOIN posts_gdocs gdocSlugSuccessor ON gdocSlugSuccessor.slug = posts.slug - GROUP BY - posts.id - ) - SELECT - posts.id AS id, - posts.title AS title, - posts.type AS TYPE, - posts.slug AS slug, - STATUS, - updated_at_in_wordpress, - posts.authors, - posts_tags_aggregated.tags AS tags, - gdocSuccessorId, - gdocSuccessor.published AS isGdocSuccessorPublished, - -- posts can either have explict successors via the gdocSuccessorId column - -- or implicit successors if a gdoc has been created that uses the same slug - -- as a Wp post (the gdoc one wins once it is published) - post_gdoc_slug_successors.gdocSlugSuccessors AS gdocSlugSuccessors - FROM - posts - LEFT JOIN post_gdoc_slug_successors ON post_gdoc_slug_successors.id = posts.id - LEFT JOIN posts_gdocs gdocSuccessor ON gdocSuccessor.id = posts.gdocSuccessorId - LEFT JOIN posts_tags_aggregated ON posts_tags_aggregated.post_id = posts.id - ORDER BY - updated_at_in_wordpress DESC`, - [] - ) - const rows = raw_rows.map((row: any) => ({ - ...row, - tags: JSON.parse(row.tags), - isGdocSuccessorPublished: !!row.isGdocSuccessorPublished, - gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors), - authors: JSON.parse(row.authors), - })) - - return { posts: rows } -}) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/setTags", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - - await setTagsForPost(trx, postId, req.body.tagIds) - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/posts/:postId.json", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx - .table(PostsTableName) - .where({ id: postId }) - .select("*") - .first()) as DbRawPost | undefined - return camelCaseProperties({ ...post }) - } -) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/createGdoc", - async (req: Request, res, trx) => { - const postId = expectInt(req.params.postId) - const allowRecreate = !!req.body.allowRecreate - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined - - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!allowRecreate && existingGdocId) - throw new JsonError("A gdoc already exists for this post", 400) - if (allowRecreate && existingGdocId && post.isGdocPublished) { - throw new JsonError( - "A gdoc already exists for this post and it is already published", - 400 - ) - } - if (post.archieml === null) - throw new JsonError( - `ArchieML was not present for post with id ${postId}`, - 500 - ) - const tagsByPostId = await getTagsByPostId(trx) - const tags = tagsByPostId.get(postId) || [] - const archieMl = JSON.parse( - // Google Docs interprets ®ion in grapher URLS as ®ion - // So we escape them here - post.archieml.replaceAll("&", "&") - ) as OwidGdocPostInterface - const gdocId = await createGdocAndInsertOwidGdocPostContent( - archieMl.content, - post.gdocSuccessorId - ) - // If we did not yet have a gdoc associated with this post, we need to register - // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise - // we don't need to make changes to the DB (only the gdoc regeneration was required) - if (!existingGdocId) { - post.gdocSuccessorId = gdocId - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", gdocId) - - const gdoc = new GdocPost(gdocId) - gdoc.slug = post.slug - gdoc.content.title = post.title - gdoc.content.type = archieMl.content.type || OwidGdocType.Article - gdoc.published = false - gdoc.createdAt = new Date() - gdoc.publishedAt = post.published_at - await upsertGdoc(trx, gdoc) - await setTagsForGdoc(trx, gdocId, tags) - } - return { googleDocsId: gdocId } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/unlinkGdoc", - async (req: Request, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined - - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!existingGdocId) - throw new JsonError("No gdoc exists for this post", 400) - if (existingGdocId && post.isGdocPublished) { - throw new JsonError( - "The GDoc is already published - you can't unlink it", - 400 - ) - } - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", null) - - await trx - .table(PostsGdocsTableName) - .where({ id: existingGdocId }) - .delete() - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/sources/:sourceId.json", - async (req: Request, res, trx) => { - const sourceId = expectInt(req.params.sourceId) - - const source = await db.knexRawFirst>( - trx, - ` - SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace - FROM sources AS s - JOIN active_datasets AS d ON d.id=s.datasetId - WHERE s.id=?`, - [sourceId] - ) - if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) - source.variables = await db.knexRaw( - trx, - `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, - [sourceId] - ) - - return { source: source } - } + getVariablesVariableIdChartsJson ) +// Deploy helpers apiRouter.get("/deploys.json", async () => ({ deploys: await new DeployQueueServer().getDeploys(), })) -apiRouter.put("/deploy", async (req, res) => { - return triggerStaticBuild(res.locals.user, "Manually triggered deploy") -}) - -getRouteWithROTransaction(apiRouter, "/gdocs", (req, res, trx) => { - return getAllGdocIndexItemsOrderedByUpdatedAt(trx) -}) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/gdocs/:id", - async (req, res, trx) => { - const id = req.params.id - const contentSource = req.query.contentSource as - | GdocsContentSource - | undefined - - try { - // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published - const gdoc = await getAndLoadGdocById(trx, id, contentSource) - - if (!gdoc.published) { - await updateGdocContentOnly(trx, id, gdoc) - } - - res.set("Cache-Control", "no-store") - res.send(gdoc) - } catch (error) { - console.error("Error fetching gdoc", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } - } -) - -/** - * Handles all four `GdocPublishingAction` cases - * - SavingDraft (no action) - * - Publishing (index and bake) - * - Updating (index and bake (potentially via lightning deploy)) - * - Unpublishing (remove from index and bake) - */ -async function indexAndBakeGdocIfNeccesary( - trx: db.KnexReadWriteTransaction, - user: Required, - prevGdoc: - | GdocPost - | GdocDataInsight - | GdocHomepage - | GdocAbout - | GdocAuthor, - nextGdoc: GdocPost | GdocDataInsight | GdocHomepage | GdocAbout | GdocAuthor -) { - const prevJson = prevGdoc.toJSON() - const nextJson = nextGdoc.toJSON() - const hasChanges = checkHasChanges(prevGdoc, nextGdoc) - const action = getPublishingAction(prevJson, nextJson) - const isGdocPost = checkIsGdocPostExcludingFragments(nextJson) - - await match(action) - .with(GdocPublishingAction.SavingDraft, lodash.noop) - .with(GdocPublishingAction.Publishing, async () => { - if (isGdocPost) { - await indexIndividualGdocPost( - nextJson, - trx, - // If the gdoc is being published for the first time, prevGdoc.slug will be undefined - // In that case, we pass nextJson.slug to see if it has any page views (i.e. from WP) - prevGdoc.slug || nextJson.slug - ) - } - await triggerStaticBuild(user, `${action} ${nextJson.slug}`) - }) - .with(GdocPublishingAction.Updating, async () => { - if (isGdocPost) { - await indexIndividualGdocPost(nextJson, trx, prevGdoc.slug) - } - if (checkIsLightningUpdate(prevJson, nextJson, hasChanges)) { - await enqueueLightningChange( - user, - `Lightning update ${nextJson.slug}`, - nextJson.slug - ) - } else { - await triggerStaticBuild(user, `${action} ${nextJson.slug}`) - } - }) - .with(GdocPublishingAction.Unpublishing, async () => { - if (isGdocPost) { - await removeIndividualGdocPostFromIndex(nextJson) - } - await triggerStaticBuild(user, `${action} ${nextJson.slug}`) - }) - .exhaustive() -} - -/** - * Only supports creating a new empty Gdoc or updating an existing one. Does not - * support creating a new Gdoc from an existing one. Relevant updates will - * trigger a deploy. - */ -putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { - const { id } = req.params - - if (isEmpty(req.body)) { - return createOrLoadGdocById(trx, id) - } - - const prevGdoc = await getAndLoadGdocById(trx, id) - if (!prevGdoc) throw new JsonError(`No Google Doc with id ${id} found`) - - const nextGdoc = gdocFromJSON(req.body) - await nextGdoc.loadState(trx) - - await addImagesToContentGraph(trx, nextGdoc) - - await setLinksForGdoc( - trx, - nextGdoc.id, - nextGdoc.links, - nextGdoc.published - ? GdocLinkUpdateMode.DeleteAndInsert - : GdocLinkUpdateMode.DeleteOnly - ) - - await upsertGdoc(trx, nextGdoc) - - await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc) - - return nextGdoc -}) - -async function validateTombstoneRelatedLinkUrl( - trx: db.KnexReadonlyTransaction, - relatedLink?: string -) { - if (!relatedLink || !relatedLink.startsWith(GDOCS_BASE_URL)) return - const id = relatedLink.match(gdocUrlRegex)?.[1] - if (!id) { - throw new JsonError(`Invalid related link: ${relatedLink}`) - } - const [gdoc] = await getMinimalGdocPostsByIds(trx, [id]) - if (!gdoc) { - throw new JsonError(`Google Doc with ID ${id} not found`) - } - if (!gdoc.published) { - throw new JsonError(`Google Doc with ID ${id} is not published`) - } -} - -deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { - const { id } = req.params - - const gdoc = await getGdocBaseObjectById(trx, id, false) - if (!gdoc) throw new JsonError(`No Google Doc with id ${id} found`) - - const gdocSlug = getCanonicalUrl("", gdoc) - const { tombstone } = req.body - - if (tombstone) { - await validateTombstoneRelatedLinkUrl(trx, tombstone.relatedLinkUrl) - const slug = gdocSlug.replace("/", "") - const { relatedLinkThumbnail } = tombstone - if (relatedLinkThumbnail) { - const thumbnailExists = await db.checkIsImageInDB( - trx, - relatedLinkThumbnail - ) - if (!thumbnailExists) { - throw new JsonError( - `Image with filename "${relatedLinkThumbnail}" not found` - ) - } - } - await trx - .table("posts_gdocs_tombstones") - .insert({ ...tombstone, gdocId: id, slug }) - await trx - .table("redirects") - .insert({ source: gdocSlug, target: `/deleted${gdocSlug}` }) - } - - await trx - .table("posts") - .where({ gdocSuccessorId: gdoc.id }) - .update({ gdocSuccessorId: null }) - - await trx.table(PostsGdocsLinksTableName).where({ sourceId: id }).delete() - await trx.table(PostsGdocsXImagesTableName).where({ gdocId: id }).delete() - await trx.table(PostsGdocsTableName).where({ id }).delete() - await trx - .table(PostsGdocsComponentsTableName) - .where({ gdocId: id }) - .delete() - if (gdoc.published && checkIsGdocPostExcludingFragments(gdoc)) { - await removeIndividualGdocPostFromIndex(gdoc) - } - if (gdoc.published) { - if (!tombstone && gdocSlug && gdocSlug !== "/") { - // Assets have TTL of one week in Cloudflare. Add a redirect to make sure - // the page is no longer accessible. - // https://developers.cloudflare.com/pages/configuration/serving-pages/#asset-retention - console.log(`Creating redirect for "${gdocSlug}" to "/"`) - await db.knexRawInsert( - trx, - `INSERT INTO redirects (source, target, ttl) - VALUES (?, ?, DATE_ADD(NOW(), INTERVAL 8 DAY))`, - [gdocSlug, "/"] - ) - } - await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`) - } - return {} -}) - -postRouteWithRWTransaction( - apiRouter, - "/gdocs/:gdocId/setTags", - async (req, res, trx) => { - const { gdocId } = req.params - const { tagIds } = req.body - const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ - id: id, - })) - - await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) - - return { success: true } - } -) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/images.json", - async (_, res, trx) => { - try { - const images = await db.getCloudflareImages(trx) - res.set("Cache-Control", "no-store") - res.send({ images }) - } catch (error) { - console.error("Error fetching images", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } - } -) - -postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { - const { filename, type, content } = validateImagePayload(req.body) - - const { asBlob, dimensions, hash } = await processImageContent( - content, - type - ) - - const collision = await trx("images") - .where({ - hash, - replacedBy: null, - }) - .first() - - if (collision) { - return { - success: false, - error: `An image with this content already exists (filename: ${collision.filename})`, - } - } - - const preexisting = await trx("images") - .where("filename", "=", filename) - .first() - - if (preexisting) { - return { - success: false, - error: "An image with this filename already exists", - } - } - - const cloudflareId = await uploadToCloudflare(filename, asBlob) - - if (!cloudflareId) { - return { - success: false, - error: "Failed to upload image", - } - } - - await trx("images").insert({ - filename, - originalWidth: dimensions.width, - originalHeight: dimensions.height, - cloudflareId, - updatedAt: new Date().getTime(), - userId: res.locals.user.id, - hash, - }) - - const image = await db.getCloudflareImage(trx, filename) - - return { - success: true, - image, - } -}) - -/** - * Similar to the POST route, but for updating an existing image. - * Creates a new image entry in the database and uploads the new image to Cloudflare. - * The old image is marked as replaced by the new image. - */ -putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { - const { type, content } = validateImagePayload(req.body) - const { asBlob, dimensions, hash } = await processImageContent( - content, - type - ) - const collision = await trx("images") - .where({ - hash, - replacedBy: null, - }) - .first() - - if (collision) { - return { - success: false, - error: `An exact copy of this image already exists (filename: ${collision.filename})`, - } - } - - const { id } = req.params - - const image = await trx("images") - .where("id", "=", id) - .first() - - if (!image) { - throw new JsonError(`No image found for id ${id}`, 404) - } - - const originalCloudflareId = image.cloudflareId - const originalFilename = image.filename - const originalAltText = image.defaultAlt - - if (!originalCloudflareId) { - throw new JsonError( - `Image with id ${id} has no associated Cloudflare image`, - 400 - ) - } - - const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob) - - if (!newCloudflareId) { - throw new JsonError("Failed to upload image", 500) - } - - const [newImageId] = await trx("images").insert({ - filename: originalFilename, - originalWidth: dimensions.width, - originalHeight: dimensions.height, - cloudflareId: newCloudflareId, - updatedAt: new Date().getTime(), - userId: res.locals.user.id, - defaultAlt: originalAltText, - hash, - version: image.version + 1, - }) - - await trx("images").where("id", "=", id).update({ - replacedBy: newImageId, - }) - - const updated = await db.getCloudflareImage(trx, originalFilename) - - await triggerStaticBuild( - res.locals.user, - `Updating image "${originalFilename}"` - ) - - return { - success: true, - image: updated, - } -}) - -// Update alt text via patch -patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { - const { id } = req.params - - const image = await trx("images") - .where("id", "=", id) - .first() - - if (!image) { - throw new JsonError(`No image found for id ${id}`, 404) - } - - const patchableImageProperties = ["defaultAlt"] as const - const patch = lodash.pick(req.body, patchableImageProperties) - - if (Object.keys(patch).length === 0) { - throw new JsonError("No patchable properties provided", 400) - } - - await trx("images").where({ id }).update(patch) - - const updated = await trx("images") - .where("id", "=", id) - .first() - - return { - success: true, - image: updated, - } -}) - -deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { - const { id } = req.params - - const image = await trx("images") - .where("id", "=", id) - .first() - - if (!image) { - throw new JsonError(`No image found for id ${id}`, 404) - } - if (!image.cloudflareId) { - throw new JsonError(`Image does not have a cloudflare ID`, 400) - } - - const replacementChain = await db.selectReplacementChainForImage(trx, id) - - await pMap( - replacementChain, - async (image) => { - if (image.cloudflareId) { - await deleteFromCloudflare(image.cloudflareId) - } - }, - { concurrency: 5 } - ) - - // There's an ON DELETE CASCADE which will delete the replacements - await trx("images").where({ id }).delete() - - return { - success: true, - } -}) - -getRouteWithROTransaction(apiRouter, "/images/usage", async (_, __, trx) => { - const usage = await db.getImageUsage(trx) - - return { - success: true, - usage, - } -}) - -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, - async ( - req: Request, - res, - trx - ): Promise> => { - const chartId = parseIntOrUndefined(req.params.chartId) - if (!chartId) throw new JsonError(`Invalid chart ID`, 400) - - const topics = await getGptTopicSuggestions(trx, chartId) - - if (!topics.length) - throw new JsonError( - `No GPT topic suggestions found for chart ${chartId}`, - 404 - ) - - return { - topics, - } - } -) - -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-alt-text/:imageId`, - async ( - req: Request, - res, - trx - ): Promise<{ - success: boolean - altText: string | null - }> => { - const imageId = parseIntOrUndefined(req.params.imageId) - if (!imageId) throw new JsonError(`Invalid image ID`, 400) - const image = await trx("images") - .where("id", imageId) - .first() - if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) - - const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` - let altText: string | null = "" - try { - altText = await fetchGptGeneratedAltText(src) - } catch (error) { - console.error( - `Error fetching GPT alt text for image ${imageId}`, - error - ) - throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) - } - - if (!altText) { - throw new JsonError(`Unable to generate alt text for image`, 404) - } - - return { success: true, altText } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - async (req, res, trx) => { - const { slug } = req.params - const { tagIds } = req.body - const explorer = await trx.table("explorers").where({ slug }).first() - if (!explorer) - throw new JsonError(`No explorer found for slug ${slug}`, 404) - - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - for (const tagId of tagIds) { - await trx - .table("explorer_tags") - .insert({ explorerSlug: slug, tagId }) - } - - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - async (req: Request, res, trx) => { - const { slug } = req.params - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - return { success: true } - } -) - -// Get an ArchieML output of all the work produced by an author. This includes -// gdoc articles, gdoc modular/linear topic pages and wordpress modular topic -// pages. Data insights are excluded. This is used to manually populate the -// [.secondary] section of the {.research-and-writing} block of author pages -// using the alternate template, which highlights topics rather than articles. -getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { - type WordpressPageRecord = { - isWordpressPage: number - } & Record< - "slug" | "title" | "subtitle" | "thumbnail" | "authors" | "publishedAt", - string - > - type GdocRecord = Pick - - const author = req.query.author || "Max Roser" - const gdocs = await db.knexRaw( - trx, - `-- sql - SELECT id, publishedAt - FROM posts_gdocs - WHERE JSON_CONTAINS(content->'$.authors', '"${author}"') - AND type NOT IN ("data-insight", "fragment") - AND published = 1 - ` - ) - - // type: page - const wpModularTopicPages = await db.knexRaw( - trx, - `-- sql - SELECT - wpApiSnapshot->>"$.slug" as slug, - wpApiSnapshot->>"$.title.rendered" as title, - wpApiSnapshot->>"$.excerpt.rendered" as subtitle, - TRUE as isWordpressPage, - wpApiSnapshot->>"$.authors_name" as authors, - wpApiSnapshot->>"$.featured_media_paths.medium_large" as thumbnail, - wpApiSnapshot->>"$.date" as publishedAt - FROM posts p - WHERE wpApiSnapshot->>"$.content" LIKE '%topic-page%' - AND JSON_CONTAINS(wpApiSnapshot->'$.authors_name', '"${author}"') - AND wpApiSnapshot->>"$.status" = 'publish' - AND NOT EXISTS ( - SELECT 1 FROM posts_gdocs pg - WHERE pg.slug = p.slug - AND pg.content->>'$.type' LIKE '%topic-page' - ) - ` - ) - - const isWordpressPage = ( - post: WordpressPageRecord | GdocRecord - ): post is WordpressPageRecord => - (post as WordpressPageRecord).isWordpressPage === 1 - - function* generateProperty(key: string, value: string) { - yield `${key}: ${value}\n` - } - - const sortByDateDesc = ( - a: GdocRecord | WordpressPageRecord, - b: GdocRecord | WordpressPageRecord - ): number => { - if (!a.publishedAt || !b.publishedAt) return 0 - return ( - new Date(b.publishedAt).getTime() - - new Date(a.publishedAt).getTime() - ) - } - - function* generateAllWorkArchieMl() { - for (const post of [...gdocs, ...wpModularTopicPages].sort( - sortByDateDesc - )) { - if (isWordpressPage(post)) { - yield* generateProperty( - "url", - `https://ourworldindata.org/${post.slug}` - ) - yield* generateProperty("title", post.title) - yield* generateProperty("subtitle", post.subtitle) - yield* generateProperty( - "authors", - JSON.parse(post.authors).join(", ") - ) - const parsedPath = path.parse(post.thumbnail) - yield* generateProperty( - "filename", - // /app/uploads/2021/09/reducing-fertilizer-768x301.png -> reducing-fertilizer.png - path.format({ - name: parsedPath.name.replace(/-\d+x\d+$/, ""), - ext: parsedPath.ext, - }) - ) - yield "\n" - } else { - // this is a gdoc - yield* generateProperty( - "url", - `https://docs.google.com/document/d/${post.id}/edit` - ) - yield "\n" - } - } - } - - res.type("text/plain") - return [...generateAllWorkArchieMl()].join("") -}) - -getRouteWithROTransaction( - apiRouter, - "/flatTagGraph.json", - async (req, res, trx) => { - const flatTagGraph = await db.getFlatTagGraph(trx) - return flatTagGraph - } -) - -postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { - const tagGraph = req.body?.tagGraph as unknown - if (!tagGraph) { - throw new JsonError("No tagGraph provided", 400) - } - - function validateFlatTagGraph( - tagGraph: Record - ): tagGraph is FlatTagGraph { - if (lodash.isObject(tagGraph)) { - for (const [key, value] of Object.entries(tagGraph)) { - if (!lodash.isString(key) && isNaN(Number(key))) { - return false - } - if (!lodash.isArray(value)) { - return false - } - for (const tag of value) { - if ( - !( - checkIsPlainObjectWithGuard(tag) && - lodash.isNumber(tag.weight) && - lodash.isNumber(tag.parentId) && - lodash.isNumber(tag.childId) - ) - ) { - return false - } - } - } - } - - return true - } - const isValid = validateFlatTagGraph(tagGraph) - if (!isValid) { - throw new JsonError("Invalid tag graph provided", 400) - } - await db.updateTagGraph(trx, tagGraph) - res.send({ success: true }) -}) - -const createPatchConfigAndQueryParamsForChartView = async ( - knex: db.KnexReadonlyTransaction, - parentChartId: number, - config: GrapherInterface -) => { - const parentChartConfig = await expectChartById(knex, parentChartId) - - config = omit(config, CHART_VIEW_PROPS_TO_OMIT) - - const patchToParentChart = diffGrapherConfigs(config, parentChartConfig) - - const fullConfigIncludingDefaults = mergeGrapherConfigs( - defaultGrapherConfig, - config - ) - const patchConfigToSave = { - ...patchToParentChart, - - // We want to make sure we're explicitly persisting some props like entity selection - // always, so they never change when the parent chart changes. - // For this, we need to ensure we include the default layer, so that we even - // persist these props when they are the same as the default. - ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST), - } - - const queryParams = grapherConfigToQueryParams(config) - - const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave) - return { patchConfig: patchConfigToSave, fullConfig, queryParams } -} - -getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { - type ChartViewRow = Pick & { - lastEditedByUser: string - chartConfigId: string - title: string - parentChartId: number - parentTitle: string - } - - const rows: ChartViewRow[] = await db.knexRaw( - trx, - `-- sql - SELECT - cv.id, - cv.name, - cv.updatedAt, - u.fullName as lastEditedByUser, - cv.chartConfigId, - cc.full ->> "$.title" as title, - cv.parentChartId, - pcc.full ->> "$.title" as parentTitle - FROM chart_views cv - JOIN chart_configs cc ON cv.chartConfigId = cc.id - JOIN charts pc ON cv.parentChartId = pc.id - JOIN chart_configs pcc ON pc.configId = pcc.id - JOIN users u ON cv.lastEditedByUserId = u.id - ORDER BY cv.updatedAt DESC - ` - ) - - const chartViews: ApiChartViewOverview[] = rows.map((row) => ({ - id: row.id, - name: row.name, - updatedAt: row.updatedAt?.toISOString() ?? null, - lastEditedByUser: row.lastEditedByUser, - chartConfigId: row.chartConfigId, - title: row.title, - parent: { - id: row.parentChartId, - title: row.parentTitle, - }, - })) - - return { chartViews } -}) - -getRouteWithROTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - type ChartViewRow = Pick< - DbPlainChartView, - "id" | "name" | "updatedAt" - > & { - lastEditedByUser: string - chartConfigId: string - configFull: JsonString - configPatch: JsonString - parentChartId: number - parentConfigFull: JsonString - queryParamsForParentChart: JsonString - } - - const row = await db.knexRawFirst( - trx, - `-- sql - SELECT - cv.id, - cv.name, - cv.updatedAt, - u.fullName as lastEditedByUser, - cv.chartConfigId, - cc.full as configFull, - cc.patch as configPatch, - cv.parentChartId, - pcc.full as parentConfigFull, - cv.queryParamsForParentChart - FROM chart_views cv - JOIN chart_configs cc ON cv.chartConfigId = cc.id - JOIN charts pc ON cv.parentChartId = pc.id - JOIN chart_configs pcc ON pc.configId = pcc.id - JOIN users u ON cv.lastEditedByUserId = u.id - WHERE cv.id = ? - `, - [id] - ) - - if (!row) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const chartView = { - ...row, - configFull: parseChartConfig(row.configFull), - configPatch: parseChartConfig(row.configPatch), - parentConfigFull: parseChartConfig(row.parentConfigFull), - queryParamsForParentChart: JSON.parse( - row.queryParamsForParentChart - ), - } - - return chartView - } -) - -postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { - const { name, parentChartId } = req.body as Pick< - DbPlainChartView, - "name" | "parentChartId" - > - const rawConfig = req.body.config as GrapherInterface - if (!name || !parentChartId || !rawConfig) { - throw new JsonError("Invalid request", 400) - } - - const { patchConfig, fullConfig, queryParams } = - await createPatchConfigAndQueryParamsForChartView( - trx, - parentChartId, - rawConfig - ) - - const { chartConfigId } = await saveNewChartConfigInDbAndR2( - trx, - undefined, - patchConfig, - fullConfig - ) - - // insert into chart_views - const insertRow: DbInsertChartView = { - name, - parentChartId, - lastEditedByUserId: res.locals.user.id, - chartConfigId: chartConfigId, - queryParamsForParentChart: JSON.stringify(queryParams), - } - const result = await trx.table(ChartViewsTableName).insert(insertRow) - const [resultId] = result - - return { chartViewId: resultId, success: true } -}) - -putRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const rawConfig = req.body.config as GrapherInterface - if (!rawConfig) { - throw new JsonError("Invalid request", 400) - } - - const existingRow: Pick< - DbPlainChartView, - "chartConfigId" | "parentChartId" - > = await trx(ChartViewsTableName) - .select("parentChartId", "chartConfigId") - .where({ id }) - .first() - - if (!existingRow) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const { patchConfig, fullConfig, queryParams } = - await createPatchConfigAndQueryParamsForChartView( - trx, - existingRow.parentChartId, - rawConfig - ) - - await updateChartConfigInDbAndR2( - trx, - existingRow.chartConfigId as Base64String, - patchConfig, - fullConfig - ) - - // update chart_views - await trx - .table(ChartViewsTableName) - .where({ id }) - .update({ - updatedAt: new Date(), - lastEditedByUserId: res.locals.user.id, - queryParamsForParentChart: JSON.stringify(queryParams), - }) - - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - const chartConfigId: string | undefined = await trx(ChartViewsTableName) - .select("chartConfigId") - .where({ id }) - .first() - .then((row) => row?.chartConfigId) - - if (!chartConfigId) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - await trx.table(ChartViewsTableName).where({ id }).delete() - - await deleteGrapherConfigFromR2ByUUID(chartConfigId) - - await trx - .table(ChartConfigsTableName) - .where({ id: chartConfigId }) - .delete() - - return { success: true } +apiRouter.put( + "/deploy", + async (_req: e.Request, res: e.Response>) => { + return triggerStaticBuild(res.locals.user, "Manually triggered deploy") } ) diff --git a/adminSiteServer/apiRoutes/bulkUpdates.ts b/adminSiteServer/apiRoutes/bulkUpdates.ts new file mode 100644 index 00000000000..eb97dba8aba --- /dev/null +++ b/adminSiteServer/apiRoutes/bulkUpdates.ts @@ -0,0 +1,241 @@ +import { + DbPlainChart, + DbRawChartConfig, + GrapherInterface, + DbRawVariable, +} from "@ourworldindata/types" +import { parseIntOrUndefined } from "@ourworldindata/utils" +import { + BulkGrapherConfigResponse, + BulkChartEditResponseRow, + chartBulkUpdateAllowedColumnNamesAndTypes, + GrapherConfigPatch, + VariableAnnotationsResponseRow, + variableAnnotationAllowedColumnNamesAndTypes, +} from "../../adminShared/AdminSessionTypes.js" +import { applyPatch } from "../../adminShared/patchHelper.js" +import { + OperationContext, + parseToOperation, +} from "../../adminShared/SqlFilterSExpression.js" +import { + getGrapherConfigsForVariable, + updateGrapherConfigAdminOfVariable, +} from "../../db/model/Variable.js" +import { saveGrapher } from "./charts.js" +import e, { Request } from "express" +import * as db from "../../db/db.js" +import * as lodash from "lodash" + +export async function getChartBulkUpdate( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise> { + const context: OperationContext = { + grapherConfigFieldName: "chart_configs.full", + whitelistedColumnNamesAndTypes: + chartBulkUpdateAllowedColumnNamesAndTypes, + } + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined + + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql + SELECT + charts.id as id, + chart_configs.full as config, + charts.createdAt as createdAt, + charts.updatedAt as updatedAt, + charts.lastEditedAt as lastEditedAt, + charts.publishedAt as publishedAt, + lastEditedByUser.fullName as lastEditedByUser, + publishedByUser.fullName as publishedByUser + FROM charts + LEFT JOIN chart_configs ON chart_configs.id = charts.configId + LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId + WHERE ${whereClause} + ORDER BY charts.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` + ) + + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql + SELECT count(*) as count + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + WHERE ${whereClause} + ` + ) + return { rows: results, numTotalRows: resultCount[0].count } +} + +export async function updateBulkChartConfigs( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const patchesList = req.body as GrapherConfigPatch[] + const chartIds = new Set(patchesList.map((patch) => patch.id)) + + const configsAndIds = await db.knexRaw< + Pick & { config: DbRawChartConfig["full"] } + >( + trx, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE c.id IN (?) + `, + [[...chartIds.values()]] + ) + const configMap = new Map( + configsAndIds.map((item: any) => [ + item.id, + // make sure that the id is set, otherwise the update behaviour is weird + // TODO: discuss if this has unintended side effects + item.config ? { ...JSON.parse(item.config), id: item.id } : {}, + ]) + ) + const oldValuesConfigMap = new Map(configMap) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) + } + + for (const [id, newConfig] of configMap.entries()) { + await saveGrapher(trx, { + user: res.locals.user, + newConfig, + existingConfig: oldValuesConfigMap.get(id), + referencedVariablesMightChange: false, + }) + } + + return { success: true } +} + +export async function getVariableAnnotations( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise> { + const context: OperationContext = { + grapherConfigFieldName: "grapherConfigAdmin", + whitelistedColumnNamesAndTypes: + variableAnnotationAllowedColumnNamesAndTypes, + } + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined + + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql + SELECT + variables.id as id, + variables.name as name, + chart_configs.patch as config, + d.name as datasetname, + namespaces.name as namespacename, + variables.createdAt as createdAt, + variables.updatedAt as updatedAt, + variables.description as description + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ORDER BY variables.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` + ) + + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql + SELECT count(*) as count + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ` + ) + return { rows: results, numTotalRows: resultCount[0].count } +} + +export async function updateVariableAnnotations( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const patchesList = req.body as GrapherConfigPatch[] + const variableIds = new Set(patchesList.map((patch) => patch.id)) + + const configsAndIds = await db.knexRaw< + Pick & { + grapherConfigAdmin: DbRawChartConfig["patch"] + } + >( + trx, + `-- sql + SELECT v.id, cc.patch AS grapherConfigAdmin + FROM variables v + LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id + WHERE v.id IN (?)`, + [[...variableIds.values()]] + ) + const configMap = new Map( + configsAndIds.map((item: any) => [ + item.id, + item.grapherConfigAdmin ? JSON.parse(item.grapherConfigAdmin) : {}, + ]) + ) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) + } + + for (const [variableId, newConfig] of configMap.entries()) { + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) continue + await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) + } + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/chartViews.ts b/adminSiteServer/apiRoutes/chartViews.ts new file mode 100644 index 00000000000..89ec2439869 --- /dev/null +++ b/adminSiteServer/apiRoutes/chartViews.ts @@ -0,0 +1,292 @@ +import { + defaultGrapherConfig, + grapherConfigToQueryParams, +} from "@ourworldindata/grapher" +import { + GrapherInterface, + CHART_VIEW_PROPS_TO_OMIT, + CHART_VIEW_PROPS_TO_PERSIST, + DbPlainChartView, + JsonString, + JsonError, + parseChartConfig, + DbInsertChartView, + ChartViewsTableName, + Base64String, + ChartConfigsTableName, +} from "@ourworldindata/types" +import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils" +import { omit, pick } from "lodash" +import { ApiChartViewOverview } from "../../adminShared/AdminTypes.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { + saveNewChartConfigInDbAndR2, + updateChartConfigInDbAndR2, +} from "../chartConfigHelpers.js" +import { deleteGrapherConfigFromR2ByUUID } from "../chartConfigR2Helpers.js" + +import * as db from "../../db/db.js" +import { expectChartById } from "./charts.js" +import e, { Request } from "express" +const createPatchConfigAndQueryParamsForChartView = async ( + knex: db.KnexReadonlyTransaction, + parentChartId: number, + config: GrapherInterface +) => { + const parentChartConfig = await expectChartById(knex, parentChartId) + + config = omit(config, CHART_VIEW_PROPS_TO_OMIT) + + const patchToParentChart = diffGrapherConfigs(config, parentChartConfig) + + const fullConfigIncludingDefaults = mergeGrapherConfigs( + defaultGrapherConfig, + config + ) + const patchConfigToSave = { + ...patchToParentChart, + + // We want to make sure we're explicitly persisting some props like entity selection + // always, so they never change when the parent chart changes. + // For this, we need to ensure we include the default layer, so that we even + // persist these props when they are the same as the default. + ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST), + } + + const queryParams = grapherConfigToQueryParams(patchConfigToSave) + + const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave) + return { patchConfig: patchConfigToSave, fullConfig, queryParams } +} + +export async function getChartViews( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + type ChartViewRow = Pick & { + lastEditedByUser: string + chartConfigId: string + title: string + parentChartId: number + parentTitle: string + } + + const rows: ChartViewRow[] = await db.knexRaw( + trx, + `-- sql + SELECT + cv.id, + cv.name, + cv.updatedAt, + u.fullName as lastEditedByUser, + cv.chartConfigId, + cc.full ->> "$.title" as title, + cv.parentChartId, + pcc.full ->> "$.title" as parentTitle + FROM chart_views cv + JOIN chart_configs cc ON cv.chartConfigId = cc.id + JOIN charts pc ON cv.parentChartId = pc.id + JOIN chart_configs pcc ON pc.configId = pcc.id + JOIN users u ON cv.lastEditedByUserId = u.id + ORDER BY cv.updatedAt DESC + ` + ) + + const chartViews: ApiChartViewOverview[] = rows.map((row) => ({ + id: row.id, + name: row.name, + updatedAt: row.updatedAt?.toISOString() ?? null, + lastEditedByUser: row.lastEditedByUser, + chartConfigId: row.chartConfigId, + title: row.title, + parent: { + id: row.parentChartId, + title: row.parentTitle, + }, + })) + + return { chartViews } +} + +export async function getChartViewById( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const id = expectInt(req.params.id) + + type ChartViewRow = Pick & { + lastEditedByUser: string + chartConfigId: string + configFull: JsonString + configPatch: JsonString + parentChartId: number + parentConfigFull: JsonString + queryParamsForParentChart: JsonString + } + + const row = await db.knexRawFirst( + trx, + `-- sql + SELECT + cv.id, + cv.name, + cv.updatedAt, + u.fullName as lastEditedByUser, + cv.chartConfigId, + cc.full as configFull, + cc.patch as configPatch, + cv.parentChartId, + pcc.full as parentConfigFull, + cv.queryParamsForParentChart + FROM chart_views cv + JOIN chart_configs cc ON cv.chartConfigId = cc.id + JOIN charts pc ON cv.parentChartId = pc.id + JOIN chart_configs pcc ON pc.configId = pcc.id + JOIN users u ON cv.lastEditedByUserId = u.id + WHERE cv.id = ? + `, + [id] + ) + + if (!row) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + const chartView = { + ...row, + configFull: parseChartConfig(row.configFull), + configPatch: parseChartConfig(row.configPatch), + parentConfigFull: parseChartConfig(row.parentConfigFull), + queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart), + } + + return chartView +} + +export async function createChartView( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { name, parentChartId } = req.body as Pick< + DbPlainChartView, + "name" | "parentChartId" + > + const rawConfig = req.body.config as GrapherInterface + if (!name || !parentChartId || !rawConfig) { + throw new JsonError("Invalid request", 400) + } + const chartViewWithName = await trx + .table(ChartViewsTableName) + .where({ name }) + .first() + if (chartViewWithName) { + return { + success: false, + errorMsg: `Narrative chart with name "${name}" already exists`, + } + } + + const { patchConfig, fullConfig, queryParams } = + await createPatchConfigAndQueryParamsForChartView( + trx, + parentChartId, + rawConfig + ) + + const { chartConfigId } = await saveNewChartConfigInDbAndR2( + trx, + undefined, + patchConfig, + fullConfig + ) + // insert into chart_views + const insertRow: DbInsertChartView = { + name, + parentChartId, + lastEditedByUserId: res.locals.user.id, + chartConfigId: chartConfigId, + queryParamsForParentChart: JSON.stringify(queryParams), + } + const result = await trx.table(ChartViewsTableName).insert(insertRow) + const [resultId] = result + + return { chartViewId: resultId, success: true } +} + +export async function updateChartView( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + const rawConfig = req.body.config as GrapherInterface + if (!rawConfig) { + throw new JsonError("Invalid request", 400) + } + + const existingRow: Pick< + DbPlainChartView, + "chartConfigId" | "parentChartId" + > = await trx(ChartViewsTableName) + .select("parentChartId", "chartConfigId") + .where({ id }) + .first() + + if (!existingRow) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + const { patchConfig, fullConfig, queryParams } = + await createPatchConfigAndQueryParamsForChartView( + trx, + existingRow.parentChartId, + rawConfig + ) + + await updateChartConfigInDbAndR2( + trx, + existingRow.chartConfigId as Base64String, + patchConfig, + fullConfig + ) + + await trx + .table(ChartViewsTableName) + .where({ id }) + .update({ + updatedAt: new Date(), + lastEditedByUserId: res.locals.user.id, + queryParamsForParentChart: JSON.stringify(queryParams), + }) + + return { success: true } +} + +export async function deleteChartView( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + + const chartConfigId: string | undefined = await trx(ChartViewsTableName) + .select("chartConfigId") + .where({ id }) + .first() + .then((row) => row?.chartConfigId) + + if (!chartConfigId) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + await trx.table(ChartViewsTableName).where({ id }).delete() + + await deleteGrapherConfigFromR2ByUUID(chartConfigId) + + await trx.table(ChartConfigsTableName).where({ id: chartConfigId }).delete() + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/charts.ts b/adminSiteServer/apiRoutes/charts.ts new file mode 100644 index 00000000000..366efba23c8 --- /dev/null +++ b/adminSiteServer/apiRoutes/charts.ts @@ -0,0 +1,808 @@ +import { migrateGrapherConfigToLatestVersion } from "@ourworldindata/grapher" +import { + GrapherInterface, + JsonError, + DbPlainUser, + Base64String, + serializeChartConfig, + DbPlainChart, + DbPlainChartSlugRedirect, + R2GrapherConfigDirectory, + DbInsertChartRevision, + DbRawChartConfig, + ChartConfigsTableName, +} from "@ourworldindata/types" +import { + diffGrapherConfigs, + mergeGrapherConfigs, + parseIntOrUndefined, + omitUndefinedValues, +} from "@ourworldindata/utils" +import Papa from "papaparse" +import { uuidv7 } from "uuidv7" +import { References } from "../../adminSiteClient/AbstractChartEditor.js" +import { ChartViewMinimalInformation } from "../../adminSiteClient/ChartEditor.js" +import { denormalizeLatestCountryData } from "../../baker/countryProfiles.js" +import { + getChartConfigById, + getPatchConfigByChartId, + getParentByChartConfig, + isInheritanceEnabledForChart, + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, + getParentByChartId, + getRedirectsByChartId, + getChartSlugById, + setChartTags, +} from "../../db/model/Chart.js" +import { + getWordpressPostReferencesByChartId, + getGdocsPostReferencesByChartId, +} from "../../db/model/Post.js" +import { expectInt, isValidSlug } from "../../serverUtils/serverUtil.js" +import { + BAKED_BASE_URL, + ADMIN_BASE_URL, +} from "../../settings/clientSettings.js" +import { + retrieveChartConfigFromDbAndSaveToR2, + updateChartConfigInDbAndR2, +} from "../chartConfigHelpers.js" +import { + deleteGrapherConfigFromR2, + deleteGrapherConfigFromR2ByUUID, + saveGrapherConfigToR2ByUUID, +} from "../chartConfigR2Helpers.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import { getLogsByChartId } from "../getLogsByChartId.js" +import { getPublishedLinksTo } from "../../db/model/Link.js" + +import e, { Request } from "express" +export const getReferencesByChartId = async ( + chartId: number, + knex: db.KnexReadonlyTransaction +): Promise => { + const postsWordpressPromise = getWordpressPostReferencesByChartId( + chartId, + knex + ) + const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex) + const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>( + knex, + `SELECT DISTINCT + explorerSlug + FROM + explorer_charts + WHERE + chartId = ?`, + [chartId] + ) + const chartViewsPromise = db.knexRaw( + knex, + `-- sql + SELECT cv.id, cv.name, cc.full ->> "$.title" AS title + FROM chart_views cv + JOIN chart_configs cc ON cc.id = cv.chartConfigId + WHERE cv.parentChartId = ?`, + [chartId] + ) + const [postsWordpress, postsGdocs, explorerSlugs, chartViews] = + await Promise.all([ + postsWordpressPromise, + postGdocsPromise, + explorerSlugsPromise, + chartViewsPromise, + ]) + + return { + postsGdocs, + postsWordpress, + explorers: explorerSlugs.map( + (row: { explorerSlug: string }) => row.explorerSlug + ), + chartViews, + } +} + +export const expectChartById = async ( + knex: db.KnexReadonlyTransaction, + chartId: any +): Promise => { + const chart = await getChartConfigById(knex, expectInt(chartId)) + if (chart) return chart.config + + throw new JsonError(`No chart found for id ${chartId}`, 404) +} + +const expectPatchConfigByChartId = async ( + knex: db.KnexReadonlyTransaction, + chartId: any +): Promise => { + const patchConfig = await getPatchConfigByChartId(knex, expectInt(chartId)) + if (!patchConfig) { + throw new JsonError(`No chart found for id ${chartId}`, 404) + } + return patchConfig +} + +const saveNewChart = async ( + knex: db.KnexReadWriteTransaction, + { + config, + user, + // new charts inherit by default + shouldInherit = true, + }: { config: GrapherInterface; user: DbPlainUser; shouldInherit?: boolean } +): Promise<{ + chartConfigId: Base64String + patchConfig: GrapherInterface + fullConfig: GrapherInterface +}> => { + // grab the parent of the chart if inheritance should be enabled + const parent = shouldInherit + ? await getParentByChartConfig(knex, config) + : undefined + + // compute patch and full configs + const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) + const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) + + // insert patch & full configs into the chart_configs table + // We can't quite use `saveNewChartConfigInDbAndR2` here, because + // we need to update the chart id in the config after inserting it. + const chartConfigId = uuidv7() as Base64String + await db.knexRaw( + knex, + `-- sql + INSERT INTO chart_configs (id, patch, full) + VALUES (?, ?, ?) + `, + [ + chartConfigId, + serializeChartConfig(patchConfig), + serializeChartConfig(fullConfig), + ] + ) + + // add a new chart to the charts table + const result = await db.knexRawInsert( + knex, + `-- sql + INSERT INTO charts (configId, isInheritanceEnabled, lastEditedAt, lastEditedByUserId) + VALUES (?, ?, ?, ?) + `, + [chartConfigId, shouldInherit, new Date(), user.id] + ) + + // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true + const chartId = result.insertId + patchConfig.id = chartId + fullConfig.id = chartId + await db.knexRaw( + knex, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch=JSON_SET(cc.patch, '$.id', ?), + cc.full=JSON_SET(cc.full, '$.id', ?) + WHERE c.id = ? + `, + [chartId, chartId, chartId] + ) + + await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId) + + return { chartConfigId, patchConfig, fullConfig } +} + +const updateExistingChart = async ( + knex: db.KnexReadWriteTransaction, + params: { + config: GrapherInterface + user: DbPlainUser + chartId: number + // if undefined, keep inheritance as is. + // if true or false, enable or disable inheritance + shouldInherit?: boolean + } +): Promise<{ + chartConfigId: Base64String + patchConfig: GrapherInterface + fullConfig: GrapherInterface +}> => { + const { config, user, chartId } = params + + // make sure that the id of the incoming config matches the chart id + config.id = chartId + + // if inheritance is enabled, grab the parent from its config + const shouldInherit = + params.shouldInherit ?? + (await isInheritanceEnabledForChart(knex, chartId)) + const parent = shouldInherit + ? await getParentByChartConfig(knex, config) + : undefined + + // compute patch and full configs + const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) + const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) + + const chartConfigIdRow = await db.knexRawFirst< + Pick + >(knex, `SELECT configId FROM charts WHERE id = ?`, [chartId]) + + if (!chartConfigIdRow) + throw new JsonError(`No chart config found for id ${chartId}`, 404) + + const now = new Date() + + const { chartConfigId } = await updateChartConfigInDbAndR2( + knex, + chartConfigIdRow.configId as Base64String, + patchConfig, + fullConfig + ) + + // update charts row + await db.knexRaw( + knex, + `-- sql + UPDATE charts + SET isInheritanceEnabled=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=? + WHERE id = ? + `, + [shouldInherit, now, now, user.id, chartId] + ) + + return { chartConfigId, patchConfig, fullConfig } +} + +export const saveGrapher = async ( + knex: db.KnexReadWriteTransaction, + { + user, + newConfig, + existingConfig, + shouldInherit, + referencedVariablesMightChange = true, + }: { + user: DbPlainUser + newConfig: GrapherInterface + existingConfig?: GrapherInterface + // if undefined, keep inheritance as is. + // if true or false, enable or disable inheritance + shouldInherit?: boolean + // if the variables a chart uses can change then we need + // to update the latest country data which takes quite a long time (hundreds of ms) + referencedVariablesMightChange?: boolean + } +) => { + // Try to migrate the new config to the latest version + newConfig = migrateGrapherConfigToLatestVersion(newConfig) + + // Slugs need some special logic to ensure public urls remain consistent whenever possible + async function isSlugUsedInRedirect() { + const rows = await db.knexRaw( + knex, + `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`, + // -1 is a placeholder ID that will never exist; but we cannot use NULL because + // in that case we would always get back an empty resultset + [existingConfig ? existingConfig.id : -1, newConfig.slug] + ) + return rows.length > 0 + } + + async function isSlugUsedInOtherGrapher() { + const rows = await db.knexRaw>( + knex, + `-- sql + SELECT c.id + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE + c.id != ? + AND cc.full ->> "$.isPublished" = "true" + AND cc.slug = ? + `, + // -1 is a placeholder ID that will never exist; but we cannot use NULL because + // in that case we would always get back an empty resultset + [existingConfig ? existingConfig.id : -1, newConfig.slug] + ) + return rows.length > 0 + } + + // When a chart is published, check for conflicts + if (newConfig.isPublished) { + if (!isValidSlug(newConfig.slug)) + throw new JsonError(`Invalid chart slug ${newConfig.slug}`) + else if (await isSlugUsedInRedirect()) + throw new JsonError( + `This chart slug was previously used by another chart: ${newConfig.slug}` + ) + else if (await isSlugUsedInOtherGrapher()) + throw new JsonError( + `This chart slug is in use by another published chart: ${newConfig.slug}` + ) + else if ( + existingConfig && + existingConfig.isPublished && + existingConfig.slug !== newConfig.slug + ) { + // Changing slug of an existing chart, delete any old redirect and create new one + await db.knexRaw( + knex, + `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`, + [existingConfig.id, existingConfig.slug] + ) + await db.knexRaw( + knex, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [existingConfig.id, existingConfig.slug] + ) + // When we rename grapher configs, make sure to delete the old one (the new one will be saved below) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${existingConfig.slug}.json` + ) + } + } + + if (existingConfig) + // Bump chart version, very important for cachebusting + newConfig.version = existingConfig.version! + 1 + else if (newConfig.version) + // If a chart is republished, we want to keep incrementing the old version number, + // otherwise it can lead to clients receiving cached versions of the old data. + newConfig.version += 1 + else newConfig.version = 1 + + // add the isPublished field if is missing + if (newConfig.isPublished === undefined) { + newConfig.isPublished = false + } + + // Execute the actual database update or creation + let chartId: number + let chartConfigId: Base64String + let patchConfig: GrapherInterface + let fullConfig: GrapherInterface + if (existingConfig) { + chartId = existingConfig.id! + const configs = await updateExistingChart(knex, { + config: newConfig, + user, + chartId, + shouldInherit, + }) + chartConfigId = configs.chartConfigId + patchConfig = configs.patchConfig + fullConfig = configs.fullConfig + } else { + const configs = await saveNewChart(knex, { + config: newConfig, + user, + shouldInherit, + }) + chartConfigId = configs.chartConfigId + patchConfig = configs.patchConfig + fullConfig = configs.fullConfig + chartId = fullConfig.id! + } + + // Record this change in version history + const chartRevisionLog = { + chartId: chartId as number, + userId: user.id, + config: serializeChartConfig(patchConfig), + createdAt: new Date(), + updatedAt: new Date(), + } satisfies DbInsertChartRevision + await db.knexRaw( + knex, + `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, + [ + chartRevisionLog.chartId, + chartRevisionLog.userId, + chartRevisionLog.config, + chartRevisionLog.createdAt, + chartRevisionLog.updatedAt, + ] + ) + + // Remove any old dimensions and store the new ones + // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json + await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chartId, + ]) + + const newDimensions = fullConfig.dimensions ?? [] + for (const [i, dim] of newDimensions.entries()) { + await db.knexRaw( + knex, + `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?, ?, ?, ?)`, + [chartId, dim.variableId, dim.property, i] + ) + } + + // So we can generate country profiles including this chart data + if (fullConfig.isPublished && referencedVariablesMightChange) + // TODO: remove this ad hoc knex transaction context when we switch the function to knex + await denormalizeLatestCountryData( + knex, + newDimensions.map((d) => d.variableId) + ) + + if (fullConfig.isPublished) { + await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId, { + directory: R2GrapherConfigDirectory.publishedGrapherBySlug, + filename: `${fullConfig.slug}.json`, + }) + } + + if ( + fullConfig.isPublished && + (!existingConfig || !existingConfig.isPublished) + ) { + // Newly published, set publication info + await db.knexRaw( + knex, + `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `, + [new Date(), user.id, chartId] + ) + await triggerStaticBuild(user, `Publishing chart ${fullConfig.slug}`) + } else if ( + !fullConfig.isPublished && + existingConfig && + existingConfig.isPublished + ) { + // Unpublishing chart, delete any existing redirects to it + await db.knexRaw( + knex, + `DELETE FROM chart_slug_redirects WHERE chart_id = ?`, + [existingConfig.id] + ) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${existingConfig.slug}.json` + ) + await triggerStaticBuild(user, `Unpublishing chart ${fullConfig.slug}`) + } else if (fullConfig.isPublished) + await triggerStaticBuild(user, `Updating chart ${fullConfig.slug}`) + + return { + chartId, + savedPatch: patchConfig, + } +} + +export async function updateGrapherConfigsInR2( + knex: db.KnexReadonlyTransaction, + updatedCharts: { chartConfigId: string; isPublished: boolean }[], + updatedMultiDimViews: { chartConfigId: string; isPublished: boolean }[] +) { + const idsToUpdate = [ + ...updatedCharts.filter(({ isPublished }) => isPublished), + ...updatedMultiDimViews, + ].map(({ chartConfigId }) => chartConfigId) + const builder = knex(ChartConfigsTableName) + .select("id", "full", "fullMd5") + .whereIn("id", idsToUpdate) + for await (const { id, full, fullMd5 } of builder.stream()) { + await saveGrapherConfigToR2ByUUID(id, full, fullMd5) + } +} + +export async function getChartsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 + const charts = await db.knexRaw( + trx, + `-- sql + SELECT ${oldChartFieldList}, + round(views_365d / 365, 1) as pageviewsPerDay + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN analytics_pageviews on (analytics_pageviews.url = CONCAT("https://ourworldindata.org/grapher/", chart_configs.slug) AND chart_configs.full ->> '$.isPublished' = "true" ) + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + ORDER BY charts.lastEditedAt DESC LIMIT ? + `, + [limit] + ) + + await assignTagsForCharts(trx, charts) + + return { charts } +} + +export async function getChartsCsv( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 + + // note: this query is extended from OldChart.listFields. + const charts = await db.knexRaw( + trx, + `-- sql + SELECT + charts.id, + chart_configs.full->>"$.version" AS version, + CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url, + CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl, + chart_configs.full->>"$.slug" AS slug, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.subtitle" AS subtitle, + chart_configs.full->>"$.sourceDesc" AS sourceDesc, + chart_configs.full->>"$.note" AS note, + chart_configs.chartType AS type, + chart_configs.full->>"$.internalNotes" AS internalNotes, + chart_configs.full->>"$.variantName" AS variantName, + chart_configs.full->>"$.isPublished" AS isPublished, + chart_configs.full->>"$.tab" AS tab, + chart_configs.chartType IS NOT NULL AS hasChartTab, + JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab, + chart_configs.full->>"$.originUrl" AS originUrl, + charts.lastEditedAt, + charts.lastEditedByUserId, + lastEditedByUser.fullName AS lastEditedBy, + charts.publishedAt, + charts.publishedByUserId, + publishedByUser.fullName AS publishedBy + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + ORDER BY charts.lastEditedAt DESC + LIMIT ? + `, + [limit] + ) + // note: retrieving references is VERY slow. + // await Promise.all( + // charts.map(async (chart: any) => { + // const references = await getReferencesByChartId(chart.id) + // chart.references = references.length + // ? references.map((ref) => ref.url) + // : "" + // }) + // ) + // await Chart.assignTagsForCharts(charts) + res.setHeader("Content-disposition", "attachment; filename=charts.csv") + res.setHeader("content-type", "text/csv") + const csv = Papa.unparse(charts) + return csv +} + +export async function getChartConfigJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return expectChartById(trx, req.params.chartId) +} + +export async function getChartParentJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const chartId = expectInt(req.params.chartId) + const parent = await getParentByChartId(trx, chartId) + const isInheritanceEnabled = await isInheritanceEnabledForChart( + trx, + chartId + ) + return omitUndefinedValues({ + variableId: parent?.variableId, + config: parent?.config, + isActive: isInheritanceEnabled, + }) +} + +export async function getChartPatchConfigJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const chartId = expectInt(req.params.chartId) + const config = await expectPatchConfigByChartId(trx, chartId) + return config +} + +export async function getChartLogsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { + logs: await getLogsByChartId( + trx, + parseInt(req.params.chartId as string) + ), + } +} + +export async function getChartReferencesJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const references = { + references: await getReferencesByChartId( + parseInt(req.params.chartId as string), + trx + ), + } + return references +} + +export async function getChartRedirectsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { + redirects: await getRedirectsByChartId( + trx, + parseInt(req.params.chartId as string) + ), + } +} + +export async function getChartPageviewsJson( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const slug = await getChartSlugById( + trx, + parseInt(req.params.chartId as string) + ) + if (!slug) return {} + + const pageviewsByUrl = await db.knexRawFirst( + trx, + `-- sql + SELECT * + FROM + analytics_pageviews + WHERE + url = ?`, + [`https://ourworldindata.org/grapher/${slug}`] + ) + + return { + pageviews: pageviewsByUrl ?? undefined, + } +} + +export async function createChart( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + let shouldInherit: boolean | undefined + if (req.query.inheritance) { + shouldInherit = req.query.inheritance === "enable" + } + + try { + const { chartId } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + shouldInherit, + }) + + return { success: true, chartId: chartId } + } catch (err) { + return { success: false, error: String(err) } + } +} + +export async function setChartTagsHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const chartId = expectInt(req.params.chartId) + + await setChartTags(trx, chartId, req.body.tags) + + return { success: true } +} + +export async function updateChart( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + let shouldInherit: boolean | undefined + if (req.query.inheritance) { + shouldInherit = req.query.inheritance === "enable" + } + + const existingConfig = await expectChartById(trx, req.params.chartId) + + try { + const { chartId, savedPatch } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + existingConfig, + shouldInherit, + }) + + const logs = await getLogsByChartId(trx, existingConfig.id as number) + return { + success: true, + chartId, + savedPatch, + newLog: logs[0], + } + } catch (err) { + return { + success: false, + error: String(err), + } + } +} + +export async function deleteChart( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const chart = await expectChartById(trx, req.params.chartId) + if (chart.slug) { + const links = await getPublishedLinksTo(trx, [chart.slug]) + if (links.length) { + const sources = links.map((link) => link.sourceSlug).join(", ") + throw new Error( + `Cannot delete chart in-use in the following published documents: ${sources}` + ) + } + } + + await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chart.id, + ]) + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE chart_id=?`, [ + chart.id, + ]) + + const row = await db.knexRawFirst>( + trx, + `SELECT configId FROM charts WHERE id = ?`, + [chart.id] + ) + if (!row || !row.configId) + throw new JsonError(`No chart config found for id ${chart.id}`, 404) + if (row) { + await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) + await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ + row.configId, + ]) + } + + if (chart.isPublished) + await triggerStaticBuild( + res.locals.user, + `Deleting chart ${chart.slug}` + ) + + await deleteGrapherConfigFromR2ByUUID(row.configId) + if (chart.isPublished) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${chart.slug}.json` + ) + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/datasets.ts b/adminSiteServer/apiRoutes/datasets.ts new file mode 100644 index 00000000000..f14d721683d --- /dev/null +++ b/adminSiteServer/apiRoutes/datasets.ts @@ -0,0 +1,407 @@ +import { + DbPlainTag, + DbPlainDatasetTag, + JsonError, + DbRawVariable, + DbRawOrigin, + parseOriginsRow, +} from "@ourworldindata/types" +import { + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, +} from "../../db/model/Chart.js" +import { getDatasetById, setTagsForDataset } from "../../db/model/Dataset.js" +import { logErrorAndMaybeSendToBugsnag } from "../../serverUtils/errorLog.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { + syncDatasetToGitRepo, + removeDatasetFromGitRepo, +} from "../gitDataExport.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import e, { Request } from "express" + +export async function getDatasets( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const datasets = await db.knexRaw>( + trx, + `-- sql + WITH variable_counts AS ( + SELECT + v.datasetId, + COUNT(DISTINCT cd.chartId) as numCharts + FROM chart_dimensions cd + JOIN variables v ON cd.variableId = v.id + GROUP BY v.datasetId + ) + SELECT + ad.id, + ad.namespace, + ad.name, + d.shortName, + ad.description, + ad.dataEditedAt, + du.fullName AS dataEditedByUserName, + ad.metadataEditedAt, + mu.fullName AS metadataEditedByUserName, + ad.isPrivate, + ad.nonRedistributable, + d.version, + vc.numCharts + FROM active_datasets ad + LEFT JOIN variable_counts vc ON ad.id = vc.datasetId + JOIN users du ON du.id=ad.dataEditedByUserId + JOIN users mu ON mu.id=ad.metadataEditedByUserId + JOIN datasets d ON d.id=ad.id + ORDER BY ad.dataEditedAt DESC + ` + ) + + const tags = await db.knexRaw< + Pick & Pick + >( + trx, + `-- sql + SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt + JOIN tags t ON dt.tagId = t.id + ` + ) + const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) + for (const dataset of datasets) { + dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => + lodash.omit(t, "datasetId") + ) + } + /*LEFT JOIN variables AS v ON v.datasetId=d.id + GROUP BY d.id*/ + + return { datasets: datasets } +} + +export async function getDataset( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await db.knexRawFirst>( + trx, + `-- sql + SELECT d.id, + d.namespace, + d.name, + d.shortName, + d.version, + d.description, + d.updatedAt, + d.dataEditedAt, + d.dataEditedByUserId, + du.fullName AS dataEditedByUserName, + d.metadataEditedAt, + d.metadataEditedByUserId, + mu.fullName AS metadataEditedByUserName, + d.isPrivate, + d.isArchived, + d.nonRedistributable, + d.updatePeriodDays + FROM datasets AS d + JOIN users du ON du.id=d.dataEditedByUserId + JOIN users mu ON mu.id=d.metadataEditedByUserId + WHERE d.id = ? + `, + [datasetId] + ) + + if (!dataset) throw new JsonError(`No dataset by id '${datasetId}'`, 404) + + const zipFile = await db.knexRawFirst<{ filename: string }>( + trx, + `SELECT filename FROM dataset_files WHERE datasetId=?`, + [datasetId] + ) + if (zipFile) dataset.zipFile = zipFile + + const variables = await db.knexRaw< + Pick< + DbRawVariable, + "id" | "name" | "description" | "display" | "catalogPath" + > + >( + trx, + `-- sql + SELECT + v.id, + v.name, + v.description, + v.display, + v.catalogPath + FROM + variables AS v + WHERE + v.datasetId = ? + `, + [datasetId] + ) + + for (const v of variables) { + v.display = JSON.parse(v.display) + } + + dataset.variables = variables + + // add all origins + const origins: DbRawOrigin[] = await db.knexRaw( + trx, + `-- sql + SELECT DISTINCT + o.* + FROM + origins_variables AS ov + JOIN origins AS o ON ov.originId = o.id + JOIN variables AS v ON ov.variableId = v.id + WHERE + v.datasetId = ? + `, + [datasetId] + ) + + const parsedOrigins = origins.map(parseOriginsRow) + + dataset.origins = parsedOrigins + + const sources = await db.knexRaw<{ + id: number + name: string + description: string + }>( + trx, + ` + SELECT s.id, s.name, s.description + FROM sources AS s + WHERE s.datasetId = ? + ORDER BY s.id ASC + `, + [datasetId] + ) + + // expand description of sources and add to dataset as variableSources + dataset.variableSources = sources.map((s: any) => { + return { + id: s.id, + name: s.name, + ...JSON.parse(s.description), + } + }) + + const charts = await db.knexRaw( + trx, + `-- sql + SELECT ${oldChartFieldList} + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN chart_dimensions AS cd ON cd.chartId = charts.id + JOIN variables AS v ON cd.variableId = v.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE v.datasetId = ? + GROUP BY charts.id + `, + [datasetId] + ) + + dataset.charts = charts + + await assignTagsForCharts(trx, charts) + + const tags = await db.knexRaw<{ id: number; name: string }>( + trx, + ` + SELECT t.id, t.name + FROM tags t + JOIN dataset_tags dt ON dt.tagId = t.id + WHERE dt.datasetId = ? + `, + [datasetId] + ) + dataset.tags = tags + + const availableTags = await db.knexRaw<{ + id: number + name: string + parentName: string + }>( + trx, + ` + SELECT t.id, t.name, p.name AS parentName + FROM tags AS t + JOIN tags AS p ON t.parentId=p.id + ` + ) + dataset.availableTags = availableTags + + return { dataset: dataset } +} + +export async function updateDataset( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + // Only updates `nonRedistributable` and `tags`, other fields come from ETL + // and are not editable + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + const newDataset = (req.body as { dataset: any }).dataset + await db.knexRaw( + trx, + ` + UPDATE datasets + SET + nonRedistributable=?, + metadataEditedAt=?, + metadataEditedByUserId=? + WHERE id=? + `, + [ + newDataset.nonRedistributable, + new Date(), + _res.locals.user.id, + datasetId, + ] + ) + + const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) + await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ + datasetId, + ]) + if (tagRows.length) + for (const tagRow of tagRows) { + await db.knexRaw( + trx, + `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, + tagRow + ) + } + + try { + await syncDatasetToGitRepo(trx, datasetId, { + oldDatasetName: dataset.name, + commitName: _res.locals.user.fullName, + commitEmail: _res.locals.user.email, + }) + } catch (err) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue + } + + return { success: true } +} + +export async function setArchived( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ + datasetId, + ]) + return { success: true } +} + +export async function setTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + await setTagsForDataset(trx, datasetId, req.body.tagIds) + + return { success: true } +} + +export async function deleteDataset( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw( + trx, + `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, + [datasetId] + ) + await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [datasetId]) + await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) + + try { + await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { + commitName: _res.locals.user.fullName, + commitEmail: _res.locals.user.email, + }) + } catch (err: any) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue + } + + return { success: true } +} + +export async function republishCharts( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + if (req.body.republish) { + await db.knexRaw( + trx, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), + cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) + WHERE c.id IN ( + SELECT DISTINCT chart_dimensions.chartId + FROM chart_dimensions + JOIN variables ON variables.id = chart_dimensions.variableId + WHERE variables.datasetId = ? + )`, + [datasetId] + ) + } + + await triggerStaticBuild( + _res.locals.user, + `Republishing all charts in dataset ${dataset.name} (${dataset.id})` + ) + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/explorer.ts b/adminSiteServer/apiRoutes/explorer.ts new file mode 100644 index 00000000000..ecdb46f54c5 --- /dev/null +++ b/adminSiteServer/apiRoutes/explorer.ts @@ -0,0 +1,32 @@ +import { JsonError } from "@ourworldindata/types" +import e, { Request } from "express" + +import * as db from "../../db/db.js" +export async function addExplorerTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { slug } = req.params + const { tagIds } = req.body + const explorer = await trx.table("explorers").where({ slug }).first() + if (!explorer) + throw new JsonError(`No explorer found for slug ${slug}`, 404) + + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + for (const tagId of tagIds) { + await trx.table("explorer_tags").insert({ explorerSlug: slug, tagId }) + } + + return { success: true } +} + +export async function deleteExplorerTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { slug } = req.params + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/gdocs.ts b/adminSiteServer/apiRoutes/gdocs.ts new file mode 100644 index 00000000000..3d5537b6154 --- /dev/null +++ b/adminSiteServer/apiRoutes/gdocs.ts @@ -0,0 +1,288 @@ +import { getCanonicalUrl } from "@ourworldindata/components" +import { + GdocsContentSource, + DbInsertUser, + JsonError, + GDOCS_BASE_URL, + gdocUrlRegex, + PostsGdocsLinksTableName, + PostsGdocsXImagesTableName, + PostsGdocsTableName, + PostsGdocsComponentsTableName, +} from "@ourworldindata/types" +import { checkIsGdocPostExcludingFragments } from "@ourworldindata/utils" +import { isEmpty } from "lodash" +import { match } from "ts-pattern" +import { + checkHasChanges, + getPublishingAction, + GdocPublishingAction, + checkIsLightningUpdate, +} from "../../adminSiteClient/gdocsDeploy.js" +import { + indexIndividualGdocPost, + removeIndividualGdocPostFromIndex, +} from "../../baker/algolia/utils/pages.js" +import { GdocAbout } from "../../db/model/Gdoc/GdocAbout.js" +import { GdocAuthor } from "../../db/model/Gdoc/GdocAuthor.js" +import { getMinimalGdocPostsByIds } from "../../db/model/Gdoc/GdocBase.js" +import { GdocDataInsight } from "../../db/model/Gdoc/GdocDataInsight.js" +import { + getAllGdocIndexItemsOrderedByUpdatedAt, + getAndLoadGdocById, + updateGdocContentOnly, + createOrLoadGdocById, + gdocFromJSON, + addImagesToContentGraph, + setLinksForGdoc, + GdocLinkUpdateMode, + upsertGdoc, + getGdocBaseObjectById, + setTagsForGdoc, +} from "../../db/model/Gdoc/GdocFactory.js" +import { GdocHomepage } from "../../db/model/Gdoc/GdocHomepage.js" +import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" +import { triggerStaticBuild, enqueueLightningChange } from "./routeUtils.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import e, { Request } from "express" + +export async function getAllGdocIndexItems( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return getAllGdocIndexItemsOrderedByUpdatedAt(trx) +} + +export async function getIndividualGdoc( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = req.params.id + const contentSource = req.query.contentSource as + | GdocsContentSource + | undefined + + try { + // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published + const gdoc = await getAndLoadGdocById(trx, id, contentSource) + + if (!gdoc.published) { + await updateGdocContentOnly(trx, id, gdoc) + } + + res.set("Cache-Control", "no-store") + res.send(gdoc) + } catch (error) { + console.error("Error fetching gdoc", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) + } +} + +/** + * Handles all four `GdocPublishingAction` cases + * - SavingDraft (no action) + * - Publishing (index and bake) + * - Updating (index and bake (potentially via lightning deploy)) + * - Unpublishing (remove from index and bake) + */ +async function indexAndBakeGdocIfNeccesary( + trx: db.KnexReadWriteTransaction, + user: Required, + prevGdoc: + | GdocPost + | GdocDataInsight + | GdocHomepage + | GdocAbout + | GdocAuthor, + nextGdoc: GdocPost | GdocDataInsight | GdocHomepage | GdocAbout | GdocAuthor +) { + const prevJson = prevGdoc.toJSON() + const nextJson = nextGdoc.toJSON() + const hasChanges = checkHasChanges(prevGdoc, nextGdoc) + const action = getPublishingAction(prevJson, nextJson) + const isGdocPost = checkIsGdocPostExcludingFragments(nextJson) + + await match(action) + .with(GdocPublishingAction.SavingDraft, lodash.noop) + .with(GdocPublishingAction.Publishing, async () => { + if (isGdocPost) { + await indexIndividualGdocPost( + nextJson, + trx, + // If the gdoc is being published for the first time, prevGdoc.slug will be undefined + // In that case, we pass nextJson.slug to see if it has any page views (i.e. from WP) + prevGdoc.slug || nextJson.slug + ) + } + await triggerStaticBuild(user, `${action} ${nextJson.slug}`) + }) + .with(GdocPublishingAction.Updating, async () => { + if (isGdocPost) { + await indexIndividualGdocPost(nextJson, trx, prevGdoc.slug) + } + if (checkIsLightningUpdate(prevJson, nextJson, hasChanges)) { + await enqueueLightningChange( + user, + `Lightning update ${nextJson.slug}`, + nextJson.slug + ) + } else { + await triggerStaticBuild(user, `${action} ${nextJson.slug}`) + } + }) + .with(GdocPublishingAction.Unpublishing, async () => { + if (isGdocPost) { + await removeIndividualGdocPostFromIndex(nextJson) + } + await triggerStaticBuild(user, `${action} ${nextJson.slug}`) + }) + .exhaustive() +} + +/** + * Only supports creating a new empty Gdoc or updating an existing one. Does not + * support creating a new Gdoc from an existing one. Relevant updates will + * trigger a deploy. + */ +export async function createOrUpdateGdoc( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { id } = req.params + + if (isEmpty(req.body)) { + return createOrLoadGdocById(trx, id) + } + + const prevGdoc = await getAndLoadGdocById(trx, id) + if (!prevGdoc) throw new JsonError(`No Google Doc with id ${id} found`) + + const nextGdoc = gdocFromJSON(req.body) + await nextGdoc.loadState(trx) + + await addImagesToContentGraph(trx, nextGdoc) + + await setLinksForGdoc( + trx, + nextGdoc.id, + nextGdoc.links, + nextGdoc.published + ? GdocLinkUpdateMode.DeleteAndInsert + : GdocLinkUpdateMode.DeleteOnly + ) + + await upsertGdoc(trx, nextGdoc) + + await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc) + + return nextGdoc +} + +async function validateTombstoneRelatedLinkUrl( + trx: db.KnexReadonlyTransaction, + relatedLink?: string +) { + if (!relatedLink || !relatedLink.startsWith(GDOCS_BASE_URL)) return + const id = relatedLink.match(gdocUrlRegex)?.[1] + if (!id) { + throw new JsonError(`Invalid related link: ${relatedLink}`) + } + const [gdoc] = await getMinimalGdocPostsByIds(trx, [id]) + if (!gdoc) { + throw new JsonError(`Google Doc with ID ${id} not found`) + } + if (!gdoc.published) { + throw new JsonError(`Google Doc with ID ${id} is not published`) + } +} + +export async function deleteGdoc( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { id } = req.params + + const gdoc = await getGdocBaseObjectById(trx, id, false) + if (!gdoc) throw new JsonError(`No Google Doc with id ${id} found`) + + const gdocSlug = getCanonicalUrl("", gdoc) + const { tombstone } = req.body + + if (tombstone) { + await validateTombstoneRelatedLinkUrl(trx, tombstone.relatedLinkUrl) + const slug = gdocSlug.replace("/", "") + const { relatedLinkThumbnail } = tombstone + if (relatedLinkThumbnail) { + const thumbnailExists = await db.checkIsImageInDB( + trx, + relatedLinkThumbnail + ) + if (!thumbnailExists) { + throw new JsonError( + `Image with filename "${relatedLinkThumbnail}" not found` + ) + } + } + await trx + .table("posts_gdocs_tombstones") + .insert({ ...tombstone, gdocId: id, slug }) + await trx + .table("redirects") + .insert({ source: gdocSlug, target: `/deleted${gdocSlug}` }) + } + + await trx + .table("posts") + .where({ gdocSuccessorId: gdoc.id }) + .update({ gdocSuccessorId: null }) + + await trx.table(PostsGdocsLinksTableName).where({ sourceId: id }).delete() + await trx.table(PostsGdocsXImagesTableName).where({ gdocId: id }).delete() + await trx.table(PostsGdocsTableName).where({ id }).delete() + await trx + .table(PostsGdocsComponentsTableName) + .where({ gdocId: id }) + .delete() + if (gdoc.published && checkIsGdocPostExcludingFragments(gdoc)) { + await removeIndividualGdocPostFromIndex(gdoc) + } + if (gdoc.published) { + if (!tombstone && gdocSlug && gdocSlug !== "/") { + // Assets have TTL of one week in Cloudflare. Add a redirect to make sure + // the page is no longer accessible. + // https://developers.cloudflare.com/pages/configuration/serving-pages/#asset-retention + console.log(`Creating redirect for "${gdocSlug}" to "/"`) + await db.knexRawInsert( + trx, + `INSERT INTO redirects (source, target, ttl) + VALUES (?, ?, DATE_ADD(NOW(), INTERVAL 8 DAY))`, + [gdocSlug, "/"] + ) + } + await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`) + } + return {} +} + +export async function setGdocTags( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { gdocId } = req.params + const { tagIds } = req.body + const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ + id: id, + })) + + await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/images.ts b/adminSiteServer/apiRoutes/images.ts new file mode 100644 index 00000000000..e99bf86d885 --- /dev/null +++ b/adminSiteServer/apiRoutes/images.ts @@ -0,0 +1,262 @@ +import { DbEnrichedImage, JsonError } from "@ourworldindata/types" +import pMap from "p-map" +import { + validateImagePayload, + processImageContent, + uploadToCloudflare, + deleteFromCloudflare, +} from "../imagesHelpers.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" + +import e, { Request } from "express" +export async function getImagesHandler( + _: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + try { + const images = await db.getCloudflareImages(trx) + res.set("Cache-Control", "no-store") + res.send({ images }) + } catch (error) { + console.error("Error fetching images", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) + } +} + +export async function postImageHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { filename, type, content } = validateImagePayload(req.body) + + const { asBlob, dimensions, hash } = await processImageContent( + content, + type + ) + + const collision = await trx("images") + .where({ + hash, + replacedBy: null, + }) + .first() + + if (collision) { + return { + success: false, + error: `An image with this content already exists (filename: ${collision.filename})`, + } + } + + const preexisting = await trx("images") + .where("filename", "=", filename) + .first() + + if (preexisting) { + return { + success: false, + error: "An image with this filename already exists", + } + } + + const cloudflareId = await uploadToCloudflare(filename, asBlob) + + if (!cloudflareId) { + return { + success: false, + error: "Failed to upload image", + } + } + + await trx("images").insert({ + filename, + originalWidth: dimensions.width, + originalHeight: dimensions.height, + cloudflareId, + updatedAt: new Date().getTime(), + userId: res.locals.user.id, + hash, + }) + + const image = await db.getCloudflareImage(trx, filename) + + return { + success: true, + image, + } +} +/** + * Similar to the POST route, but for updating an existing image. + * Creates a new image entry in the database and uploads the new image to Cloudflare. + * The old image is marked as replaced by the new image. + */ +export async function putImageHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { type, content } = validateImagePayload(req.body) + const { asBlob, dimensions, hash } = await processImageContent( + content, + type + ) + const collision = await trx("images") + .where({ + hash, + replacedBy: null, + }) + .first() + + if (collision) { + return { + success: false, + error: `An exact copy of this image already exists (filename: ${collision.filename})`, + } + } + + const { id } = req.params + + const image = await trx("images") + .where("id", "=", id) + .first() + + if (!image) { + throw new JsonError(`No image found for id ${id}`, 404) + } + + const originalCloudflareId = image.cloudflareId + const originalFilename = image.filename + const originalAltText = image.defaultAlt + + if (!originalCloudflareId) { + throw new JsonError( + `Image with id ${id} has no associated Cloudflare image`, + 400 + ) + } + + const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob) + + if (!newCloudflareId) { + throw new JsonError("Failed to upload image", 500) + } + + const [newImageId] = await trx("images").insert({ + filename: originalFilename, + originalWidth: dimensions.width, + originalHeight: dimensions.height, + cloudflareId: newCloudflareId, + updatedAt: new Date().getTime(), + userId: res.locals.user.id, + defaultAlt: originalAltText, + hash, + version: image.version + 1, + }) + + await trx("images").where("id", "=", id).update({ + replacedBy: newImageId, + }) + + const updated = await db.getCloudflareImage(trx, originalFilename) + + await triggerStaticBuild( + res.locals.user, + `Updating image "${originalFilename}"` + ) + + return { + success: true, + image: updated, + } +} +// Update alt text via patch +export async function patchImageHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { id } = req.params + + const image = await trx("images") + .where("id", "=", id) + .first() + + if (!image) { + throw new JsonError(`No image found for id ${id}`, 404) + } + + const patchableImageProperties = ["defaultAlt"] as const + const patch = lodash.pick(req.body, patchableImageProperties) + + if (Object.keys(patch).length === 0) { + throw new JsonError("No patchable properties provided", 400) + } + + await trx("images").where({ id }).update(patch) + + const updated = await trx("images") + .where("id", "=", id) + .first() + + return { + success: true, + image: updated, + } +} + +export async function deleteImageHandler( + req: Request, + _: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const { id } = req.params + + const image = await trx("images") + .where("id", "=", id) + .first() + + if (!image) { + throw new JsonError(`No image found for id ${id}`, 404) + } + if (!image.cloudflareId) { + throw new JsonError(`Image does not have a cloudflare ID`, 400) + } + + const replacementChain = await db.selectReplacementChainForImage(trx, id) + + await pMap( + replacementChain, + async (image) => { + if (image.cloudflareId) { + await deleteFromCloudflare(image.cloudflareId) + } + }, + { concurrency: 5 } + ) + + // There's an ON DELETE CASCADE which will delete the replacements + await trx("images").where({ id }).delete() + + return { + success: true, + } +} + +export async function getImageUsageHandler( + _: Request, + __: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const usage = await db.getImageUsage(trx) + + return { + success: true, + usage, + } +} diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts new file mode 100644 index 00000000000..9556e3d4860 --- /dev/null +++ b/adminSiteServer/apiRoutes/mdims.ts @@ -0,0 +1,34 @@ +import { JsonError, MultiDimDataPageConfigRaw } from "@ourworldindata/types" +import { isMultiDimDataPagePublished } from "../../db/model/MultiDimDataPage.js" +import { isValidSlug } from "../../serverUtils/serverUtil.js" +import { + FEATURE_FLAGS, + FeatureFlagFeature, +} from "../../settings/clientSettings.js" +import { createMultiDimConfig } from "../multiDim.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import e, { Request } from "express" + +export async function handleMultiDimDataPageRequest( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { slug } = req.params + if (!isValidSlug(slug)) { + throw new JsonError(`Invalid multi-dim slug ${slug}`) + } + const rawConfig = req.body as MultiDimDataPageConfigRaw + const id = await createMultiDimConfig(trx, slug, rawConfig) + if ( + FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && + (await isMultiDimDataPagePublished(trx, slug)) + ) { + await triggerStaticBuild( + res.locals.user, + `Publishing multidimensional chart ${slug}` + ) + } + return { success: true, id } +} diff --git a/adminSiteServer/apiRoutes/misc.ts b/adminSiteServer/apiRoutes/misc.ts new file mode 100644 index 00000000000..80ec73ba290 --- /dev/null +++ b/adminSiteServer/apiRoutes/misc.ts @@ -0,0 +1,176 @@ +// Get an ArchieML output of all the work produced by an author. This includes +// gdoc articles, gdoc modular/linear topic pages and wordpress modular topic +// pages. Data insights are excluded. This is used to manually populate the +// [.secondary] section of the {.research-and-writing} block of author pages + +import { DbRawPostGdoc, JsonError } from "@ourworldindata/types" + +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import path from "path" +import { expectInt } from "../../serverUtils/serverUtil.js" +import e, { Request } from "express" +// using the alternate template, which highlights topics rather than articles. +export async function fetchAllWork( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + type WordpressPageRecord = { + isWordpressPage: number + } & Record< + "slug" | "title" | "subtitle" | "thumbnail" | "authors" | "publishedAt", + string + > + type GdocRecord = Pick + + const author = req.query.author + const gdocs = await db.knexRaw( + trx, + `-- sql + SELECT id, publishedAt + FROM posts_gdocs + WHERE JSON_CONTAINS(content->'$.authors', '"${author}"') + AND type NOT IN ("data-insight", "fragment") + AND published = 1 + ` + ) + + // type: page + const wpModularTopicPages = await db.knexRaw( + trx, + `-- sql + SELECT + wpApiSnapshot->>"$.slug" as slug, + wpApiSnapshot->>"$.title.rendered" as title, + wpApiSnapshot->>"$.excerpt.rendered" as subtitle, + TRUE as isWordpressPage, + wpApiSnapshot->>"$.authors_name" as authors, + wpApiSnapshot->>"$.featured_media_paths.medium_large" as thumbnail, + wpApiSnapshot->>"$.date" as publishedAt + FROM posts p + WHERE wpApiSnapshot->>"$.content" LIKE '%topic-page%' + AND JSON_CONTAINS(wpApiSnapshot->'$.authors_name', '"${author}"') + AND wpApiSnapshot->>"$.status" = 'publish' + AND NOT EXISTS ( + SELECT 1 FROM posts_gdocs pg + WHERE pg.slug = p.slug + AND pg.content->>'$.type' LIKE '%topic-page' + ) + ` + ) + + const isWordpressPage = ( + post: WordpressPageRecord | GdocRecord + ): post is WordpressPageRecord => + (post as WordpressPageRecord).isWordpressPage === 1 + + function* generateProperty(key: string, value: string) { + yield `${key}: ${value}\n` + } + + const sortByDateDesc = ( + a: GdocRecord | WordpressPageRecord, + b: GdocRecord | WordpressPageRecord + ): number => { + if (!a.publishedAt || !b.publishedAt) return 0 + return ( + new Date(b.publishedAt).getTime() - + new Date(a.publishedAt).getTime() + ) + } + + function* generateAllWorkArchieMl() { + for (const post of [...gdocs, ...wpModularTopicPages].sort( + sortByDateDesc + )) { + if (isWordpressPage(post)) { + yield* generateProperty( + "url", + `https://ourworldindata.org/${post.slug}` + ) + yield* generateProperty("title", post.title) + yield* generateProperty("subtitle", post.subtitle) + yield* generateProperty( + "authors", + JSON.parse(post.authors).join(", ") + ) + const parsedPath = path.parse(post.thumbnail) + yield* generateProperty( + "filename", + // /app/uploads/2021/09/reducing-fertilizer-768x301.png -> reducing-fertilizer.png + path.format({ + name: parsedPath.name.replace(/-\d+x\d+$/, ""), + ext: parsedPath.ext, + }) + ) + yield "\n" + } else { + // this is a gdoc + yield* generateProperty( + "url", + `https://docs.google.com/document/d/${post.id}/edit` + ) + yield "\n" + } + } + } + + res.type("text/plain") + return [...generateAllWorkArchieMl()].join("") +} + +export async function fetchNamespaces( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const rows = await db.knexRaw<{ + name: string + description?: string + isArchived: boolean + }>( + trx, + `SELECT DISTINCT + namespace AS name, + namespaces.description AS description, + namespaces.isArchived AS isArchived + FROM active_datasets + JOIN namespaces ON namespaces.name = active_datasets.namespace` + ) + + return { + namespaces: lodash + .sortBy(rows, (row) => row.description) + .map((namespace) => ({ + ...namespace, + isArchived: !!namespace.isArchived, + })), + } +} + +export async function fetchSourceById( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const sourceId = expectInt(req.params.sourceId) + + const source = await db.knexRawFirst>( + trx, + ` + SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace + FROM sources AS s + JOIN active_datasets AS d ON d.id=s.datasetId + WHERE s.id=?`, + [sourceId] + ) + if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) + source.variables = await db.knexRaw( + trx, + `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, + [sourceId] + ) + + return { source: source } +} diff --git a/adminSiteServer/apiRoutes/posts.ts b/adminSiteServer/apiRoutes/posts.ts new file mode 100644 index 00000000000..78dad0983ff --- /dev/null +++ b/adminSiteServer/apiRoutes/posts.ts @@ -0,0 +1,214 @@ +import { + PostsTableName, + DbRawPost, + DbRawPostWithGdocPublishStatus, + JsonError, + OwidGdocPostInterface, + OwidGdocType, + PostsGdocsTableName, +} from "@ourworldindata/types" +import { camelCaseProperties } from "@ourworldindata/utils" +import { createGdocAndInsertOwidGdocPostContent } from "../../db/model/Gdoc/archieToGdoc.js" +import { upsertGdoc, setTagsForGdoc } from "../../db/model/Gdoc/GdocFactory.js" +import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" +import { setTagsForPost, getTagsByPostId } from "../../db/model/Post.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import * as db from "../../db/db.js" +import e, { Request } from "express" +export async function handleGetPostsJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const raw_rows = await db.knexRaw( + trx, + `-- sql + WITH + posts_tags_aggregated AS ( + SELECT + post_id, + IF( + COUNT(tags.id) = 0, + JSON_ARRAY(), + JSON_ARRAYAGG(JSON_OBJECT("id", tags.id, "name", tags.name)) + ) AS tags + FROM + post_tags + LEFT JOIN tags ON tags.id = post_tags.tag_id + GROUP BY + post_id + ), + post_gdoc_slug_successors AS ( + SELECT + posts.id, + IF( + COUNT(gdocSlugSuccessor.id) = 0, + JSON_ARRAY(), + JSON_ARRAYAGG( + JSON_OBJECT("id", gdocSlugSuccessor.id, "published", gdocSlugSuccessor.published) + ) + ) AS gdocSlugSuccessors + FROM + posts + LEFT JOIN posts_gdocs gdocSlugSuccessor ON gdocSlugSuccessor.slug = posts.slug + GROUP BY + posts.id + ) + SELECT + posts.id AS id, + posts.title AS title, + posts.type AS TYPE, + posts.slug AS slug, + STATUS, + updated_at_in_wordpress, + posts.authors, + posts_tags_aggregated.tags AS tags, + gdocSuccessorId, + gdocSuccessor.published AS isGdocSuccessorPublished, + -- posts can either have explict successors via the gdocSuccessorId column + -- or implicit successors if a gdoc has been created that uses the same slug + -- as a Wp post (the gdoc one wins once it is published) + post_gdoc_slug_successors.gdocSlugSuccessors AS gdocSlugSuccessors + FROM + posts + LEFT JOIN post_gdoc_slug_successors ON post_gdoc_slug_successors.id = posts.id + LEFT JOIN posts_gdocs gdocSuccessor ON gdocSuccessor.id = posts.gdocSuccessorId + LEFT JOIN posts_tags_aggregated ON posts_tags_aggregated.post_id = posts.id + ORDER BY + updated_at_in_wordpress DESC`, + [] + ) + const rows = raw_rows.map((row: any) => ({ + ...row, + tags: JSON.parse(row.tags), + isGdocSuccessorPublished: !!row.isGdocSuccessorPublished, + gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors), + authors: JSON.parse(row.authors), + })) + + return { posts: rows } +} + +export async function handleSetTagsForPost( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + await setTagsForPost(trx, postId, req.body.tagIds) + return { success: true } +} + +export async function handleGetPostById( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const postId = expectInt(req.params.postId) + const post = (await trx + .table(PostsTableName) + .where({ id: postId }) + .select("*") + .first()) as DbRawPost | undefined + return camelCaseProperties({ ...post }) +} + +export async function handleCreateGdoc( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + const allowRecreate = !!req.body.allowRecreate + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined + + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!allowRecreate && existingGdocId) + throw new JsonError("A gdoc already exists for this post", 400) + if (allowRecreate && existingGdocId && post.isGdocPublished) { + throw new JsonError( + "A gdoc already exists for this post and it is already published", + 400 + ) + } + if (post.archieml === null) + throw new JsonError( + `ArchieML was not present for post with id ${postId}`, + 500 + ) + const tagsByPostId = await getTagsByPostId(trx) + const tags = tagsByPostId.get(postId) || [] + const archieMl = JSON.parse( + // Google Docs interprets ®ion in grapher URLS as ®ion + // So we escape them here + post.archieml.replaceAll("&", "&") + ) as OwidGdocPostInterface + const gdocId = await createGdocAndInsertOwidGdocPostContent( + archieMl.content, + post.gdocSuccessorId + ) + // If we did not yet have a gdoc associated with this post, we need to register + // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise + // we don't need to make changes to the DB (only the gdoc regeneration was required) + if (!existingGdocId) { + post.gdocSuccessorId = gdocId + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx + .table(PostsTableName) + .where({ id: postId }) + .update("gdocSuccessorId", gdocId) + + const gdoc = new GdocPost(gdocId) + gdoc.slug = post.slug + gdoc.content.title = post.title + gdoc.content.type = archieMl.content.type || OwidGdocType.Article + gdoc.published = false + gdoc.createdAt = new Date() + gdoc.publishedAt = post.published_at + await upsertGdoc(trx, gdoc) + await setTagsForGdoc(trx, gdocId, tags) + } + return { googleDocsId: gdocId } +} + +export async function handleUnlinkGdoc( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined + + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!existingGdocId) + throw new JsonError("No gdoc exists for this post", 400) + if (existingGdocId && post.isGdocPublished) { + throw new JsonError( + "The GDoc is already published - you can't unlink it", + 400 + ) + } + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx + .table(PostsTableName) + .where({ id: postId }) + .update("gdocSuccessorId", null) + + await trx.table(PostsGdocsTableName).where({ id: existingGdocId }).delete() + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/redirects.ts b/adminSiteServer/apiRoutes/redirects.ts new file mode 100644 index 00000000000..342bdb8b817 --- /dev/null +++ b/adminSiteServer/apiRoutes/redirects.ts @@ -0,0 +1,146 @@ +import { DbPlainChartSlugRedirect, JsonError } from "@ourworldindata/types" +import { getRedirects } from "../../baker/redirects.js" +import { + redirectWithSourceExists, + getChainedRedirect, + getRedirectById, +} from "../../db/model/Redirect.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import e, { Request } from "express" +export async function handleGetSiteRedirects( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await getRedirects(trx) } +} + +export async function handlePostNewSiteRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const { source, target } = req.body + const sourceAsUrl = new URL(source, "https://ourworldindata.org") + if (sourceAsUrl.pathname === "/") + throw new JsonError("Cannot redirect from /", 400) + if (await redirectWithSourceExists(trx, source)) { + throw new JsonError( + `Redirect with source ${source} already exists`, + 400 + ) + } + const chainedRedirect = await getChainedRedirect(trx, source, target) + if (chainedRedirect) { + throw new JsonError( + "Creating this redirect would create a chain, redirect from " + + `${chainedRedirect.source} to ${chainedRedirect.target} ` + + "already exists. " + + (target === chainedRedirect.source + ? `Please create the redirect from ${source} to ` + + `${chainedRedirect.target} directly instead.` + : `Please delete the existing redirect and create a ` + + `new redirect from ${chainedRedirect.source} to ` + + `${target} instead.`), + 400 + ) + } + const { insertId: id } = await db.knexRawInsert( + trx, + `INSERT INTO redirects (source, target) VALUES (?, ?)`, + [source, target] + ) + await triggerStaticBuild( + res.locals.user, + `Creating redirect id=${id} source=${source} target=${target}` + ) + return { success: true, redirect: { id, source, target } } +} + +export async function handleDeleteSiteRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + const redirect = await getRedirectById(trx, id) + if (!redirect) { + throw new JsonError(`No redirect found for id ${id}`, 404) + } + await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` + ) + return { success: true } +} + +export async function handleGetRedirects( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { + redirects: await db.knexRaw( + trx, + `-- sql + SELECT + r.id, + r.slug, + r.chart_id as chartId, + chart_configs.slug AS chartSlug + FROM chart_slug_redirects AS r + JOIN charts ON charts.id = r.chart_id + JOIN chart_configs ON chart_configs.id = charts.configId + ORDER BY r.id DESC + ` + ), + } +} + +export async function handlePostNewChartRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const chartId = expectInt(req.params.chartId) + const fields = req.body as { slug: string } + const result = await db.knexRawInsert( + trx, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [chartId, fields.slug] + ) + const redirectId = result.insertId + const redirect = await db.knexRaw( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [redirectId] + ) + return { success: true, redirect: redirect } +} + +export async function handleDeleteChartRedirect( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + + const redirect = await db.knexRawFirst( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [id] + ) + + if (!redirect) throw new JsonError(`No redirect found for id ${id}`, 404) + + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect from ${redirect.slug}` + ) + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/routeUtils.ts b/adminSiteServer/apiRoutes/routeUtils.ts new file mode 100644 index 00000000000..0e647f290c0 --- /dev/null +++ b/adminSiteServer/apiRoutes/routeUtils.ts @@ -0,0 +1,44 @@ +import { DbPlainUser } from "@ourworldindata/types" +import { DeployQueueServer } from "../../baker/DeployQueueServer.js" +import { BAKE_ON_CHANGE } from "../../settings/serverSettings.js" + +// Call this to trigger build and deployment of static charts on change +export const triggerStaticBuild = async ( + user: DbPlainUser, + commitMessage: string +) => { + if (!BAKE_ON_CHANGE) { + console.log( + "Not triggering static build because BAKE_ON_CHANGE is false" + ) + return + } + + return new DeployQueueServer().enqueueChange({ + timeISOString: new Date().toISOString(), + authorName: user.fullName, + authorEmail: user.email, + message: commitMessage, + }) +} + +export const enqueueLightningChange = async ( + user: DbPlainUser, + commitMessage: string, + slug: string +) => { + if (!BAKE_ON_CHANGE) { + console.log( + "Not triggering static build because BAKE_ON_CHANGE is false" + ) + return + } + + return new DeployQueueServer().enqueueChange({ + timeISOString: new Date().toISOString(), + authorName: user.fullName, + authorEmail: user.email, + message: commitMessage, + slug, + }) +} diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts new file mode 100644 index 00000000000..bc165128e2e --- /dev/null +++ b/adminSiteServer/apiRoutes/suggest.ts @@ -0,0 +1,63 @@ +import { + DbChartTagJoin, + JsonError, + DbEnrichedImage, +} from "@ourworldindata/types" +import { parseIntOrUndefined } from "@ourworldindata/utils" +import { getGptTopicSuggestions } from "../../db/model/Chart.js" +import { CLOUDFLARE_IMAGES_URL } from "../../settings/clientSettings.js" +import { fetchGptGeneratedAltText } from "../imagesHelpers.js" +import * as db from "../../db/db.js" +import e, { Request } from "express" + +export async function suggestGptTopics( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise> { + const chartId = parseIntOrUndefined(req.params.chartId) + if (!chartId) throw new JsonError(`Invalid chart ID`, 400) + + const topics = await getGptTopicSuggestions(trx, chartId) + + if (!topics.length) + throw new JsonError( + `No GPT topic suggestions found for chart ${chartId}`, + 404 + ) + + return { + topics, + } +} + +export async function suggestGptAltText( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +): Promise<{ + success: boolean + altText: string | null +}> { + const imageId = parseIntOrUndefined(req.params.imageId) + if (!imageId) throw new JsonError(`Invalid image ID`, 400) + const image = await trx("images") + .where("id", imageId) + .first() + if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) + + const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` + let altText: string | null = "" + try { + altText = await fetchGptGeneratedAltText(src) + } catch (error) { + console.error(`Error fetching GPT alt text for image ${imageId}`, error) + throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) + } + + if (!altText) { + throw new JsonError(`Unable to generate alt text for image`, 404) + } + + return { success: true, altText } +} diff --git a/adminSiteServer/apiRoutes/tagGraph.ts b/adminSiteServer/apiRoutes/tagGraph.ts new file mode 100644 index 00000000000..5e0df422a22 --- /dev/null +++ b/adminSiteServer/apiRoutes/tagGraph.ts @@ -0,0 +1,61 @@ +import { JsonError, FlatTagGraph } from "@ourworldindata/types" +import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils" +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import e, { Request } from "express" + +export async function handleGetFlatTagGraph( + req: Request, + res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const flatTagGraph = await db.getFlatTagGraph(trx) + return flatTagGraph +} + +export async function handlePostTagGraph( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tagGraph = req.body?.tagGraph as unknown + if (!tagGraph) { + throw new JsonError("No tagGraph provided", 400) + } + + function validateFlatTagGraph( + tagGraph: Record + ): tagGraph is FlatTagGraph { + if (lodash.isObject(tagGraph)) { + for (const [key, value] of Object.entries(tagGraph)) { + if (!lodash.isString(key) && isNaN(Number(key))) { + return false + } + if (!lodash.isArray(value)) { + return false + } + for (const tag of value) { + if ( + !( + checkIsPlainObjectWithGuard(tag) && + lodash.isNumber(tag.weight) && + lodash.isNumber(tag.parentId) && + lodash.isNumber(tag.childId) + ) + ) { + return false + } + } + } + } + + return true + } + + const isValid = validateFlatTagGraph(tagGraph) + if (!isValid) { + throw new JsonError("Invalid tag graph provided", 400) + } + await db.updateTagGraph(trx, tagGraph) + res.send({ success: true }) +} diff --git a/adminSiteServer/apiRoutes/tags.ts b/adminSiteServer/apiRoutes/tags.ts new file mode 100644 index 00000000000..d40d3bfd475 --- /dev/null +++ b/adminSiteServer/apiRoutes/tags.ts @@ -0,0 +1,260 @@ +import { + DbPlainTag, + DbPlainDataset, + DbRawPostGdoc, + JsonError, +} from "@ourworldindata/types" +import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils" +import { + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, +} from "../../db/model/Chart.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { UNCATEGORIZED_TAG_ID } from "../../settings/serverSettings.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import e, { Request } from "express" + +export async function getTagById( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const tagId = expectInt(req.params.tagId) as number | null + + // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff + // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag + // every time we create a new chart etcs + const uncategorized = tagId === UNCATEGORIZED_TAG_ID + + // TODO: when we have types for our endpoints, make tag of that type instead of any + const tag: any = await db.knexRawFirst< + Pick< + DbPlainTag, + "id" | "name" | "specialType" | "updatedAt" | "parentId" | "slug" + > + >( + trx, + `-- sql + SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug + FROM tags t LEFT JOIN tags p ON t.parentId=p.id + WHERE t.id = ? + `, + [tagId] + ) + + // Datasets tagged with this tag + const datasets = await db.knexRaw< + Pick< + DbPlainDataset, + | "id" + | "namespace" + | "name" + | "description" + | "createdAt" + | "updatedAt" + | "dataEditedAt" + | "isPrivate" + | "nonRedistributable" + > & { dataEditedByUserName: string } + >( + trx, + `-- sql + SELECT + d.id, + d.namespace, + d.name, + d.description, + d.createdAt, + d.updatedAt, + d.dataEditedAt, + du.fullName AS dataEditedByUserName, + d.isPrivate, + d.nonRedistributable + FROM active_datasets d + JOIN users du ON du.id=d.dataEditedByUserId + LEFT JOIN dataset_tags dt ON dt.datasetId = d.id + WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} + ORDER BY d.dataEditedAt DESC + `, + uncategorized ? [] : [tagId] + ) + tag.datasets = datasets + + // The other tags for those datasets + if (tag.datasets.length) { + if (uncategorized) { + for (const dataset of tag.datasets) dataset.tags = [] + } else { + const datasetTags = await db.knexRaw<{ + datasetId: number + id: number + name: string + }>( + trx, + `-- sql + SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt + JOIN tags t ON dt.tagId = t.id + WHERE dt.datasetId IN (?) + `, + [tag.datasets.map((d: any) => d.id)] + ) + const tagsByDatasetId = lodash.groupBy( + datasetTags, + (t) => t.datasetId + ) + for (const dataset of tag.datasets) { + dataset.tags = tagsByDatasetId[dataset.id].map((t) => + lodash.omit(t, "datasetId") + ) + } + } + } + + // Charts using datasets under this tag + const charts = await db.knexRaw( + trx, + `-- sql + SELECT ${oldChartFieldList} FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + LEFT JOIN chart_tags ct ON ct.chartId=charts.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"} + GROUP BY charts.id + ORDER BY charts.updatedAt DESC + `, + uncategorized ? [] : [tagId] + ) + tag.charts = charts + + await assignTagsForCharts(trx, charts) + + // Subcategories + const children = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql + SELECT t.id, t.name FROM tags t + WHERE t.parentId = ? + `, + [tag.id] + ) + tag.children = children + + const possibleParents = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql + SELECT t.id, t.name FROM tags t + WHERE t.parentId IS NULL + ` + ) + tag.possibleParents = possibleParents + + return { + tag, + } +} + +export async function updateTag( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tagId = expectInt(req.params.tagId) + const tag = (req.body as { tag: any }).tag + await db.knexRaw( + trx, + `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, + [tag.name, new Date(), tag.slug, tagId] + ) + if (tag.slug) { + // See if there's a published gdoc with a matching slug. + // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, + // where the page for the topic is just an article. + const gdoc = await db.knexRaw>( + trx, + `-- sql + SELECT slug FROM posts_gdocs pg + WHERE EXISTS ( + SELECT 1 + FROM posts_gdocs_x_tags gt + WHERE pg.id = gt.gdocId AND gt.tagId = ? + ) AND pg.published = TRUE AND pg.slug = ?`, + [tagId, tag.slug] + ) + if (!gdoc.length) { + return { + success: true, + tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. + +Are you sure you haven't made a typo?`, + } + } + } + return { success: true } +} + +export async function createTag( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tag = req.body + function validateTag( + tag: unknown + ): tag is { name: string; slug: string | null } { + return ( + checkIsPlainObjectWithGuard(tag) && + typeof tag.name === "string" && + (tag.slug === null || + (typeof tag.slug === "string" && tag.slug !== "")) + ) + } + if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) + + const conflictingTag = await db.knexRawFirst<{ + name: string + slug: string | null + }>( + trx, + `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, + [tag.name, tag.slug] + ) + if (conflictingTag) + throw new JsonError( + conflictingTag.name === tag.name + ? `Tag with name ${tag.name} already exists` + : `Tag with slug ${tag.slug} already exists`, + 400 + ) + + const now = new Date() + const result = await db.knexRawInsert( + trx, + `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, + // parentId will be deprecated soon once we migrate fully to the tag graph + [tag.name, tag.slug, now, now] + ) + return { success: true, tagId: result.insertId } +} + +export async function getAllTags( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { tags: await db.getMinimalTagsWithIsTopic(trx) } +} + +export async function deleteTag( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const tagId = expectInt(req.params.tagId) + + await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) + + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/users.ts b/adminSiteServer/apiRoutes/users.ts new file mode 100644 index 00000000000..82e8da958c3 --- /dev/null +++ b/adminSiteServer/apiRoutes/users.ts @@ -0,0 +1,114 @@ +import { DbPlainUser, UsersTableName, JsonError } from "@ourworldindata/types" +import { parseIntOrUndefined } from "@ourworldindata/utils" +import { pick } from "lodash" +import { getUserById, updateUser, insertUser } from "../../db/model/User.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import * as db from "../../db/db.js" +import e, { Request } from "express" +export async function getUsers( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + return { + users: await trx + .select( + "id" satisfies keyof DbPlainUser, + "email" satisfies keyof DbPlainUser, + "fullName" satisfies keyof DbPlainUser, + "isActive" satisfies keyof DbPlainUser, + "isSuperuser" satisfies keyof DbPlainUser, + "createdAt" satisfies keyof DbPlainUser, + "updatedAt" satisfies keyof DbPlainUser, + "lastLogin" satisfies keyof DbPlainUser, + "lastSeen" satisfies keyof DbPlainUser + ) + .from(UsersTableName) + .orderBy("lastSeen", "desc"), + } +} + +export async function getUserByIdHandler( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const id = parseIntOrUndefined(req.params.userId) + if (!id) throw new JsonError("No user id given") + const user = await getUserById(trx, id) + return { user } +} + +export async function deleteUser( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = expectInt(req.params.userId) + await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) + + return { success: true } +} + +export async function updateUserHandler( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = parseIntOrUndefined(req.params.userId) + const user = userId !== undefined ? await getUserById(trx, userId) : null + if (!user) throw new JsonError("No such user", 404) + + user.fullName = req.body.fullName + user.isActive = req.body.isActive + + await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) + + return { success: true } +} + +export async function addUser( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const { email, fullName } = req.body + + await insertUser(trx, { + email, + fullName, + }) + + return { success: true } +} + +export async function addImageToUser( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId }).update({ userId }) + return { success: true } +} + +export async function removeUserImage( + req: Request, + _res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId, userId }).update({ userId: null }) + return { success: true } +} diff --git a/adminSiteServer/apiRoutes/variables.ts b/adminSiteServer/apiRoutes/variables.ts new file mode 100644 index 00000000000..414c265db7a --- /dev/null +++ b/adminSiteServer/apiRoutes/variables.ts @@ -0,0 +1,541 @@ +import { + getVariableDataRoute, + getVariableMetadataRoute, + migrateGrapherConfigToLatestVersion, +} from "@ourworldindata/grapher" +import { + DbRawVariable, + DbPlainDataset, + JsonError, + DbPlainChart, + DbRawChartConfig, + GrapherInterface, + OwidVariableWithSource, + parseChartConfig, +} from "@ourworldindata/types" +import { + fetchS3DataValuesByPath, + fetchS3MetadataByPath, + getAllChartsForIndicator, + getGrapherConfigsForVariable, + getMergedGrapherConfigForVariable, + searchVariables, + updateAllChartsThatInheritFromIndicator, + updateAllMultiDimViewsThatInheritFromIndicator, + updateGrapherConfigAdminOfVariable, + updateGrapherConfigETLOfVariable, +} from "../../db/model/Variable.js" +import { DATA_API_URL } from "../../settings/clientSettings.js" +import * as db from "../../db/db.js" +import { + getParentVariableIdFromChartConfig, + omit, + parseIntOrUndefined, +} from "@ourworldindata/utils" +import { + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, +} from "../../db/model/Chart.js" +import { updateExistingFullConfig } from "../../db/model/ChartConfigs.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as lodash from "lodash" +import { updateGrapherConfigsInR2 } from "./charts.js" +import e, { Request } from "express" + +export async function getEditorVariablesJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const datasets = [] + const rows = await db.knexRaw< + Pick & { + datasetId: number + datasetName: string + datasetVersion: string + } & Pick< + DbPlainDataset, + "namespace" | "isPrivate" | "nonRedistributable" + > + >( + trx, + `-- sql + SELECT + v.name, + v.id, + d.id as datasetId, + d.name as datasetName, + d.version as datasetVersion, + d.namespace, + d.isPrivate, + d.nonRedistributable + FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id + ORDER BY d.updatedAt DESC + ` + ) + + let dataset: + | { + id: number + name: string + version: string + namespace: string + isPrivate: boolean + nonRedistributable: boolean + variables: { id: number; name: string }[] + } + | undefined + for (const row of rows) { + if (!dataset || row.datasetName !== dataset.name) { + if (dataset) datasets.push(dataset) + + dataset = { + id: row.datasetId, + name: row.datasetName, + version: row.datasetVersion, + namespace: row.namespace, + isPrivate: !!row.isPrivate, + nonRedistributable: !!row.nonRedistributable, + variables: [], + } + } + + dataset.variables.push({ + id: row.id, + name: row.name ?? "", + }) + } + + if (dataset) datasets.push(dataset) + + return { datasets: datasets } +} + +export async function getVariableDataJson( + req: Request, + _res: e.Response>, + _trx: db.KnexReadonlyTransaction +) { + const variableStr = req.params.variableStr as string + if (!variableStr) throw new JsonError("No variable id given") + if (variableStr.includes("+")) + throw new JsonError( + "Requesting multiple variables at the same time is no longer supported" + ) + const variableId = parseInt(variableStr) + if (isNaN(variableId)) throw new JsonError("Invalid variable id") + return await fetchS3DataValuesByPath( + getVariableDataRoute(DATA_API_URL, variableId) + "?nocache" + ) +} + +export async function getVariableMetadataJson( + req: Request, + _res: e.Response>, + _trx: db.KnexReadonlyTransaction +) { + const variableStr = req.params.variableStr as string + if (!variableStr) throw new JsonError("No variable id given") + if (variableStr.includes("+")) + throw new JsonError( + "Requesting multiple variables at the same time is no longer supported" + ) + const variableId = parseInt(variableStr) + if (isNaN(variableId)) throw new JsonError("Invalid variable id") + return await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) +} + +export async function getVariablesJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 + const query = req.query.search as string + return await searchVariables(query, limit, trx) +} + +export async function getVariablesUsagesJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const query = `-- sql + SELECT + variableId, + COUNT(DISTINCT chartId) AS usageCount + FROM + chart_dimensions + GROUP BY + variableId + ORDER BY + usageCount DESC` + + const rows = await db.knexRaw(trx, query) + + return rows +} + +export async function getVariablesGrapherConfigETLPatchConfigJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + return variable.etl?.patchConfig ?? {} +} + +export async function getVariablesGrapherConfigAdminPatchConfigJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + return variable.admin?.patchConfig ?? {} +} + +export async function getVariablesMergedGrapherConfigJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const config = await getMergedGrapherConfigForVariable(trx, variableId) + return config ?? {} +} + +export async function getVariablesVariableIdJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + + const variable = await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) + + // XXX: Patch shortName onto the end of catalogPath when it's missing, + // a temporary hack since our S3 metadata is out of date with our DB. + // See: https://github.com/owid/etl/issues/2135 + if (variable.catalogPath && !variable.catalogPath.includes("#")) { + variable.catalogPath += `#${variable.shortName}` + } + + const rawCharts = await db.knexRaw< + OldChartFieldList & { + isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] + config: DbRawChartConfig["full"] + } + >( + trx, + `-- sql + SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + JOIN chart_dimensions cd ON cd.chartId = charts.id + WHERE cd.variableId = ? + GROUP BY charts.id + `, + [variableId] + ) + + // check for parent indicators + const charts = rawCharts.map((chart) => { + const parentIndicatorId = getParentVariableIdFromChartConfig( + parseChartConfig(chart.config) + ) + const hasParentIndicator = parentIndicatorId !== undefined + return omit({ ...chart, hasParentIndicator }, "config") + }) + + await assignTagsForCharts(trx, charts) + + const variableWithConfigs = await getGrapherConfigsForVariable( + trx, + variableId + ) + const grapherConfigETL = variableWithConfigs?.etl?.patchConfig + const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig + const mergedGrapherConfig = + variableWithConfigs?.admin?.fullConfig ?? + variableWithConfigs?.etl?.fullConfig + + // add the variable's display field to the merged grapher config + if (mergedGrapherConfig) { + const [varDims, otherDims] = lodash.partition( + mergedGrapherConfig.dimensions ?? [], + (dim) => dim.variableId === variableId + ) + const varDimsWithDisplay = varDims.map((dim) => ({ + display: variable.display, + ...dim, + })) + mergedGrapherConfig.dimensions = [...varDimsWithDisplay, ...otherDims] + } + + const variableWithCharts: OwidVariableWithSource & { + charts: Record + grapherConfig: GrapherInterface | undefined + grapherConfigETL: GrapherInterface | undefined + grapherConfigAdmin: GrapherInterface | undefined + } = { + ...variable, + charts, + grapherConfig: mergedGrapherConfig, + grapherConfigETL, + grapherConfigAdmin, + } + + return { + variable: variableWithCharts, + } /*, vardata: await getVariableData([variableId]) }*/ +} + +export async function putVariablesVariableIdGrapherConfigETL( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), + } + } + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigETLOfVariable(trx, variable, validConfig) + + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) + } + + return { success: true, savedPatch } +} + +export async function deleteVariablesVariableIdGrapherConfigETL( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + // no-op if the variable doesn't have an ETL config + if (!variable.etl) return { success: true } + + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql + UPDATE variables + SET grapherConfigIdETL = NULL + WHERE id = ? + `, + [variableId] + ) + + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql + DELETE FROM chart_configs + WHERE id = ? + `, + [variable.etl.configId] + ) + + // update admin config if there is one + if (variable.admin) { + await updateExistingFullConfig(trx, { + configId: variable.admin.configId, + config: variable.admin.patchConfig, + updatedAt: now, + }) + } + + const updates = { + patchConfigAdmin: variable.admin?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( + trx, + variableId, + updates + ) + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) + } + + return { success: true } +} + +export async function putVariablesVariableIdGrapherConfigAdmin( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), + } + } + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) + + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) + } + + return { success: true, savedPatch } +} + +export async function deleteVariablesVariableIdGrapherConfigAdmin( + req: Request, + res: e.Response>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + // no-op if the variable doesn't have an admin-authored config + if (!variable.admin) return { success: true } + + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql + UPDATE variables + SET grapherConfigIdAdmin = NULL + WHERE id = ? + `, + [variableId] + ) + + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql + DELETE FROM chart_configs + WHERE id = ? + `, + [variable.admin.configId] + ) + + const updates = { + patchConfigETL: variable.etl?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( + trx, + variableId, + updates + ) + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) + } + + return { success: true } +} + +export async function getVariablesVariableIdChartsJson( + req: Request, + _res: e.Response>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const charts = await getAllChartsForIndicator(trx, variableId) + return charts.map((chart) => ({ + id: chart.chartId, + title: chart.config.title, + variantName: chart.config.variantName, + isChild: chart.isChild, + isInheritanceEnabled: chart.isInheritanceEnabled, + isPublished: chart.isPublished, + })) +} diff --git a/adminSiteServer/appClass.tsx b/adminSiteServer/appClass.tsx index 1ef3cc0546b..247a23a17b6 100644 --- a/adminSiteServer/appClass.tsx +++ b/adminSiteServer/appClass.tsx @@ -108,8 +108,8 @@ export class OwidAdminApp { app.use("/fonts", express.static("public/fonts")) app.use("/assets-admin", express.static("dist/assets-admin")) - app.use("/api", publicApiRouter.router) - app.use("/admin/api", apiRouter.router) + app.use("/api", publicApiRouter) + app.use("/admin/api", apiRouter) app.use("/admin/test", testPageRouter) app.use("/admin/storybook", express.static(".storybook/build")) app.use("/admin", adminRouter) diff --git a/adminSiteServer/authentication.ts b/adminSiteServer/authentication.ts index 605fb4c6747..6bc420976ee 100644 --- a/adminSiteServer/authentication.ts +++ b/adminSiteServer/authentication.ts @@ -14,12 +14,6 @@ import { Secret, verify } from "jsonwebtoken" import { DbPlainSession, DbPlainUser, JsonError } from "@ourworldindata/utils" import { exec } from "child_process" -export type Request = express.Request - -export interface Response extends express.Response { - locals: { user: DbPlainUser; session: Session } -} - interface Session { id: string expiryDate: Date diff --git a/adminSiteServer/functionalRouterHelpers.ts b/adminSiteServer/functionalRouterHelpers.ts index e9ae007d2bc..d3b1d83ee3b 100644 --- a/adminSiteServer/functionalRouterHelpers.ts +++ b/adminSiteServer/functionalRouterHelpers.ts @@ -1,8 +1,7 @@ -import { FunctionalRouter } from "./FunctionalRouter.js" -import { Request, Response } from "express" +import { Request, Response, Router } from "express" import * as db from "../db/db.js" export function getRouteWithROTransaction( - router: FunctionalRouter, + router: Router, targetPath: string, handler: ( req: Request, @@ -22,7 +21,7 @@ export function getRouteWithROTransaction( fetching it from the google API. */ export function getRouteNonIdempotentWithRWTransaction( - router: FunctionalRouter, + router: Router, targetPath: string, handler: ( req: Request, @@ -38,7 +37,7 @@ export function getRouteNonIdempotentWithRWTransaction( } export function postRouteWithRWTransaction( - router: FunctionalRouter, + router: Router, targetPath: string, handler: ( req: Request, @@ -54,7 +53,7 @@ export function postRouteWithRWTransaction( } export function putRouteWithRWTransaction( - router: FunctionalRouter, + router: Router, targetPath: string, handler: ( req: Request, @@ -70,7 +69,7 @@ export function putRouteWithRWTransaction( } export function patchRouteWithRWTransaction( - router: FunctionalRouter, + router: Router, targetPath: string, handler: ( req: Request, @@ -86,7 +85,7 @@ export function patchRouteWithRWTransaction( } export function deleteRouteWithRWTransaction( - router: FunctionalRouter, + router: Router, targetPath: string, handler: ( req: Request, diff --git a/adminSiteServer/getLogsByChartId.ts b/adminSiteServer/getLogsByChartId.ts new file mode 100644 index 00000000000..bbffc943807 --- /dev/null +++ b/adminSiteServer/getLogsByChartId.ts @@ -0,0 +1,34 @@ +import { Json } from "@ourworldindata/utils" +import * as db from "../db/db.js" + +export async function getLogsByChartId( + knex: db.KnexReadonlyTransaction, + chartId: number +): Promise< + { + userId: number + config: Json + userName: string + createdAt: Date + }[] +> { + const logs = await db.knexRaw<{ + userId: number + config: string + userName: string + createdAt: Date + }>( + knex, + `SELECT userId, config, fullName as userName, l.createdAt + FROM chart_revisions l + LEFT JOIN users u on u.id = userId + WHERE chartId = ? + ORDER BY l.id DESC + LIMIT 50`, + [chartId] + ) + return logs.map((log) => ({ + ...log, + config: JSON.parse(log.config), + })) +} diff --git a/adminSiteServer/publicApiRouter.ts b/adminSiteServer/publicApiRouter.ts index 946c8859ced..21078dfe9c6 100644 --- a/adminSiteServer/publicApiRouter.ts +++ b/adminSiteServer/publicApiRouter.ts @@ -1,14 +1,13 @@ -import { FunctionalRouter } from "./FunctionalRouter.js" -import { Request, Response } from "./authentication.js" +import { Router, Request, Response } from "express" import * as db from "../db/db.js" -export const publicApiRouter = new FunctionalRouter() +export const publicApiRouter = Router() function rejectAfterDelay(ms: number) { return new Promise((resolve, reject) => setTimeout(reject, ms)) } -publicApiRouter.router.get("/health", async (req: Request, res: Response) => { +publicApiRouter.get("/health", async (req: Request, res: Response) => { try { const sqlPromise = db.knexRaw( db.knexInstance() as db.KnexReadonlyTransaction, diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index c422adcaa4e..52f9b4cb51a 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -56,6 +56,7 @@ import { grabMetadataForGdocLinkedIndicator, TombstonePageData, gdocUrlRegex, + ChartViewInfo, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -109,6 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" +import { getChartViewsInfo } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -120,6 +122,7 @@ type PrefetchedAttachments = { explorers: Record } linkedIndicators: Record + linkedChartViews: Record } // These aren't all "wordpress" steps @@ -176,7 +179,7 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number { bakeSteps.has("dataInsights") || bakeSteps.has("authors") ) { - total += 8 + total += 9 } return total } @@ -345,7 +348,7 @@ export class SiteBaker { _prefetchedAttachmentsCache: PrefetchedAttachments | undefined = undefined private async getPrefetchedGdocAttachments( knex: db.KnexReadonlyTransaction, - picks?: [string[], string[], string[], string[], string[]] + picks?: [string[], string[], string[], string[], string[], string[]] ): Promise { if (!this._prefetchedAttachmentsCache) { console.log("Prefetching attachments...") @@ -459,6 +462,12 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) + const chartViewsInfo = await getChartViewsInfo(knex) + const chartViewsInfoByName = keyBy(chartViewsInfo, "name") + this.progressBar.tick({ + name: `✅ Prefetched ${chartViewsInfo.length} chart views`, + }) + const prefetchedAttachments = { donors, linkedAuthors: publishedAuthors, @@ -469,6 +478,7 @@ export class SiteBaker { graphers: publishedChartsBySlug, }, linkedIndicators: datapageIndicatorsById, + linkedChartViews: chartViewsInfoByName, } this.progressBar.tick({ name: "✅ Prefetched attachments" }) this._prefetchedAttachmentsCache = prefetchedAttachments @@ -480,6 +490,7 @@ export class SiteBaker { imageFilenames, linkedGrapherSlugs, linkedExplorerSlugs, + linkedChartViewNames, ] = picks const linkedDocuments = pick( this._prefetchedAttachmentsCache.linkedDocuments, @@ -528,6 +539,10 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), + linkedChartViews: pick( + this._prefetchedAttachmentsCache.linkedChartViews, + linkedChartViewNames + ), } } return this._prefetchedAttachmentsCache @@ -619,6 +634,7 @@ export class SiteBaker { publishedGdoc.linkedImageFilenames, publishedGdoc.linkedChartSlugs.grapher, publishedGdoc.linkedChartSlugs.explorer, + publishedGdoc.linkedChartViewNames, ]) publishedGdoc.donors = attachments.donors publishedGdoc.linkedAuthors = attachments.linkedAuthors @@ -629,6 +645,7 @@ export class SiteBaker { ...attachments.linkedCharts.explorers, } publishedGdoc.linkedIndicators = attachments.linkedIndicators + publishedGdoc.linkedChartViews = attachments.linkedChartViews // this is a no-op if the gdoc doesn't have an all-chart block if ("loadRelatedCharts" in publishedGdoc) { @@ -876,6 +893,7 @@ export class SiteBaker { dataInsight.linkedImageFilenames, dataInsight.linkedChartSlugs.grapher, dataInsight.linkedChartSlugs.explorer, + dataInsight.linkedChartViewNames, ]) dataInsight.linkedDocuments = attachments.linkedDocuments dataInsight.imageMetadata = { @@ -949,6 +967,7 @@ export class SiteBaker { publishedAuthor.linkedImageFilenames, publishedAuthor.linkedChartSlugs.grapher, publishedAuthor.linkedChartSlugs.explorer, + publishedAuthor.linkedChartViewNames, ]) // We don't need these to be attached to the gdoc in the current diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index e53a9e3e521..e64743f9324 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -441,6 +441,7 @@ ${dataInsights latestDataInsights: get(post, "latestDataInsights", []), homepageMetadata: get(post, "homepageMetadata", {}), latestWorkLinks: get(post, "latestWorkLinks", []), + linkedChartViews: get(post, "linkedChartViews", {}), }} > diff --git a/db/migrateWpPostsToArchieMl.ts b/db/migrateWpPostsToArchieMl.ts index f9c35c0e84a..a0938d6c546 100644 --- a/db/migrateWpPostsToArchieMl.ts +++ b/db/migrateWpPostsToArchieMl.ts @@ -20,7 +20,7 @@ import { adjustHeadingLevels, findMinimumHeadingLevel, } from "./model/Gdoc/htmlToEnriched.js" -import { getPostRelatedCharts, isPostSlugCitable } from "./model/Post.js" +import { getPostRelatedCharts } from "./model/Post.js" import { enrichedBlocksToMarkdown } from "./model/Gdoc/enrichedToMarkdown.js" // slugs from all the linear entries we want to migrate from @edomt @@ -131,15 +131,6 @@ const migrate = async (trx: db.KnexReadWriteTransaction): Promise => { relatedCharts = await getPostRelatedCharts(trx, post.id) } - const shouldIncludeMaxAsAuthor = isPostSlugCitable(post.slug) - if ( - shouldIncludeMaxAsAuthor && - post.authors && - !post.authors.includes("Max Roser") - ) { - post.authors.push("Max Roser") - } - // We don't get the first and last nodes if they are comments. // This can cause issues with the wp:components so here we wrap // everything in a div diff --git a/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts b/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts new file mode 100644 index 00000000000..06596eea89b --- /dev/null +++ b/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class PostsGdocsLinksAddChartViews1734454799588 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE posts_gdocs_links + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer', 'chart-view') NULL`) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + ALTER TABLE posts_gdocs_links + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer') NULL`) + } +} diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts new file mode 100644 index 00000000000..2bfd5ad8ed3 --- /dev/null +++ b/db/model/ChartView.ts @@ -0,0 +1,34 @@ +import { ChartViewInfo, JsonString } from "@ourworldindata/types" +import * as db from "../db.js" + +export const getChartViewsInfo = async ( + knex: db.KnexReadonlyTransaction, + names?: string[] +): Promise => { + type RawRow = Omit & { + queryParamsForParentChart: JsonString + } + let rows: RawRow[] + + const query = `-- sql +SELECT cv.name, + cc.full ->> "$.title" as title, + chartConfigId, + pcc.slug as parentChartSlug, + cv.queryParamsForParentChart +FROM chart_views cv +JOIN chart_configs cc on cc.id = cv.chartConfigId +JOIN charts pc on cv.parentChartId = pc.id +JOIN chart_configs pcc on pc.configId = pcc.id + ` + + if (names) { + if (names.length === 0) return [] + rows = await db.knexRaw(knex, `${query} WHERE cv.name IN (?)`, [names]) + } else rows = await db.knexRaw(knex, query) + + return rows.map((row) => ({ + ...row, + queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart), + })) +} diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 754471c247a..173fd06e6fb 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -48,7 +48,7 @@ import { getVariableMetadata, getVariableOfDatapageIfApplicable, } from "../Variable.js" -import { createLinkFromUrl } from "../Link.js" +import { createLinkForChartView, createLinkFromUrl } from "../Link.js" import { getMultiDimDataPageBySlug, isMultiDimDataPagePublished, @@ -56,6 +56,7 @@ import { import { ARCHVED_THUMBNAIL_FILENAME, ChartConfigType, + ChartViewInfo, DEFAULT_THUMBNAIL_FILENAME, GrapherInterface, LatestDataInsight, @@ -66,6 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" +import { getChartViewsInfo } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -89,6 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface { linkedIndicators: Record = {} linkedDocuments: Record = {} latestDataInsights: LatestDataInsight[] = [] + linkedChartViews?: Record = {} _omittableFields: string[] = [] constructor(id?: string) { @@ -292,6 +295,14 @@ export class GdocBase implements OwidGdocBaseInterface { return { grapher: [...grapher], explorer: [...explorer] } } + get linkedChartViewNames(): string[] { + const filteredLinks = this.links + .filter((link) => link.linkType === OwidGdocLinkType.ChartView) + .map((link) => link.target) + + return filteredLinks + } + get hasAllChartsBlock(): boolean { let hasAllChartsBlock = false for (const enrichedBlockSource of this.enrichedBlockSources) { @@ -349,6 +360,13 @@ export class GdocBase implements OwidGdocBaseInterface { componentType: block.type, }), ]) + .with({ type: "narrative-chart" }, (block) => [ + createLinkForChartView({ + name: block.name, + source: this, + componentType: block.type, + }), + ]) .with({ type: "all-charts" }, (block) => block.top.map((item) => createLinkFromUrl({ @@ -710,6 +728,11 @@ export class GdocBase implements OwidGdocBaseInterface { } } + async loadChartViewsInfo(knex: db.KnexReadonlyTransaction): Promise { + const result = await getChartViewsInfo(knex, this.linkedChartViewNames) + this.linkedChartViews = keyBy(result, "name") + } + async fetchAndEnrichGdoc(): Promise { const docsClient = google.docs({ version: "v1", @@ -855,6 +878,7 @@ export class GdocBase implements OwidGdocBaseInterface { await this.loadImageMetadataFromDB(knex) await this.loadLinkedCharts(knex) await this.loadLinkedIndicators() // depends on linked charts + await this.loadChartViewsInfo(knex) await this._loadSubclassAttachments(knex) await this.validate(knex) } diff --git a/db/model/Gdoc/enrichedToMarkdown.ts b/db/model/Gdoc/enrichedToMarkdown.ts index 2556f74ef2b..794700f30a3 100644 --- a/db/model/Gdoc/enrichedToMarkdown.ts +++ b/db/model/Gdoc/enrichedToMarkdown.ts @@ -127,6 +127,17 @@ ${items} exportComponents ) ) + .with({ type: "narrative-chart" }, (b): string | undefined => + markdownComponent( + "NarrativeChart", + { + name: b.name, + caption: b.caption ? spansToMarkdown(b.caption) : undefined, + // Note: truncated + }, + exportComponents + ) + ) .with({ type: "code" }, (b): string | undefined => { return ( "```\n" + diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index bc27e3356ed..08e8e1fa288 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -48,6 +48,7 @@ import { RawBlockPeople, RawBlockPeopleRows, RawBlockPerson, + RawBlockNarrativeChart, RawBlockCode, } from "@ourworldindata/types" import { spanToHtmlString } from "./gdocUtils.js" @@ -123,6 +124,20 @@ export function enrichedBlockToRawBlock( }, }) ) + .with( + { type: "narrative-chart" }, + (b): RawBlockNarrativeChart => ({ + type: b.type, + value: { + name: b.name, + height: b.height, + row: b.row, + column: b.column, + position: b.position, + caption: b.caption ? spansToHtmlText(b.caption) : undefined, + }, + }) + ) .with( { type: "code" }, (b): RawBlockCode => ({ diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index 7e37a16bc27..74b2a0f8842 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -121,6 +121,16 @@ export const enrichedBlockExamples: Record< caption: boldLinkExampleText, parseErrors: [], }, + "narrative-chart": { + type: "narrative-chart", + name: "world-has-become-less-democratic", + height: "400", + row: "1", + column: "1", + position: "featured", + caption: boldLinkExampleText, + parseErrors: [], + }, code: { type: "code", text: [ diff --git a/db/model/Gdoc/extractGdocComponentInfo.ts b/db/model/Gdoc/extractGdocComponentInfo.ts index 66b4c3c608a..308f06953c6 100644 --- a/db/model/Gdoc/extractGdocComponentInfo.ts +++ b/db/model/Gdoc/extractGdocComponentInfo.ts @@ -327,6 +327,7 @@ export function enumerateGdocComponentsWithoutChildren( type: P.union( "chart-story", "chart", + "narrative-chart", "horizontal-rule", "html", "image", @@ -353,8 +354,8 @@ export function enumerateGdocComponentsWithoutChildren( "additional-charts", "simple-text", "donors", - "socials" - // "narrative-chart" should go here once it's done + "socials", + "narrative-chart" ), }, (c) => handleComponent(c, [], parentPath, path) diff --git a/db/model/Gdoc/gdocUtils.ts b/db/model/Gdoc/gdocUtils.ts index f08dfd85786..d53f3f41a5d 100644 --- a/db/model/Gdoc/gdocUtils.ts +++ b/db/model/Gdoc/gdocUtils.ts @@ -237,6 +237,7 @@ export function extractFilenamesFromBlock( "latest-data-insights", "list", "missing-data", + "narrative-chart", "numbered-list", "people", "people-rows", diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index c2c9b50803d..b1b344a4491 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -47,6 +47,7 @@ import { RawBlockPeople, RawBlockPeopleRows, RawBlockPerson, + RawBlockNarrativeChart, RawBlockCode, } from "@ourworldindata/types" import { isArray } from "@ourworldindata/utils" @@ -128,6 +129,21 @@ function* rawBlockChartToArchieMLString( yield "{}" } +function* rawBlockNarrativeChartToArchieMLString( + block: RawBlockNarrativeChart +): Generator { + yield "{.narrative-chart}" + if (typeof block.value !== "string") { + yield* propertyToArchieMLString("name", block.value) + yield* propertyToArchieMLString("height", block.value) + yield* propertyToArchieMLString("row", block.value) + yield* propertyToArchieMLString("column", block.value) + yield* propertyToArchieMLString("position", block.value) + yield* propertyToArchieMLString("caption", block.value) + } + yield "{}" +} + function* rawBlockCodeToArchieMLString( block: RawBlockCode ): Generator { @@ -840,6 +856,10 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( .with({ type: "all-charts" }, rawBlockAllChartsToArchieMLString) .with({ type: "aside" }, rawBlockAsideToArchieMLString) .with({ type: "chart" }, rawBlockChartToArchieMLString) + .with( + { type: "narrative-chart" }, + rawBlockNarrativeChartToArchieMLString + ) .with({ type: "code" }, rawBlockCodeToArchieMLString) .with({ type: "donors" }, rawBlockDonorListToArchieMLString) .with({ type: "scroller" }, rawBlockScrollerToArchieMLString) diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index a60bd1f4e79..209fcc59847 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -129,6 +129,8 @@ import { EnrichedBlockPerson, RawBlockPeopleRows, EnrichedBlockPeopleRows, + RawBlockNarrativeChart, + EnrichedBlockNarrativeChart, RawBlockCode, EnrichedBlockCode, } from "@ourworldindata/types" @@ -172,6 +174,7 @@ export function parseRawBlocksToEnrichedBlocks( .with({ type: "blockquote" }, parseBlockquote) .with({ type: "callout" }, parseCallout) .with({ type: "chart" }, parseChart) + .with({ type: "narrative-chart" }, parseNarrativeChart) .with({ type: "code" }, parseCode) .with({ type: "donors" }, parseDonorList) .with({ type: "scroller" }, parseScroller) @@ -496,6 +499,67 @@ const parseChart = (raw: RawBlockChart): EnrichedBlockChart => { } } +const parseNarrativeChart = ( + raw: RawBlockNarrativeChart +): EnrichedBlockNarrativeChart => { + const createError = ( + error: ParseError, + name: string, + caption: Span[] = [] + ): EnrichedBlockNarrativeChart => ({ + type: "narrative-chart", + name, + caption, + parseErrors: [error], + }) + + const val = raw.value + + if (typeof val === "string") { + return { + type: "narrative-chart", + name: val, + parseErrors: [], + } + } else { + if (!val.name) + return createError( + { + message: "name property is missing", + }, + "" + ) + + const warnings: ParseError[] = [] + + const height = val.height + const row = val.row + const column = val.column + // This property is currently unused, a holdover from @mathisonian's gdocs demo. + // We will decide soon™️ if we want to use it for something + let position: ChartPositionChoice | undefined = undefined + if (val.position) + if (val.position === "featured") position = val.position + else { + warnings.push({ + message: "position must be 'featured' or unset", + }) + } + const caption = val.caption ? htmlToSpans(val.caption) : [] + + return omitUndefinedValues({ + type: "narrative-chart", + name: val.name, + height, + row, + column, + position, + caption: caption.length > 0 ? caption : undefined, + parseErrors: [], + }) as EnrichedBlockNarrativeChart + } +} + const parseCode = (raw: RawBlockCode): EnrichedBlockCode => { return { type: "code", diff --git a/db/model/Link.ts b/db/model/Link.ts index 4468e6832dd..bb0d2941cf4 100644 --- a/db/model/Link.ts +++ b/db/model/Link.ts @@ -62,3 +62,23 @@ export function createLinkFromUrl({ sourceId: source.id, } satisfies DbInsertPostGdocLink } + +export function createLinkForChartView({ + name, + source, + componentType, +}: { + name: string + source: GdocBase + componentType: string +}): DbInsertPostGdocLink { + return { + target: name, + linkType: OwidGdocLinkType.ChartView, + queryString: "", + hash: "", + text: "", + componentType, + sourceId: source.id, + } satisfies DbInsertPostGdocLink +} diff --git a/devTools/svgTester/update-configs.sh b/devTools/svgTester/update-configs.sh index 6604fc59dc4..95298b748da 100755 --- a/devTools/svgTester/update-configs.sh +++ b/devTools/svgTester/update-configs.sh @@ -20,29 +20,49 @@ Make sure to run \`make refresh\` and \`make refresh.pageviews\` before running main() { echo "=> Resetting owid-grapher-svgs to origin/master" - cd $SVGS_REPO\ - && git fetch\ - && git checkout -f master\ - && git reset --hard origin/master\ - && git clean -fd\ + cd $SVGS_REPO \ + && git fetch \ + && git checkout -f master \ + && git reset --hard origin/master \ + && git clean -fdx \ && cd - - echo "=> Removing existing configs and reference svgs" - rm -rf $CONFIGS_DIR $REFERENCES_DIR $ALL_VIEWS_DIR - echo "=> Dumping new configs and data" + rm -rf $CONFIGS_DIR node itsJustJavascript/devTools/svgTester/dump-data.js -o $CONFIGS_DIR node itsJustJavascript/devTools/svgTester/dump-chart-ids.js -o $CHART_IDS_FILE - echo "=> Generating reference SVGs" - node itsJustJavascript/devTools/svgTester/export-graphs.js\ - -i $CONFIGS_DIR\ + echo "=> Committing new configs and chart ids" + cd $SVGS_REPO \ + && git add --all \ + && git commit -m "chore: update configs and chart ids" \ + && cd - + + echo "=> Generating reference SVGs (default views)" + rm -rf $REFERENCES_DIR + node itsJustJavascript/devTools/svgTester/export-graphs.js \ + -i $CONFIGS_DIR \ -o $REFERENCES_DIR - node itsJustJavascript/devTools/svgTester/export-graphs.js\ - -i $CONFIGS_DIR\ - -o $ALL_VIEWS_SVG_DIR\ - -f $CHART_IDS_FILE\ + + echo "=> Committing reference SVGs (default views)" + cd $SVGS_REPO \ + && git add --all \ + && git commit -m 'chore: update reference svgs (default views)' \ + && cd - + + echo "=> Generating reference SVGs (all views)" + rm -rf $ALL_VIEWS_DIR + node itsJustJavascript/devTools/svgTester/export-graphs.js \ + -i $CONFIGS_DIR \ + -o $ALL_VIEWS_SVG_DIR \ + -f $CHART_IDS_FILE \ --all-views + + echo "=> Committing reference SVGs (all views)" + cd $SVGS_REPO \ + && git add --all \ + && git commit -m 'chore: update reference svgs (all views)' \ + && cd - } # show help diff --git a/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx b/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx index 36357a0c806..206b71cb57a 100644 --- a/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx +++ b/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx @@ -43,11 +43,11 @@ export const getDefaultFailMessage = (manager: ChartManager): string => { export const getSeriesKey = ( series: LineChartSeries, - suffix?: string + index: number ): string => { return `${series.seriesName}-${series.color}-${ series.isProjection ? "projection" : "" - }${suffix ? "-" + suffix : ""}` + }-${index}` } export const autoDetectSeriesStrategy = ( diff --git a/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss b/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss index d3f3a464aec..cc0b23bd31a 100644 --- a/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss +++ b/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss @@ -26,10 +26,20 @@ $paddingX: 12px; // keep in sync with PADDING_X } } -.ActionButton { - $light-fill: $gray-10; - $hover-fill: $gray-20; - $active-fill: $blue-20; +div.ActionButton { + --light-fill: #{$gray-10}; + --hover-fill: #{$gray-20}; + --active-fill: #{$blue-20}; + --text-color: #{$dark-text}; + + &.ActionButton--exploreData { + --light-fill: #{$blue-20}; + --hover-fill: #{$blue-20}; + --active-fill: #{$blue-10}; + --text-color: #{$blue-90}; + + --hover-decoration: underline; + } height: 100%; border-radius: 4px; @@ -43,12 +53,12 @@ $paddingX: 12px; // keep in sync with PADDING_X height: 100%; width: 100%; cursor: pointer; - color: $dark-text; + color: var(--text-color); font-size: 13px; font-weight: 500; padding: 0 $paddingX; border-radius: inherit; - background-color: $light-fill; + background-color: var(--light-fill); position: relative; letter-spacing: 0.01em; @@ -62,13 +72,14 @@ $paddingX: 12px; // keep in sync with PADDING_X } &:hover { - background-color: $hover-fill; + background-color: var(--hover-fill); + text-decoration: var(--hover-decoration); } &:active, &.active { color: $active-text; - background-color: $active-fill; + background-color: var(--active-fill); } } diff --git a/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx b/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx index dadc693d304..13e0e2f1646 100644 --- a/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx +++ b/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx @@ -305,7 +305,7 @@ export class ActionButtons extends React.Component<{ {this.hasExploreTheDataButton && (
    • + manager?: GrapherManager instanceRef?: React.RefObject } @@ -506,6 +512,11 @@ export class Grapher isEmbeddedInAnOwidPage?: boolean = this.props.isEmbeddedInAnOwidPage isEmbeddedInADataPage?: boolean = this.props.isEmbeddedInADataPage + chartViewInfo?: Pick< + ChartViewInfo, + "parentChartSlug" | "queryParamsForParentChart" + > = undefined + selection = this.manager?.selection ?? new SelectionArray( @@ -3521,10 +3532,27 @@ export class Grapher return this.props.manager } + @computed get canonicalUrlIfIsChartView(): string | undefined { + if (!this.chartViewInfo) return undefined + + const { parentChartSlug, queryParamsForParentChart } = + this.chartViewInfo + + const combinedQueryParams = { + ...queryParamsForParentChart, + ...this.changedParams, + } + + return `${this.bakedGrapherURL}/${parentChartSlug}${queryParamsToStr( + combinedQueryParams + )}` + } + // Get the full url representing the canonical location of this grapher state @computed get canonicalUrl(): string | undefined { return ( this.manager?.canonicalUrl ?? + this.canonicalUrlIfIsChartView ?? (this.baseUrl ? this.baseUrl + this.queryStr : undefined) ) } diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index 1601f36f65e..dc7143bc79e 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -5,6 +5,9 @@ import type { GrapherProgrammaticInterface } from "./Grapher" export const GRAPHER_EMBEDDED_FIGURE_ATTR = "data-grapher-src" export const GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR = "data-grapher-config" +export const GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR = + "data-grapher-chart-view-config" + export const GRAPHER_PAGE_BODY_CLASS = "StandaloneGrapherOrExplorerPage" export const GRAPHER_IS_IN_IFRAME_CLASS = "IsInIframe" export const GRAPHER_TIMELINE_CLASS = "timeline-component" diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index 8d52675e86c..13ca26a69cf 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -10,6 +10,7 @@ export { ChartDimension } from "./chart/ChartDimension" export { GRAPHER_EMBEDDED_FIGURE_ATTR, GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR, + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, GRAPHER_PAGE_BODY_CLASS, GRAPHER_IS_IN_IFRAME_CLASS, DEFAULT_GRAPHER_WIDTH, diff --git a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx index dd54175a663..3369bbbfa26 100644 --- a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx +++ b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx @@ -279,8 +279,8 @@ class Lines extends React.Component { private renderLines(): React.ReactElement { return ( <> - {this.props.series.map((series) => ( - + {this.props.series.map((series, index) => ( + {this.renderLine(series)} {this.renderLineMarkers(series)} @@ -556,7 +556,7 @@ export class LineChart y2={verticalAxis.range[1]} stroke="rgba(180,180,180,.4)" /> - {this.renderSeries.map((series) => { + {this.renderSeries.map((series, index) => { const value = series.points.find( (point) => point.x === activeX ) @@ -574,7 +574,7 @@ export class LineChart return ( !!series.isProjection)) - seriesToShow = seriesToShow.filter((series) => series.isProjection) + // If there are any projections, ignore non-projection legends (bit of a hack) + let series = this.series + if (series.some((series) => !!series.isProjection)) + series = series.filter((series) => series.isProjection) + + // Deduplicate series by seriesName to avoid showing the same label multiple times + const deduplicatedSeries: LineChartSeries[] = [] + const seriesGroupedByName = groupBy(series, "seriesName") + for (const duplicates of Object.values(seriesGroupedByName)) { + // keep only the label for the series with the most recent data + // (series are sorted by time, so we can just take the last one) + deduplicatedSeries.push(last(duplicates)!) + } - return seriesToShow.map((series) => { + return deduplicatedSeries.map((series) => { const { seriesName, color } = series const lastValue = last(series.points)!.y return { diff --git a/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx b/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx index ed14c1c17d2..dbcc3d0127d 100644 --- a/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx +++ b/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx @@ -39,6 +39,7 @@ import { MARKER_MARGIN, NON_FOCUSED_TEXT_COLOR, } from "./LineLegendConstants.js" +import { getSeriesKey } from "./LineLegendHelpers" export interface LineLabelSeries extends ChartSeries { label: string @@ -150,7 +151,7 @@ class LineLabels extends React.Component<{ @computed private get textLabels(): React.ReactElement { return ( - {this.markers.map(({ series, labelText }) => { + {this.markers.map(({ series, labelText }, index) => { const textColor = !series.focus?.background || series.hover?.active ? darkenColorForText(series.color) @@ -164,7 +165,7 @@ class LineLabels extends React.Component<{ return series.textWrap instanceof TextWrap ? ( @@ -197,12 +198,12 @@ class LineLabels extends React.Component<{ if (!markersWithAnnotations) return return ( - {markersWithAnnotations.map(({ series, labelText }) => { + {markersWithAnnotations.map(({ series, labelText }, index) => { if (!series.annotationTextWrap) return return ( @@ -232,7 +233,7 @@ class LineLabels extends React.Component<{ if (!this.props.needsConnectorLines) return return ( - {this.markers.map(({ series, connectorLine }) => { + {this.markers.map(({ series, connectorLine }, index) => { const { x1, x2 } = connectorLine const { level, @@ -253,7 +254,7 @@ class LineLabels extends React.Component<{ return ( - {this.props.series.map((series) => { + {this.props.series.map((series, index) => { const x = this.anchor === "start" ? series.origBounds.x : series.origBounds.x - series.bounds.width return ( this.props.onMouseOver?.(series)} onMouseLeave={() => this.props.onMouseLeave?.(series) diff --git a/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts b/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts index 17310a4d4e6..286c5cf7f36 100644 --- a/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts +++ b/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts @@ -245,3 +245,7 @@ export function computeCandidateScores( return scoreMap } + +export function getSeriesKey(series: PlacedSeries, index: number): string { + return `${series.seriesName}-${index}` +} diff --git a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx index e8c9930dab4..b89031653bc 100644 --- a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx +++ b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx @@ -875,9 +875,7 @@ export const DownloadModalDataTab = (props: DownloadModalProps) => { onClick={() => onDownloadClick(CsvDownloadType.Full)} tracking={ "chart_download_full_data--" + - serverSideDownloadAvailable - ? "server" - : "client" + (serverSideDownloadAvailable ? "server" : "client") } /> { } tracking={ "chart_download_filtered_data--" + - serverSideDownloadAvailable - ? "server" - : "client" + (serverSideDownloadAvailable ? "server" : "client") } />
      diff --git a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts index cbb50d10c51..06c200041d1 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts @@ -86,6 +86,31 @@ export type EnrichedBlockChart = { tabs?: ChartTabKeyword[] } & EnrichedBlockWithParseErrors +export type RawBlockNarrativeChartValue = { + name?: string + height?: string + row?: string + column?: string + // TODO: position is used as a classname apparently? Should be renamed or split + position?: string + caption?: string +} + +export type RawBlockNarrativeChart = { + type: "narrative-chart" + value: RawBlockNarrativeChartValue | string +} + +export type EnrichedBlockNarrativeChart = { + type: "narrative-chart" + name: string + height?: string + row?: string + column?: string + position?: ChartPositionChoice + caption?: Span[] +} & EnrichedBlockWithParseErrors + export type RawBlockCode = { type: "code" value: RawBlockText[] @@ -950,6 +975,7 @@ export type OwidRawGdocBlock = | RawBlockAside | RawBlockCallout | RawBlockChart + | RawBlockNarrativeChart | RawBlockCode | RawBlockDonorList | RawBlockScroller @@ -1001,6 +1027,7 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockAside | EnrichedBlockCallout | EnrichedBlockChart + | EnrichedBlockNarrativeChart | EnrichedBlockCode | EnrichedBlockDonorList | EnrichedBlockScroller diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index a6f35022ea3..318c5d1e5b2 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -13,6 +13,7 @@ import { } from "./ArchieMlComponents.js" import { MinimalTag } from "../dbTypes/Tags.js" import { DbEnrichedLatestWork } from "../domainTypes/Author.js" +import { QueryParams } from "../domainTypes/Various.js" export enum OwidGdocPublicationContext { unlisted = "unlisted", @@ -53,6 +54,15 @@ export interface LinkedChart { indicatorId?: number // in case of a datapage } +// An object containing metadata needed for embedded narrative charts +export interface ChartViewInfo { + name: string + title: string + chartConfigId: string + parentChartSlug: string + queryParamsForParentChart: QueryParams +} + /** * A linked indicator is derived from a linked grapher's config (see: getVariableOfDatapageIfApplicable) * e.g. https://ourworldindata.org/grapher/tomato-production -> config for grapher with { slug: "tomato-production" } -> indicator metadata @@ -271,6 +281,7 @@ export enum OwidGdocLinkType { Url = "url", Grapher = "grapher", Explorer = "explorer", + ChartView = "chart-view", } export interface OwidGdocLinkJSON { diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index f5fd8971661..a8a3f80db50 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -287,6 +287,8 @@ export { SocialLinkType, type RawSocialLink, type EnrichedSocialLink, + type RawBlockNarrativeChart, + type EnrichedBlockNarrativeChart, } from "./gdocTypes/ArchieMlComponents.js" export { ChartConfigType, @@ -330,6 +332,7 @@ export { type OwidGdocContent, type OwidGdocIndexItem, extractGdocIndexItem, + type ChartViewInfo, } from "./gdocTypes/Gdoc.js" export { diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index cd2978a218b..547e9404216 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -1711,6 +1711,7 @@ export function traverseEnrichedBlock( type: P.union( "chart-story", "chart", + "narrative-chart", "code", "donors", "horizontal-rule", diff --git a/packages/@ourworldindata/utils/src/metadataHelpers.ts b/packages/@ourworldindata/utils/src/metadataHelpers.ts index 6a99a3d3d2b..9fcf200dd47 100644 --- a/packages/@ourworldindata/utils/src/metadataHelpers.ts +++ b/packages/@ourworldindata/utils/src/metadataHelpers.ts @@ -73,16 +73,11 @@ export const getETLPathComponents = (path: string): ETLPathComponents => { export const formatAuthors = ({ authors, - requireMax, forBibtex, }: { authors: string[] - requireMax?: boolean forBibtex?: boolean }): string => { - if (requireMax && !authors.includes("Max Roser")) - authors = [...authors, "Max Roser"] - let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ") if (authorsText.length === 0) authorsText = authors[0] else authorsText += ` and ${last(authors)}` diff --git a/public/owid-logo.svg b/public/owid-logo.svg index 38643f2e383..5c7ea553986 100644 --- a/public/owid-logo.svg +++ b/public/owid-logo.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/site/Byline.tsx b/site/Byline.tsx index 0042eaff219..738f35769f5 100644 --- a/site/Byline.tsx +++ b/site/Byline.tsx @@ -2,11 +2,9 @@ import { formatAuthors } from "./clientFormatting.js" export const Byline = ({ authors, - withMax, override, }: { authors: string[] - withMax: boolean override?: string }) => { return ( @@ -20,7 +18,6 @@ export const Byline = ({ ) : ( {`by ${formatAuthors({ authors, - requireMax: withMax, })}`} )}
    • diff --git a/site/CitationMeta.tsx b/site/CitationMeta.tsx index c312f4db6ee..6e5c9a46002 100644 --- a/site/CitationMeta.tsx +++ b/site/CitationMeta.tsx @@ -8,12 +8,7 @@ export const CitationMeta = (props: { date: Date canonicalUrl: string }) => { - const { title, date, canonicalUrl } = props - let { authors } = props - - if (authors.indexOf("Max Roser") === -1) - authors = authors.concat(["Max Roser"]) - + const { authors, title, date, canonicalUrl } = props return ( diff --git a/site/DataCatalog/DataCatalog.tsx b/site/DataCatalog/DataCatalog.tsx index e68f8d47a27..a2b5738a1c4 100644 --- a/site/DataCatalog/DataCatalog.tsx +++ b/site/DataCatalog/DataCatalog.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useReducer, useRef, useState } from "react" import * as React from "react" import cx from "classnames" import { + commafyNumber, countriesByName, Country, Region, @@ -419,7 +420,7 @@ const DataCatalogRibbon = ({

      {result.title}

      - {result.nbHits}{" "} + {commafyNumber(result.nbHits)}{" "} {result.nbHits === 1 ? "chart" : "charts"} @@ -455,7 +456,7 @@ const DataCatalogRibbon = ({ > {result.nbHits === 1 ? `See 1 chart` - : `See ${result.nbHits} charts`} + : `See ${commafyNumber(result.nbHits)} charts`}
      @@ -562,7 +563,8 @@ const DataCatalogResults = ({
      {nbHits && (

      - {nbHits} {nbHits === 1 ? "indicator" : "indicators"} + {commafyNumber(nbHits)}{" "} + {nbHits === 1 ? "indicator" : "indicators"}

      )}
        @@ -673,7 +675,7 @@ const TopicsRefinementList = ({ onClick={() => addTopic(facetName)} > - {facetName} ({count}) + {facetName} ({commafyNumber(count)}) diff --git a/site/DataCatalog/DataCatalogPage.tsx b/site/DataCatalog/DataCatalogPage.tsx index 20a4b53cc50..c227886d740 100644 --- a/site/DataCatalog/DataCatalogPage.tsx +++ b/site/DataCatalog/DataCatalogPage.tsx @@ -20,7 +20,7 @@ export const DataCatalogPage = (props: { return ( )} diff --git a/site/clientFormatting.tsx b/site/clientFormatting.tsx index d6fb816761d..71596ab8e66 100644 --- a/site/clientFormatting.tsx +++ b/site/clientFormatting.tsx @@ -2,16 +2,11 @@ import { last } from "@ourworldindata/utils" export const formatAuthors = ({ authors, - requireMax, forBibtex, }: { authors: string[] - requireMax?: boolean forBibtex?: boolean }) => { - if (requireMax && !authors.includes("Max Roser")) - authors = [...authors, "Max Roser"] - let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ") if (authorsText.length === 0) authorsText = authors[0] else authorsText += ` and ${last(authors)}` diff --git a/site/formatting.test.ts b/site/formatting.test.ts index 61848ac98b9..ae7e7e50b90 100644 --- a/site/formatting.test.ts +++ b/site/formatting.test.ts @@ -162,17 +162,17 @@ describe(formatAuthors, () => { "Author 1, Author 2 and Author 3" ) - expect(formatAuthors({ authors, requireMax: true })).toEqual( - "Author 1, Author 2, Author 3 and Max Roser" - ) - expect(formatAuthors({ authors: ["Author 1"] })).toEqual("Author 1") expect(formatAuthors({ authors: ["Author 1", "Author 2"] })).toEqual( "Author 1 and Author 2" ) - expect( - formatAuthors({ authors, requireMax: true, forBibtex: true }) - ).toEqual("Author 1 and Author 2 and Author 3 and Max Roser") + expect(formatAuthors({ authors, forBibtex: true })).toEqual( + "Author 1 and Author 2 and Author 3" + ) + + expect(formatAuthors({ authors, forBibtex: false })).toEqual( + "Author 1, Author 2 and Author 3" + ) }) }) diff --git a/site/formatting.tsx b/site/formatting.tsx index 8123e7ca8bf..7c10b9856de 100644 --- a/site/formatting.tsx +++ b/site/formatting.tsx @@ -336,11 +336,7 @@ const addPostHeader = (cheerioEl: CheerioStatic, post: FormattedPost) => { ReactDOMServer.renderToStaticMarkup(
        {post.excerpt &&
        {post.excerpt}
        } - +
        diff --git a/site/gdocs/AttachmentsContext.tsx b/site/gdocs/AttachmentsContext.tsx index e5b5747889c..ec1b767fe12 100644 --- a/site/gdocs/AttachmentsContext.tsx +++ b/site/gdocs/AttachmentsContext.tsx @@ -9,6 +9,7 @@ import { LatestDataInsight, OwidGdocHomepageMetadata, DbEnrichedLatestWork, + ChartViewInfo, } from "@ourworldindata/types" export type Attachments = { @@ -22,6 +23,7 @@ export type Attachments = { latestDataInsights?: LatestDataInsight[] homepageMetadata?: OwidGdocHomepageMetadata latestWorkLinks?: DbEnrichedLatestWork[] + linkedChartViews?: Record } export const AttachmentsContext = createContext({ @@ -34,4 +36,5 @@ export const AttachmentsContext = createContext({ latestDataInsights: [], homepageMetadata: {}, latestWorkLinks: [], + linkedChartViews: {}, }) diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index da82c3a30f7..8c3162a6187 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -93,6 +93,7 @@ export function OwidGdoc({ latestDataInsights: get(props, "latestDataInsights", []), homepageMetadata: get(props, "homepageMetadata", {}), latestWorkLinks: get(props, "latestWorkLinks", []), + linkedChartViews: get(props, "linkedChartViews", {}), }} > diff --git a/site/gdocs/OwidGdocPage.tsx b/site/gdocs/OwidGdocPage.tsx index 9f4525b363a..ed80dd3699d 100644 --- a/site/gdocs/OwidGdocPage.tsx +++ b/site/gdocs/OwidGdocPage.tsx @@ -84,7 +84,7 @@ export default function OwidGdocPage({ debug?: boolean isPreviewing?: boolean }) { - const { content, createdAt, updatedAt } = gdoc + const { content, createdAt, publishedAt } = gdoc const pageDesc = getPageDesc(gdoc) const featuredImageFilename = getFeaturedImageFilename(gdoc) @@ -125,7 +125,7 @@ export default function OwidGdocPage({ )} diff --git a/site/gdocs/components/ArticleBlock.tsx b/site/gdocs/components/ArticleBlock.tsx index 07539cd649a..dc705f22b29 100644 --- a/site/gdocs/components/ArticleBlock.tsx +++ b/site/gdocs/components/ArticleBlock.tsx @@ -44,6 +44,8 @@ import { HomepageSearch } from "./HomepageSearch.js" import LatestDataInsightsBlock from "./LatestDataInsightsBlock.js" import { Socials } from "./Socials.js" import Person from "./Person.js" + +import NarrativeChart from "./NarrativeChart.js" import { Container, getLayout } from "./layout.js" export default function ArticleBlock({ @@ -106,6 +108,15 @@ export default function ArticleBlock({ /> ) }) + .with({ type: "narrative-chart" }, (block) => { + return ( + + ) + }) .with({ type: "code" }, (block) => ( (null) + useEmbedChart(0, refChartContainer) + + const viewMetadata = useLinkedChartView(d.name) + + const { isPreviewing } = useContext(DocumentContext) + + if (!viewMetadata) { + if (isPreviewing) { + return ( + + ) + } else return null // If not previewing, just don't render anything + } + + const metadataStringified = JSON.stringify(viewMetadata) + + const resolvedUrl = `${BAKED_GRAPHER_URL}/${ + viewMetadata.parentChartSlug + }${queryParamsToStr(viewMetadata.queryParamsForParentChart)}` + + return ( +
        +
        + + {viewMetadata.title} + + +
        + {d.caption ? ( +
        + +
        + ) : null} +
        + ) +} diff --git a/site/gdocs/components/Person.scss b/site/gdocs/components/Person.scss index d6db0c63feb..94dee7d4c74 100644 --- a/site/gdocs/components/Person.scss +++ b/site/gdocs/components/Person.scss @@ -45,6 +45,10 @@ a { color: inherit; + + &:hover { + text-decoration: underline; + } } } diff --git a/site/gdocs/components/Person.tsx b/site/gdocs/components/Person.tsx index 915e461e1de..b3816d17c08 100644 --- a/site/gdocs/components/Person.tsx +++ b/site/gdocs/components/Person.tsx @@ -31,16 +31,20 @@ export default function Person({ person }: { person: EnrichedBlockPerson }) {
        ) + const image = person.image ? ( + + ) : null + return (
        {person.image && (
        - + {url ? {image} : image} {isSmallScreen && header}
        )} diff --git a/site/gdocs/pages/GdocPost.tsx b/site/gdocs/pages/GdocPost.tsx index 48ddb81c054..64468b80906 100644 --- a/site/gdocs/pages/GdocPost.tsx +++ b/site/gdocs/pages/GdocPost.tsx @@ -179,6 +179,8 @@ export function GdocPost({ src={`${BAKED_BASE_URL}/owid-logo.svg`} className="img-raw" alt="Our World in Data logo" + width={104} + height={57} />

        Reuse this work freely

        diff --git a/site/gdocs/utils.ts b/site/gdocs/utils.ts index e5501b02994..18675be683c 100644 --- a/site/gdocs/utils.ts +++ b/site/gdocs/utils.ts @@ -148,6 +148,11 @@ export function useDonors(): string[] | undefined { return donors } +export const useLinkedChartView = (name: string) => { + const { linkedChartViews } = useContext(AttachmentsContext) + return linkedChartViews?.[name] +} + export function getShortPageCitation( authors: string[], title: string, diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index 45a41a03948..5d2bcec0884 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -10,6 +10,7 @@ import { migrateSelectedEntityNamesParam, SelectionArray, migrateGrapherConfigToLatestVersion, + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, } from "@ourworldindata/grapher" import { fetchText, @@ -21,6 +22,7 @@ import { MultiDimDataPageConfig, extractMultiDimChoicesFromQueryStr, fetchWithRetry, + ChartViewInfo, } from "@ourworldindata/utils" import { action } from "mobx" import ReactDOM from "react-dom" @@ -41,6 +43,9 @@ import { } from "../../settings/clientSettings.js" import Bugsnag from "@bugsnag/js" import { embedDynamicCollectionGrapher } from "../collections/DynamicCollection.js" +import { match } from "ts-pattern" + +type EmbedType = "grapher" | "explorer" | "multiDim" | "chartView" const figuresFromDOM = ( container: HTMLElement | Document = document, @@ -109,10 +114,16 @@ class MultiEmbedder { * Use this when you programmatically create/replace charts. */ observeFigures(container: HTMLElement | Document = document) { - const figures = figuresFromDOM( - container, - GRAPHER_EMBEDDED_FIGURE_ATTR - ).concat(figuresFromDOM(container, EXPLORER_EMBEDDED_FIGURE_SELECTOR)) + const figures = figuresFromDOM(container, GRAPHER_EMBEDDED_FIGURE_ATTR) + .concat( + figuresFromDOM(container, EXPLORER_EMBEDDED_FIGURE_SELECTOR) + ) + .concat( + figuresFromDOM( + container, + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + ) figures.forEach((figure) => { this.figuresObserver?.observe(figure) @@ -127,33 +138,41 @@ class MultiEmbedder { }) } - @action.bound - async renderInteractiveFigure(figure: Element) { - const isExplorer = figure.hasAttribute( + async renderExplorerIntoFigure(figure: Element) { + const explorerUrl = figure.getAttribute( EXPLORER_EMBEDDED_FIGURE_SELECTOR ) - const isMultiDim = figure.hasAttribute("data-is-multi-dim") - const dataSrc = figure.getAttribute( - isExplorer - ? EXPLORER_EMBEDDED_FIGURE_SELECTOR - : GRAPHER_EMBEDDED_FIGURE_ATTR - ) + if (!explorerUrl) return - if (!dataSrc) return + const { fullUrl, queryStr } = Url.fromURL(explorerUrl) - const hasPreview = isExplorer ? false : !!figure.querySelector("img") - if (!shouldProgressiveEmbed() && hasPreview) return - - // Stop observing visibility as soon as possible, that is not before - // shouldProgressiveEmbed gets a chance to reevaluate a possible change - // in screen size on mobile (i.e. after a rotation). Stopping before - // shouldProgressiveEmbed would prevent rendering interactive charts - // when going from portrait to landscape mode (without page reload). - this.figuresObserver?.unobserve(figure) + const html = await fetchText(fullUrl) + const props: ExplorerProps = await buildExplorerProps( + html, + queryStr, + this.selection + ) + if (props.selection) + this.graphersAndExplorersToUpdate.add(props.selection) + ReactDOM.render(, figure) + } - const { fullUrl, queryStr, queryParams } = Url.fromURL(dataSrc) + private async _renderGrapherComponentIntoFigure( + figure: Element, + { + configUrl, + embedUrl, + additionalConfig, + }: { + configUrl: string + embedUrl?: Url + additionalConfig?: Partial + } + ) { + const { queryStr, queryParams } = embedUrl ?? {} + figure.classList.remove(GRAPHER_PREVIEW_CLASS) const common: GrapherProgrammaticInterface = { isEmbeddedInAnOwidPage: true, queryStr, @@ -162,95 +181,150 @@ class MultiEmbedder { dataApiUrl: DATA_API_URL, } - if (isExplorer) { - const html = await fetchText(fullUrl) - const props: ExplorerProps = await buildExplorerProps( - html, - queryStr, - this.selection - ) - if (props.selection) - this.graphersAndExplorersToUpdate.add(props.selection) - ReactDOM.render(, figure) - } else { - figure.classList.remove(GRAPHER_PREVIEW_CLASS) - const url = new URL(fullUrl) - const slug = url.pathname.split("/").pop() - let configUrl - if (isMultiDim) { - const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json` - const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then( - (res) => res.json() - ) - const mdimConfig = - MultiDimDataPageConfig.fromObject(mdimJsonConfig) - const dimensions = extractMultiDimChoicesFromQueryStr( - url.search, - mdimConfig - ) - const view = mdimConfig.findViewByDimensions(dimensions) - if (!view) { - throw new Error( - `No view found for dimensions ${JSON.stringify( - dimensions - )}` - ) - } - configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${view.fullConfigId}.config.json` - } else { - configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${slug}.config.json` - } - const fetchedGrapherPageConfig = await fetchWithRetry( - configUrl - ).then((res) => res.json()) - const grapherPageConfig = migrateGrapherConfigToLatestVersion( - fetchedGrapherPageConfig - ) + const fetchedGrapherPageConfig = await fetchWithRetry(configUrl).then( + (res) => res.json() + ) + const grapherPageConfig = migrateGrapherConfigToLatestVersion( + fetchedGrapherPageConfig + ) - const figureConfigAttr = figure.getAttribute( - GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR - ) - const localConfig = figureConfigAttr - ? JSON.parse(figureConfigAttr) - : {} - - // make sure the tab of the active pane is visible - if (figureConfigAttr && !isEmpty(localConfig)) { - const activeTab = queryParams.tab || grapherPageConfig.tab - if (activeTab === GRAPHER_TAB_OPTIONS.chart) - localConfig.hideChartTabs = false - if (activeTab === GRAPHER_TAB_OPTIONS.map) - localConfig.hasMapTab = true - if (activeTab === GRAPHER_TAB_OPTIONS.table) - localConfig.hasTableTab = true + const figureConfigAttr = figure.getAttribute( + GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR + ) + const localConfig = figureConfigAttr ? JSON.parse(figureConfigAttr) : {} + + // make sure the tab of the active pane is visible + if (figureConfigAttr && !isEmpty(localConfig)) { + const activeTab = queryParams?.tab || grapherPageConfig.tab + if (activeTab === GRAPHER_TAB_OPTIONS.chart) + localConfig.hideChartTabs = false + if (activeTab === GRAPHER_TAB_OPTIONS.map) + localConfig.hasMapTab = true + if (activeTab === GRAPHER_TAB_OPTIONS.table) + localConfig.hasTableTab = true + } + + const config = merge( + {}, // merge mutates the first argument + grapherPageConfig, + common, + additionalConfig, + localConfig, + { + manager: { + selection: new SelectionArray( + this.selection.selectedEntityNames + ), + }, } + ) + if (config.manager?.selection) + this.graphersAndExplorersToUpdate.add(config.manager.selection) - const config = merge( - {}, // merge mutates the first argument - grapherPageConfig, - common, - localConfig, - { - manager: { - selection: new SelectionArray( - this.selection.selectedEntityNames - ), - }, - } - ) - if (config.manager?.selection) - this.graphersAndExplorersToUpdate.add(config.manager.selection) + const grapherRef = Grapher.renderGrapherIntoContainer(config, figure) - const grapherRef = Grapher.renderGrapherIntoContainer( - config, - figure - ) + // Special handling for shared collections + if (window.location.pathname.startsWith("/collection/custom")) { + embedDynamicCollectionGrapher(grapherRef, figure) + } + } + async renderGrapherIntoFigure(figure: Element) { + const embedUrlRaw = figure.getAttribute(GRAPHER_EMBEDDED_FIGURE_ATTR) + if (!embedUrlRaw) return + const embedUrl = Url.fromURL(embedUrlRaw) - // Special handling for shared collections - if (window.location.pathname.startsWith("/collection/custom")) { - embedDynamicCollectionGrapher(grapherRef, figure) - } + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${embedUrl.slug}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + embedUrl, + }) + } + async renderMultiDimIntoFigure(figure: Element) { + const embedUrlRaw = figure.getAttribute(GRAPHER_EMBEDDED_FIGURE_ATTR) + if (!embedUrlRaw) return + const embedUrl = Url.fromURL(embedUrlRaw) + + const { queryStr, slug } = embedUrl + + const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json` + const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then((res) => + res.json() + ) + const mdimConfig = MultiDimDataPageConfig.fromObject(mdimJsonConfig) + const dimensions = extractMultiDimChoicesFromQueryStr( + queryStr, + mdimConfig + ) + const view = mdimConfig.findViewByDimensions(dimensions) + if (!view) { + throw new Error( + `No view found for dimensions ${JSON.stringify(dimensions)}` + ) } + + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${view.fullConfigId}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + embedUrl, + }) + } + async renderChartViewIntoFigure(figure: Element) { + const viewConfigRaw = figure.getAttribute( + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + if (!viewConfigRaw) return + const viewConfig: ChartViewInfo = JSON.parse(viewConfigRaw) + if (!viewConfig) return + + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + additionalConfig: { + hideRelatedQuestion: true, + hideShareButton: true, + hideExploreTheDataButton: false, + chartViewInfo: viewConfig, + }, + }) + } + + @action.bound + async renderInteractiveFigure(figure: Element) { + const isExplorer = figure.hasAttribute( + EXPLORER_EMBEDDED_FIGURE_SELECTOR + ) + const isMultiDim = figure.hasAttribute("data-is-multi-dim") + const isChartView = figure.hasAttribute( + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + + const embedType: EmbedType = isExplorer + ? "explorer" + : isMultiDim + ? "multiDim" + : isChartView + ? "chartView" + : "grapher" + + const hasPreview = isExplorer ? false : !!figure.querySelector("img") + if (!shouldProgressiveEmbed() && hasPreview) return + + // Stop observing visibility as soon as possible, that is not before + // shouldProgressiveEmbed gets a chance to reevaluate a possible change + // in screen size on mobile (i.e. after a rotation). Stopping before + // shouldProgressiveEmbed would prevent rendering interactive charts + // when going from portrait to landscape mode (without page reload). + this.figuresObserver?.unobserve(figure) + + await match(embedType) + .with("explorer", () => this.renderExplorerIntoFigure(figure)) + .with("multiDim", () => this.renderMultiDimIntoFigure(figure)) + .with("chartView", () => this.renderChartViewIntoFigure(figure)) + .with("grapher", () => this.renderGrapherIntoFigure(figure)) + .exhaustive() } setUpGlobalEntitySelectorForEmbeds() {