From a8e242d83a2d3988011affa8889eba85604ca187 Mon Sep 17 00:00:00 2001 From: Daniel Bachler <daniel@danielbachler.de> Date: Thu, 26 Dec 2024 18:39:16 +0100 Subject: [PATCH 01/39] =?UTF-8?q?=F0=9F=90=9D=20set=20row=20key=20on=20ima?= =?UTF-8?q?ges=20table=20JJ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteClient/ImagesIndexPage.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/adminSiteClient/ImagesIndexPage.tsx b/adminSiteClient/ImagesIndexPage.tsx index ecd8a452f9..01f0a12799 100644 --- a/adminSiteClient/ImagesIndexPage.tsx +++ b/adminSiteClient/ImagesIndexPage.tsx @@ -656,7 +656,11 @@ export function ImageIndexPage() { /> <PostImageButton postImage={api.postImage} /> </Flex> - <Table columns={columns} dataSource={filteredImages} /> + <Table + columns={columns} + dataSource={filteredImages} + rowKey={(x) => x.id} + /> </main> </NotificationContext.Provider> </AdminLayout> From 9e6c9c82e10d96ff725bbab5302297d69f21844d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= <rakyi@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:58:16 +0100 Subject: [PATCH 02/39] Make links to author pages more discoverable (#4396) - Add underline on hover to person's name - Link also from the person's image --- site/gdocs/components/Person.scss | 4 ++++ site/gdocs/components/Person.tsx | 16 ++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/site/gdocs/components/Person.scss b/site/gdocs/components/Person.scss index d6db0c63fe..94dee7d4c7 100644 --- a/site/gdocs/components/Person.scss +++ b/site/gdocs/components/Person.scss @@ -45,6 +45,10 @@ a { color: inherit; + + &:hover { + text-decoration: underline; + } } } diff --git a/site/gdocs/components/Person.tsx b/site/gdocs/components/Person.tsx index 915e461e1d..b3816d17c0 100644 --- a/site/gdocs/components/Person.tsx +++ b/site/gdocs/components/Person.tsx @@ -31,16 +31,20 @@ export default function Person({ person }: { person: EnrichedBlockPerson }) { </div> ) + const image = person.image ? ( + <Image + className="person-image" + filename={person.image} + containerType="person" + shouldLightbox={false} + /> + ) : null + return ( <div className="person"> {person.image && ( <div className="person-image-container"> - <Image - className="person-image" - filename={person.image} - containerType="person" - shouldLightbox={false} - /> + {url ? <a href={url}>{image}</a> : image} {isSmallScreen && header} </div> )} From c74c45d984969b589382b667698be388e053e9ae Mon Sep 17 00:00:00 2001 From: Bobbie Macdonald <bnjmacdonald@gmail.com> Date: Wed, 8 Jan 2025 13:08:36 -0400 Subject: [PATCH 03/39] :bug: fix event_action Google analytics param for `chart_download_*` --- .../@ourworldindata/grapher/src/modal/DownloadModal.tsx | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx index e8c9930dab..b89031653b 100644 --- a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx +++ b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx @@ -875,9 +875,7 @@ export const DownloadModalDataTab = (props: DownloadModalProps) => { onClick={() => onDownloadClick(CsvDownloadType.Full)} tracking={ "chart_download_full_data--" + - serverSideDownloadAvailable - ? "server" - : "client" + (serverSideDownloadAvailable ? "server" : "client") } /> <DownloadButton @@ -889,9 +887,7 @@ export const DownloadModalDataTab = (props: DownloadModalProps) => { } tracking={ "chart_download_filtered_data--" + - serverSideDownloadAvailable - ? "server" - : "client" + (serverSideDownloadAvailable ? "server" : "client") } /> </div> From 9b19a737ca313a9d171234733fa2d75cab8f62fb Mon Sep 17 00:00:00 2001 From: Ike Saunders <ikesau@protonmail.com> Date: Tue, 7 Jan 2025 16:07:54 -0500 Subject: [PATCH 04/39] =?UTF-8?q?=E2=9C=A8=20improve=20images=20admin=20ui?= =?UTF-8?q?=20on=20small=20screens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteClient/ImagesIndexPage.tsx | 14 ++++++++------ adminSiteClient/admin.scss | 8 ++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/adminSiteClient/ImagesIndexPage.tsx b/adminSiteClient/ImagesIndexPage.tsx index e18b000492..09a4400c86 100644 --- a/adminSiteClient/ImagesIndexPage.tsx +++ b/adminSiteClient/ImagesIndexPage.tsx @@ -281,12 +281,13 @@ function createColumns({ title: "Filename", dataIndex: "filename", key: "filename", - width: 300, + width: 200, }, { title: "Alt text", dataIndex: "defaultAlt", key: "defaultAlt", + width: "auto", sorter: (a, b) => a.defaultAlt && b.defaultAlt ? a.defaultAlt.localeCompare(b.defaultAlt) @@ -309,7 +310,7 @@ function createColumns({ a.originalWidth && b.originalWidth ? a.originalWidth - b.originalWidth : 0, - width: 100, + width: 50, }, { title: "Height", @@ -319,13 +320,13 @@ function createColumns({ a.originalHeight && b.originalHeight ? a.originalHeight - b.originalHeight : 0, - width: 100, + width: 50, }, { title: "Last updated", dataIndex: "updatedAt", key: "updatedAt", - width: 150, + width: 50, defaultSortOrder: "descend", sorter: (a, b) => a.updatedAt && b.updatedAt ? a.updatedAt - b.updatedAt : 0, @@ -334,7 +335,7 @@ function createColumns({ { title: "Owner", key: "userId", - width: 200, + width: 100, filters: [ { text: "Unassigned", @@ -375,7 +376,7 @@ function createColumns({ { title: "Action", key: "action", - width: 100, + width: 50, render: (_, image) => { const isDeleteDisabled = !!(usage && usage[image.id]?.length) return ( @@ -658,6 +659,7 @@ export function ImageIndexPage() { <PostImageButton postImage={api.postImage} /> </Flex> <Table + size="small" columns={columns} dataSource={filteredImages} rowKey={(x) => x.id} diff --git a/adminSiteClient/admin.scss b/adminSiteClient/admin.scss index 966b05e0a5..7fc8df085e 100644 --- a/adminSiteClient/admin.scss +++ b/adminSiteClient/admin.scss @@ -1224,6 +1224,14 @@ main:not(.ChartEditorPage):not(.GdocsEditPage) { } .ImageIndexPage { + @media (max-width: 1300px) { + padding-left: 0 !important; + padding-right: 0 !important; + .ant-table-cell { + padding-left: 4px !important; + padding-right: 4px !important; + } + } .ImageIndexPage__delete-user-button { border-radius: 50%; margin-left: 8px; From ea5361d510ee15c16ef8a9570dd963938e49857e Mon Sep 17 00:00:00 2001 From: Sophia Mersmann <sophia.mersmann1@gmail.com> Date: Thu, 9 Jan 2025 10:45:27 +0100 Subject: [PATCH 05/39] =?UTF-8?q?=F0=9F=90=9B=20(line)=20unselected=20line?= =?UTF-8?q?s=20don't=20disappear=20(#4417)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grapher/src/chart/ChartUtils.tsx | 4 ++-- .../grapher/src/lineCharts/LineChart.tsx | 8 ++++---- .../grapher/src/lineLegend/LineLegend.tsx | 17 +++++++++-------- .../grapher/src/lineLegend/LineLegendHelpers.ts | 4 ++++ 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx b/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx index 36357a0c80..206b71cb57 100644 --- a/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx +++ b/packages/@ourworldindata/grapher/src/chart/ChartUtils.tsx @@ -43,11 +43,11 @@ export const getDefaultFailMessage = (manager: ChartManager): string => { export const getSeriesKey = ( series: LineChartSeries, - suffix?: string + index: number ): string => { return `${series.seriesName}-${series.color}-${ series.isProjection ? "projection" : "" - }${suffix ? "-" + suffix : ""}` + }-${index}` } export const autoDetectSeriesStrategy = ( diff --git a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx index dd54175a66..207959ea00 100644 --- a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx +++ b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx @@ -279,8 +279,8 @@ class Lines extends React.Component<LinesProps> { private renderLines(): React.ReactElement { return ( <> - {this.props.series.map((series) => ( - <React.Fragment key={getSeriesKey(series)}> + {this.props.series.map((series, index) => ( + <React.Fragment key={getSeriesKey(series, index)}> {this.renderLine(series)} {this.renderLineMarkers(series)} </React.Fragment> @@ -556,7 +556,7 @@ export class LineChart y2={verticalAxis.range[1]} stroke="rgba(180,180,180,.4)" /> - {this.renderSeries.map((series) => { + {this.renderSeries.map((series, index) => { const value = series.points.find( (point) => point.x === activeX ) @@ -574,7 +574,7 @@ export class LineChart return ( <circle - key={getSeriesKey(series)} + key={getSeriesKey(series, index)} cx={horizontalAxis.place(value.x)} cy={verticalAxis.place(value.y)} r={this.lineStrokeWidth / 2 + 3.5} diff --git a/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx b/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx index ed14c1c17d..dbcc3d0127 100644 --- a/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx +++ b/packages/@ourworldindata/grapher/src/lineLegend/LineLegend.tsx @@ -39,6 +39,7 @@ import { MARKER_MARGIN, NON_FOCUSED_TEXT_COLOR, } from "./LineLegendConstants.js" +import { getSeriesKey } from "./LineLegendHelpers" export interface LineLabelSeries extends ChartSeries { label: string @@ -150,7 +151,7 @@ class LineLabels extends React.Component<{ @computed private get textLabels(): React.ReactElement { return ( <g id={makeIdForHumanConsumption("text-labels")}> - {this.markers.map(({ series, labelText }) => { + {this.markers.map(({ series, labelText }, index) => { const textColor = !series.focus?.background || series.hover?.active ? darkenColorForText(series.color) @@ -164,7 +165,7 @@ class LineLabels extends React.Component<{ return series.textWrap instanceof TextWrap ? ( <Halo id={series.seriesName} - key={series.seriesName} + key={getSeriesKey(series, index)} show={this.showTextOutline} outlineColor={this.textOutlineColor} > @@ -197,12 +198,12 @@ class LineLabels extends React.Component<{ if (!markersWithAnnotations) return return ( <g id={makeIdForHumanConsumption("text-annotations")}> - {markersWithAnnotations.map(({ series, labelText }) => { + {markersWithAnnotations.map(({ series, labelText }, index) => { if (!series.annotationTextWrap) return return ( <Halo id={series.seriesName} - key={series.seriesName} + key={getSeriesKey(series, index)} show={this.showTextOutline} outlineColor={this.textOutlineColor} > @@ -232,7 +233,7 @@ class LineLabels extends React.Component<{ if (!this.props.needsConnectorLines) return return ( <g id={makeIdForHumanConsumption("connectors")}> - {this.markers.map(({ series, connectorLine }) => { + {this.markers.map(({ series, connectorLine }, index) => { const { x1, x2 } = connectorLine const { level, @@ -253,7 +254,7 @@ class LineLabels extends React.Component<{ return ( <path id={makeIdForHumanConsumption(series.seriesName)} - key={series.seriesName} + key={getSeriesKey(series, index)} d={d} stroke={lineColor} strokeWidth={0.5} @@ -268,14 +269,14 @@ class LineLabels extends React.Component<{ @computed private get interactions(): React.ReactElement | void { return ( <g> - {this.props.series.map((series) => { + {this.props.series.map((series, index) => { const x = this.anchor === "start" ? series.origBounds.x : series.origBounds.x - series.bounds.width return ( <g - key={series.seriesName} + key={getSeriesKey(series, index)} onMouseOver={() => this.props.onMouseOver?.(series)} onMouseLeave={() => this.props.onMouseLeave?.(series) diff --git a/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts b/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts index 17310a4d4e..286c5cf7f3 100644 --- a/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts +++ b/packages/@ourworldindata/grapher/src/lineLegend/LineLegendHelpers.ts @@ -245,3 +245,7 @@ export function computeCandidateScores( return scoreMap } + +export function getSeriesKey(series: PlacedSeries, index: number): string { + return `${series.seriesName}-${index}` +} From afcddde3c3e020465d5ff1669c812150bd1cb88b Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Wed, 11 Dec 2024 17:22:47 +0100 Subject: [PATCH 06/39] feat: pre-fetch chart views metadata in gdocs --- baker/SiteBaker.tsx | 10 ++++++- baker/siteRenderers.tsx | 1 + db/model/ChartView.ts | 29 +++++++++++++++++++ .../types/src/gdocTypes/Gdoc.ts | 10 +++++++ site/gdocs/AttachmentsContext.tsx | 3 ++ site/gdocs/OwidGdoc.tsx | 1 + 6 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 db/model/ChartView.ts diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index c422adcaa4..e6a7495b3f 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -109,6 +109,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" +import { getAllChartViewsMetadata } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -176,7 +177,7 @@ function getProgressBarTotal(bakeSteps: BakeStepConfig): number { bakeSteps.has("dataInsights") || bakeSteps.has("authors") ) { - total += 8 + total += 9 } return total } @@ -459,6 +460,12 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) + const chartViewMetadata = await getAllChartViewsMetadata(knex) + const chartViewMetadataByName = keyBy(chartViewMetadata, "name") + this.progressBar.tick({ + name: `✅ Prefetched ${chartViewMetadata.length} chart views`, + }) + const prefetchedAttachments = { donors, linkedAuthors: publishedAuthors, @@ -469,6 +476,7 @@ export class SiteBaker { graphers: publishedChartsBySlug, }, linkedIndicators: datapageIndicatorsById, + chartViewMetadata: chartViewMetadataByName, } this.progressBar.tick({ name: "✅ Prefetched attachments" }) this._prefetchedAttachmentsCache = prefetchedAttachments diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index e53a9e3e52..ae9d5a0633 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -441,6 +441,7 @@ ${dataInsights latestDataInsights: get(post, "latestDataInsights", []), homepageMetadata: get(post, "homepageMetadata", {}), latestWorkLinks: get(post, "latestWorkLinks", []), + chartViewMetadata: get(post, "chartViewMetadata", {}), }} > <AtomArticleBlocks blocks={post.content.body} /> diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts new file mode 100644 index 0000000000..6cdce8e469 --- /dev/null +++ b/db/model/ChartView.ts @@ -0,0 +1,29 @@ +import { ChartViewMetadata, JsonString } from "@ourworldindata/types" +import * as db from "../db.js" + +export const getAllChartViewsMetadata = async ( + knex: db.KnexReadonlyTransaction +): Promise<ChartViewMetadata[]> => { + type RawRow = Omit<ChartViewMetadata, "queryParamsForParentChart"> & { + queryParamsForParentChart: JsonString + } + const rows: RawRow[] = await db.knexRaw( + knex, + `-- sql +SELECT cv.name, + cc.full ->> "$.title" as title, + chartConfigId, + pcc.slug as parentChartSlug, + cv.queryParamsForParentChart +FROM chart_views cv +JOIN chart_configs cc on cc.id = cv.chartConfigId +JOIN charts pc on cv.parentChartId = pc.id +JOIN chart_configs pcc on pc.configId = pcc.id + ` + ) + + return rows.map((row) => ({ + ...row, + queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart), + })) +} diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index a6f35022ea..9731db1b70 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -13,6 +13,7 @@ import { } from "./ArchieMlComponents.js" import { MinimalTag } from "../dbTypes/Tags.js" import { DbEnrichedLatestWork } from "../domainTypes/Author.js" +import { QueryParams } from "../domainTypes/Various.js" export enum OwidGdocPublicationContext { unlisted = "unlisted", @@ -53,6 +54,15 @@ export interface LinkedChart { indicatorId?: number // in case of a datapage } +// An object containing metadata needed for embedded narrative charts +export interface ChartViewMetadata { + name: string + title: string + chartConfigId: string + parentChartSlug: string + queryParamsForParentChart: QueryParams +} + /** * A linked indicator is derived from a linked grapher's config (see: getVariableOfDatapageIfApplicable) * e.g. https://ourworldindata.org/grapher/tomato-production -> config for grapher with { slug: "tomato-production" } -> indicator metadata diff --git a/site/gdocs/AttachmentsContext.tsx b/site/gdocs/AttachmentsContext.tsx index e5b5747889..49fc9b30ab 100644 --- a/site/gdocs/AttachmentsContext.tsx +++ b/site/gdocs/AttachmentsContext.tsx @@ -9,6 +9,7 @@ import { LatestDataInsight, OwidGdocHomepageMetadata, DbEnrichedLatestWork, + ChartViewMetadata, } from "@ourworldindata/types" export type Attachments = { @@ -22,6 +23,7 @@ export type Attachments = { latestDataInsights?: LatestDataInsight[] homepageMetadata?: OwidGdocHomepageMetadata latestWorkLinks?: DbEnrichedLatestWork[] + chartViewMetadata?: Record<string, ChartViewMetadata> } export const AttachmentsContext = createContext<Attachments>({ @@ -34,4 +36,5 @@ export const AttachmentsContext = createContext<Attachments>({ latestDataInsights: [], homepageMetadata: {}, latestWorkLinks: [], + chartViewMetadata: {}, }) diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index da82c3a30f..d1502a766d 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -93,6 +93,7 @@ export function OwidGdoc({ latestDataInsights: get(props, "latestDataInsights", []), homepageMetadata: get(props, "homepageMetadata", {}), latestWorkLinks: get(props, "latestWorkLinks", []), + chartViewMetadata: get(props, "chartViewMetadata", {}), }} > <DocumentContext.Provider value={{ isPreviewing }}> From c1067dfa19d01638db36ae2f6738eced20cee3cf Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Wed, 11 Dec 2024 17:23:22 +0100 Subject: [PATCH 07/39] feat: NarrativeChart component --- db/model/Gdoc/GdocBase.ts | 4 ++ db/model/Gdoc/enrichedToMarkdown.ts | 11 +++ db/model/Gdoc/enrichedToRaw.ts | 15 ++++ db/model/Gdoc/exampleEnrichedBlocks.ts | 10 +++ db/model/Gdoc/extractGdocComponentInfo.ts | 4 +- db/model/Gdoc/gdocUtils.ts | 1 + db/model/Gdoc/rawToArchie.ts | 20 ++++++ db/model/Gdoc/rawToEnriched.ts | 64 +++++++++++++++++ .../types/src/gdocTypes/ArchieMlComponents.ts | 27 +++++++ packages/@ourworldindata/types/src/index.ts | 3 + packages/@ourworldindata/utils/src/Util.ts | 1 + site/gdocs/components/ArticleBlock.tsx | 10 +++ site/gdocs/components/NarrativeChart.tsx | 70 +++++++++++++++++++ 13 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 site/gdocs/components/NarrativeChart.tsx diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 754471c247..f8ba74610a 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -577,6 +577,10 @@ export class GdocBase implements OwidGdocBaseInterface { "key-indicator-collection", "list", "missing-data", + + // Open question: there's not a direct link to a chart here, but there is a chart and also a parent chart + "narrative-chart", + "numbered-list", "people", "people-rows", diff --git a/db/model/Gdoc/enrichedToMarkdown.ts b/db/model/Gdoc/enrichedToMarkdown.ts index 2556f74ef2..794700f30a 100644 --- a/db/model/Gdoc/enrichedToMarkdown.ts +++ b/db/model/Gdoc/enrichedToMarkdown.ts @@ -127,6 +127,17 @@ ${items} exportComponents ) ) + .with({ type: "narrative-chart" }, (b): string | undefined => + markdownComponent( + "NarrativeChart", + { + name: b.name, + caption: b.caption ? spansToMarkdown(b.caption) : undefined, + // Note: truncated + }, + exportComponents + ) + ) .with({ type: "code" }, (b): string | undefined => { return ( "```\n" + diff --git a/db/model/Gdoc/enrichedToRaw.ts b/db/model/Gdoc/enrichedToRaw.ts index bc27e3356e..08e8e1fa28 100644 --- a/db/model/Gdoc/enrichedToRaw.ts +++ b/db/model/Gdoc/enrichedToRaw.ts @@ -48,6 +48,7 @@ import { RawBlockPeople, RawBlockPeopleRows, RawBlockPerson, + RawBlockNarrativeChart, RawBlockCode, } from "@ourworldindata/types" import { spanToHtmlString } from "./gdocUtils.js" @@ -123,6 +124,20 @@ export function enrichedBlockToRawBlock( }, }) ) + .with( + { type: "narrative-chart" }, + (b): RawBlockNarrativeChart => ({ + type: b.type, + value: { + name: b.name, + height: b.height, + row: b.row, + column: b.column, + position: b.position, + caption: b.caption ? spansToHtmlText(b.caption) : undefined, + }, + }) + ) .with( { type: "code" }, (b): RawBlockCode => ({ diff --git a/db/model/Gdoc/exampleEnrichedBlocks.ts b/db/model/Gdoc/exampleEnrichedBlocks.ts index 7e37a16bc2..74b2a0f884 100644 --- a/db/model/Gdoc/exampleEnrichedBlocks.ts +++ b/db/model/Gdoc/exampleEnrichedBlocks.ts @@ -121,6 +121,16 @@ export const enrichedBlockExamples: Record< caption: boldLinkExampleText, parseErrors: [], }, + "narrative-chart": { + type: "narrative-chart", + name: "world-has-become-less-democratic", + height: "400", + row: "1", + column: "1", + position: "featured", + caption: boldLinkExampleText, + parseErrors: [], + }, code: { type: "code", text: [ diff --git a/db/model/Gdoc/extractGdocComponentInfo.ts b/db/model/Gdoc/extractGdocComponentInfo.ts index 66b4c3c608..dadac4f23e 100644 --- a/db/model/Gdoc/extractGdocComponentInfo.ts +++ b/db/model/Gdoc/extractGdocComponentInfo.ts @@ -353,8 +353,8 @@ export function enumerateGdocComponentsWithoutChildren( "additional-charts", "simple-text", "donors", - "socials" - // "narrative-chart" should go here once it's done + "socials", + "narrative-chart" ), }, (c) => handleComponent(c, [], parentPath, path) diff --git a/db/model/Gdoc/gdocUtils.ts b/db/model/Gdoc/gdocUtils.ts index f08dfd8578..d53f3f41a5 100644 --- a/db/model/Gdoc/gdocUtils.ts +++ b/db/model/Gdoc/gdocUtils.ts @@ -237,6 +237,7 @@ export function extractFilenamesFromBlock( "latest-data-insights", "list", "missing-data", + "narrative-chart", "numbered-list", "people", "people-rows", diff --git a/db/model/Gdoc/rawToArchie.ts b/db/model/Gdoc/rawToArchie.ts index c2c9b50803..b1b344a449 100644 --- a/db/model/Gdoc/rawToArchie.ts +++ b/db/model/Gdoc/rawToArchie.ts @@ -47,6 +47,7 @@ import { RawBlockPeople, RawBlockPeopleRows, RawBlockPerson, + RawBlockNarrativeChart, RawBlockCode, } from "@ourworldindata/types" import { isArray } from "@ourworldindata/utils" @@ -128,6 +129,21 @@ function* rawBlockChartToArchieMLString( yield "{}" } +function* rawBlockNarrativeChartToArchieMLString( + block: RawBlockNarrativeChart +): Generator<string, void, undefined> { + yield "{.narrative-chart}" + if (typeof block.value !== "string") { + yield* propertyToArchieMLString("name", block.value) + yield* propertyToArchieMLString("height", block.value) + yield* propertyToArchieMLString("row", block.value) + yield* propertyToArchieMLString("column", block.value) + yield* propertyToArchieMLString("position", block.value) + yield* propertyToArchieMLString("caption", block.value) + } + yield "{}" +} + function* rawBlockCodeToArchieMLString( block: RawBlockCode ): Generator<string, void, undefined> { @@ -840,6 +856,10 @@ export function* OwidRawGdocBlockToArchieMLStringGenerator( .with({ type: "all-charts" }, rawBlockAllChartsToArchieMLString) .with({ type: "aside" }, rawBlockAsideToArchieMLString) .with({ type: "chart" }, rawBlockChartToArchieMLString) + .with( + { type: "narrative-chart" }, + rawBlockNarrativeChartToArchieMLString + ) .with({ type: "code" }, rawBlockCodeToArchieMLString) .with({ type: "donors" }, rawBlockDonorListToArchieMLString) .with({ type: "scroller" }, rawBlockScrollerToArchieMLString) diff --git a/db/model/Gdoc/rawToEnriched.ts b/db/model/Gdoc/rawToEnriched.ts index a60bd1f4e7..209fcc5984 100644 --- a/db/model/Gdoc/rawToEnriched.ts +++ b/db/model/Gdoc/rawToEnriched.ts @@ -129,6 +129,8 @@ import { EnrichedBlockPerson, RawBlockPeopleRows, EnrichedBlockPeopleRows, + RawBlockNarrativeChart, + EnrichedBlockNarrativeChart, RawBlockCode, EnrichedBlockCode, } from "@ourworldindata/types" @@ -172,6 +174,7 @@ export function parseRawBlocksToEnrichedBlocks( .with({ type: "blockquote" }, parseBlockquote) .with({ type: "callout" }, parseCallout) .with({ type: "chart" }, parseChart) + .with({ type: "narrative-chart" }, parseNarrativeChart) .with({ type: "code" }, parseCode) .with({ type: "donors" }, parseDonorList) .with({ type: "scroller" }, parseScroller) @@ -496,6 +499,67 @@ const parseChart = (raw: RawBlockChart): EnrichedBlockChart => { } } +const parseNarrativeChart = ( + raw: RawBlockNarrativeChart +): EnrichedBlockNarrativeChart => { + const createError = ( + error: ParseError, + name: string, + caption: Span[] = [] + ): EnrichedBlockNarrativeChart => ({ + type: "narrative-chart", + name, + caption, + parseErrors: [error], + }) + + const val = raw.value + + if (typeof val === "string") { + return { + type: "narrative-chart", + name: val, + parseErrors: [], + } + } else { + if (!val.name) + return createError( + { + message: "name property is missing", + }, + "" + ) + + const warnings: ParseError[] = [] + + const height = val.height + const row = val.row + const column = val.column + // This property is currently unused, a holdover from @mathisonian's gdocs demo. + // We will decide soon™️ if we want to use it for something + let position: ChartPositionChoice | undefined = undefined + if (val.position) + if (val.position === "featured") position = val.position + else { + warnings.push({ + message: "position must be 'featured' or unset", + }) + } + const caption = val.caption ? htmlToSpans(val.caption) : [] + + return omitUndefinedValues({ + type: "narrative-chart", + name: val.name, + height, + row, + column, + position, + caption: caption.length > 0 ? caption : undefined, + parseErrors: [], + }) as EnrichedBlockNarrativeChart + } +} + const parseCode = (raw: RawBlockCode): EnrichedBlockCode => { return { type: "code", diff --git a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts index cbb50d10c5..06c200041d 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/ArchieMlComponents.ts @@ -86,6 +86,31 @@ export type EnrichedBlockChart = { tabs?: ChartTabKeyword[] } & EnrichedBlockWithParseErrors +export type RawBlockNarrativeChartValue = { + name?: string + height?: string + row?: string + column?: string + // TODO: position is used as a classname apparently? Should be renamed or split + position?: string + caption?: string +} + +export type RawBlockNarrativeChart = { + type: "narrative-chart" + value: RawBlockNarrativeChartValue | string +} + +export type EnrichedBlockNarrativeChart = { + type: "narrative-chart" + name: string + height?: string + row?: string + column?: string + position?: ChartPositionChoice + caption?: Span[] +} & EnrichedBlockWithParseErrors + export type RawBlockCode = { type: "code" value: RawBlockText[] @@ -950,6 +975,7 @@ export type OwidRawGdocBlock = | RawBlockAside | RawBlockCallout | RawBlockChart + | RawBlockNarrativeChart | RawBlockCode | RawBlockDonorList | RawBlockScroller @@ -1001,6 +1027,7 @@ export type OwidEnrichedGdocBlock = | EnrichedBlockAside | EnrichedBlockCallout | EnrichedBlockChart + | EnrichedBlockNarrativeChart | EnrichedBlockCode | EnrichedBlockDonorList | EnrichedBlockScroller diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index f5fd897166..3be6749e83 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -287,6 +287,8 @@ export { SocialLinkType, type RawSocialLink, type EnrichedSocialLink, + type RawBlockNarrativeChart, + type EnrichedBlockNarrativeChart, } from "./gdocTypes/ArchieMlComponents.js" export { ChartConfigType, @@ -330,6 +332,7 @@ export { type OwidGdocContent, type OwidGdocIndexItem, extractGdocIndexItem, + type ChartViewMetadata, } from "./gdocTypes/Gdoc.js" export { diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index cd2978a218..547e940421 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -1711,6 +1711,7 @@ export function traverseEnrichedBlock( type: P.union( "chart-story", "chart", + "narrative-chart", "code", "donors", "horizontal-rule", diff --git a/site/gdocs/components/ArticleBlock.tsx b/site/gdocs/components/ArticleBlock.tsx index 07539cd649..eebaaaad04 100644 --- a/site/gdocs/components/ArticleBlock.tsx +++ b/site/gdocs/components/ArticleBlock.tsx @@ -44,6 +44,7 @@ import { HomepageSearch } from "./HomepageSearch.js" import LatestDataInsightsBlock from "./LatestDataInsightsBlock.js" import { Socials } from "./Socials.js" import Person from "./Person.js" +import NarrativeChart from "./NarrativeChart.js" import { Container, getLayout } from "./layout.js" export default function ArticleBlock({ @@ -106,6 +107,15 @@ export default function ArticleBlock({ /> ) }) + .with({ type: "narrative-chart" }, (block) => { + return ( + <NarrativeChart + className={getLayout("chart", containerType)} + d={block} + fullWidthOnMobile={true} + /> + ) + }) .with({ type: "code" }, (block) => ( <CodeSnippet className={getLayout("code-snippet", containerType)} diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx new file mode 100644 index 0000000000..f1451d164f --- /dev/null +++ b/site/gdocs/components/NarrativeChart.tsx @@ -0,0 +1,70 @@ +import React, { useContext, useRef } from "react" +import { useEmbedChart } from "../../hooks.js" +import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" +import cx from "classnames" +import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" +import { AttachmentsContext } from "../../gdocs/AttachmentsContext.js" +import { BlockErrorFallback } from "./BlockErrorBoundary.js" +import SpanElements from "./SpanElements.js" + +export default function NarrativeChart({ + d, + className, + fullWidthOnMobile = false, +}: { + d: EnrichedBlockNarrativeChart + className?: string + fullWidthOnMobile?: boolean +}) { + const refChartContainer = useRef<HTMLDivElement>(null) + useEmbedChart(0, refChartContainer) + + const attachments = useContext(AttachmentsContext) + + const viewMetadata = attachments.chartViewMetadata?.[d.name] + + if (!viewMetadata) + return ( + <BlockErrorFallback + className={className} + error={{ + name: "Narrative view not found", + message: `Narrative view with name "${d.name}" couldn't be found.`, + }} + /> + ) + + const metadataStringified = JSON.stringify(viewMetadata) + + return ( + <div + className={cx(d.position, className, { + "full-width-on-mobile": fullWidthOnMobile, + })} + style={{ gridRow: d.row, gridColumn: d.column }} + ref={refChartContainer} + > + <figure + key={metadataStringified} + className={cx(GRAPHER_PREVIEW_CLASS, "chart")} + data-grapher-view-config={metadataStringified} + // data-grapher-src={isExplorer ? undefined : resolvedUrl} + style={{ + width: "100%", + border: "0px none", + height: d.height, + }} + > + {/* <a href={resolvedUrl} target="_blank" rel="noopener"> + <GrapherImage slug={resolvedSlug} alt={d.title} /> + <InteractionNotice /> + </a> */} + </figure> + {d.caption ? ( + <figcaption> + <SpanElements spans={d.caption} /> + </figcaption> + ) : null} + </div> + ) +} From 8f3e381ee3f6892ab7efe6945431afdcfb2c807c Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 12 Dec 2024 10:33:40 +0100 Subject: [PATCH 08/39] refactor: properly attach gdocs attachments --- baker/SiteBaker.tsx | 5 +++++ db/model/Gdoc/GdocBase.ts | 12 ++++++++++++ site/gdocs/components/NarrativeChart.tsx | 7 +++---- site/gdocs/utils.ts | 5 +++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index e6a7495b3f..a4ce0036a9 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -56,6 +56,7 @@ import { grabMetadataForGdocLinkedIndicator, TombstonePageData, gdocUrlRegex, + ChartViewMetadata, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -121,6 +122,7 @@ type PrefetchedAttachments = { explorers: Record<string, LinkedChart> } linkedIndicators: Record<number, LinkedIndicator> + chartViewMetadata: Record<string, ChartViewMetadata> } // These aren't all "wordpress" steps @@ -536,6 +538,8 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), + chartViewMetadata: + this._prefetchedAttachmentsCache.chartViewMetadata, // TODO: Filter } } return this._prefetchedAttachmentsCache @@ -637,6 +641,7 @@ export class SiteBaker { ...attachments.linkedCharts.explorers, } publishedGdoc.linkedIndicators = attachments.linkedIndicators + publishedGdoc.chartViewMetadata = attachments.chartViewMetadata // this is a no-op if the gdoc doesn't have an all-chart block if ("loadRelatedCharts" in publishedGdoc) { diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index f8ba74610a..baff9b0535 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -56,6 +56,7 @@ import { import { ARCHVED_THUMBNAIL_FILENAME, ChartConfigType, + ChartViewMetadata, DEFAULT_THUMBNAIL_FILENAME, GrapherInterface, LatestDataInsight, @@ -66,6 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" +import { getAllChartViewsMetadata } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -89,6 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface { linkedIndicators: Record<number, LinkedIndicator> = {} linkedDocuments: Record<string, OwidGdocMinimalPostInterface> = {} latestDataInsights: LatestDataInsight[] = [] + chartViewMetadata?: Record<string, ChartViewMetadata> = {} _omittableFields: string[] = [] constructor(id?: string) { @@ -714,6 +717,14 @@ export class GdocBase implements OwidGdocBaseInterface { } } + async loadChartViewMetadata( + knex: db.KnexReadonlyTransaction + ): Promise<void> { + // TODO: Filter down to only those that are used in the Gdoc + const result = await getAllChartViewsMetadata(knex) + this.chartViewMetadata = keyBy(result, "name") + } + async fetchAndEnrichGdoc(): Promise<void> { const docsClient = google.docs({ version: "v1", @@ -859,6 +870,7 @@ export class GdocBase implements OwidGdocBaseInterface { await this.loadImageMetadataFromDB(knex) await this.loadLinkedCharts(knex) await this.loadLinkedIndicators() // depends on linked charts + await this.loadChartViewMetadata(knex) await this._loadSubclassAttachments(knex) await this.validate(knex) } diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index f1451d164f..4ca43b37c8 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,6 +1,7 @@ -import React, { useContext, useRef } from "react" +import React, { useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" +import { useChartViewMetadata } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" import { AttachmentsContext } from "../../gdocs/AttachmentsContext.js" @@ -19,9 +20,7 @@ export default function NarrativeChart({ const refChartContainer = useRef<HTMLDivElement>(null) useEmbedChart(0, refChartContainer) - const attachments = useContext(AttachmentsContext) - - const viewMetadata = attachments.chartViewMetadata?.[d.name] + const viewMetadata = useChartViewMetadata(d.name) if (!viewMetadata) return ( diff --git a/site/gdocs/utils.ts b/site/gdocs/utils.ts index e5501b0299..ffd255f73c 100644 --- a/site/gdocs/utils.ts +++ b/site/gdocs/utils.ts @@ -148,6 +148,11 @@ export function useDonors(): string[] | undefined { return donors } +export const useChartViewMetadata = (name: string) => { + const { chartViewMetadata } = useContext(AttachmentsContext) + return chartViewMetadata?.[name] +} + export function getShortPageCitation( authors: string[], title: string, From 7b90855850de4e6a66745eace479fc42b6f06dda Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 12 Dec 2024 10:38:53 +0100 Subject: [PATCH 09/39] enhance: ability to filter `chartViewMetadata` --- baker/SiteBaker.tsx | 4 ++-- db/model/ChartView.ts | 17 +++++++++++------ db/model/Gdoc/GdocBase.ts | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index a4ce0036a9..a62a4b1f40 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -110,7 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" -import { getAllChartViewsMetadata } from "../db/model/ChartView.js" +import { getChartViewsMetadata } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -462,7 +462,7 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) - const chartViewMetadata = await getAllChartViewsMetadata(knex) + const chartViewMetadata = await getChartViewsMetadata(knex) const chartViewMetadataByName = keyBy(chartViewMetadata, "name") this.progressBar.tick({ name: `✅ Prefetched ${chartViewMetadata.length} chart views`, diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts index 6cdce8e469..2113aab0d8 100644 --- a/db/model/ChartView.ts +++ b/db/model/ChartView.ts @@ -1,15 +1,16 @@ import { ChartViewMetadata, JsonString } from "@ourworldindata/types" import * as db from "../db.js" -export const getAllChartViewsMetadata = async ( - knex: db.KnexReadonlyTransaction +export const getChartViewsMetadata = async ( + knex: db.KnexReadonlyTransaction, + names?: string[] ): Promise<ChartViewMetadata[]> => { type RawRow = Omit<ChartViewMetadata, "queryParamsForParentChart"> & { queryParamsForParentChart: JsonString } - const rows: RawRow[] = await db.knexRaw( - knex, - `-- sql + let rows: RawRow[] + + const query = `-- sql SELECT cv.name, cc.full ->> "$.title" as title, chartConfigId, @@ -20,7 +21,11 @@ JOIN chart_configs cc on cc.id = cv.chartConfigId JOIN charts pc on cv.parentChartId = pc.id JOIN chart_configs pcc on pc.configId = pcc.id ` - ) + + if (names) { + if (names.length === 0) return [] + rows = await db.knexRaw(knex, `${query} WHERE cv.name IN (?)`, [names]) + } else rows = await db.knexRaw(knex, query) return rows.map((row) => ({ ...row, diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index baff9b0535..90d02c2257 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -67,7 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" -import { getAllChartViewsMetadata } from "../ChartView.js" +import { getChartViewsMetadata } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -721,7 +721,7 @@ export class GdocBase implements OwidGdocBaseInterface { knex: db.KnexReadonlyTransaction ): Promise<void> { // TODO: Filter down to only those that are used in the Gdoc - const result = await getAllChartViewsMetadata(knex) + const result = await getChartViewsMetadata(knex) this.chartViewMetadata = keyBy(result, "name") } From 8e45ff47b1ea2c9c19fb1856cc6307fd24f49d11 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Mon, 16 Dec 2024 23:07:24 +0100 Subject: [PATCH 10/39] refactor: add narrative-chart to `extractGdocComponentInfo` --- db/model/Gdoc/extractGdocComponentInfo.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/db/model/Gdoc/extractGdocComponentInfo.ts b/db/model/Gdoc/extractGdocComponentInfo.ts index dadac4f23e..308f06953c 100644 --- a/db/model/Gdoc/extractGdocComponentInfo.ts +++ b/db/model/Gdoc/extractGdocComponentInfo.ts @@ -327,6 +327,7 @@ export function enumerateGdocComponentsWithoutChildren( type: P.union( "chart-story", "chart", + "narrative-chart", "horizontal-rule", "html", "image", From 7b61123f11ce850ef42223354529b74f2cbeb89e Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Tue, 17 Dec 2024 15:33:53 +0100 Subject: [PATCH 11/39] refactor: chartViewMetadata -> narrativeViewInfo --- baker/SiteBaker.tsx | 20 +++++++++---------- baker/siteRenderers.tsx | 2 +- db/model/ChartView.ts | 8 ++++---- db/model/Gdoc/GdocBase.ts | 14 ++++++------- .../types/src/gdocTypes/Gdoc.ts | 2 +- packages/@ourworldindata/types/src/index.ts | 2 +- site/gdocs/AttachmentsContext.tsx | 6 +++--- site/gdocs/OwidGdoc.tsx | 2 +- site/gdocs/components/NarrativeChart.tsx | 4 ++-- site/gdocs/utils.ts | 6 +++--- 10 files changed, 33 insertions(+), 33 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index a62a4b1f40..24ca2315cc 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -56,7 +56,7 @@ import { grabMetadataForGdocLinkedIndicator, TombstonePageData, gdocUrlRegex, - ChartViewMetadata, + NarrativeViewInfo, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -110,7 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" -import { getChartViewsMetadata } from "../db/model/ChartView.js" +import { getNarrativeViewsInfo } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -122,7 +122,7 @@ type PrefetchedAttachments = { explorers: Record<string, LinkedChart> } linkedIndicators: Record<number, LinkedIndicator> - chartViewMetadata: Record<string, ChartViewMetadata> + narrativeViewsInfo: Record<string, NarrativeViewInfo> } // These aren't all "wordpress" steps @@ -462,10 +462,10 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) - const chartViewMetadata = await getChartViewsMetadata(knex) - const chartViewMetadataByName = keyBy(chartViewMetadata, "name") + const narrativeViewsInfo = await getNarrativeViewsInfo(knex) + const narrativeViewsInfoByName = keyBy(narrativeViewsInfo, "name") this.progressBar.tick({ - name: `✅ Prefetched ${chartViewMetadata.length} chart views`, + name: `✅ Prefetched ${narrativeViewsInfo.length} chart views`, }) const prefetchedAttachments = { @@ -478,7 +478,7 @@ export class SiteBaker { graphers: publishedChartsBySlug, }, linkedIndicators: datapageIndicatorsById, - chartViewMetadata: chartViewMetadataByName, + narrativeViewsInfo: narrativeViewsInfoByName, } this.progressBar.tick({ name: "✅ Prefetched attachments" }) this._prefetchedAttachmentsCache = prefetchedAttachments @@ -538,8 +538,8 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), - chartViewMetadata: - this._prefetchedAttachmentsCache.chartViewMetadata, // TODO: Filter + narrativeViewsInfo: + this._prefetchedAttachmentsCache.narrativeViewsInfo, // TODO: Filter } } return this._prefetchedAttachmentsCache @@ -641,7 +641,7 @@ export class SiteBaker { ...attachments.linkedCharts.explorers, } publishedGdoc.linkedIndicators = attachments.linkedIndicators - publishedGdoc.chartViewMetadata = attachments.chartViewMetadata + publishedGdoc.narrativeViewsInfo = attachments.narrativeViewsInfo // this is a no-op if the gdoc doesn't have an all-chart block if ("loadRelatedCharts" in publishedGdoc) { diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index ae9d5a0633..ba9d4aecbf 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -441,7 +441,7 @@ ${dataInsights latestDataInsights: get(post, "latestDataInsights", []), homepageMetadata: get(post, "homepageMetadata", {}), latestWorkLinks: get(post, "latestWorkLinks", []), - chartViewMetadata: get(post, "chartViewMetadata", {}), + narrativeViewsInfo: get(post, "narrativeViewsInfo", {}), }} > <AtomArticleBlocks blocks={post.content.body} /> diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts index 2113aab0d8..5a7e609694 100644 --- a/db/model/ChartView.ts +++ b/db/model/ChartView.ts @@ -1,11 +1,11 @@ -import { ChartViewMetadata, JsonString } from "@ourworldindata/types" +import { NarrativeViewInfo, JsonString } from "@ourworldindata/types" import * as db from "../db.js" -export const getChartViewsMetadata = async ( +export const getNarrativeViewsInfo = async ( knex: db.KnexReadonlyTransaction, names?: string[] -): Promise<ChartViewMetadata[]> => { - type RawRow = Omit<ChartViewMetadata, "queryParamsForParentChart"> & { +): Promise<NarrativeViewInfo[]> => { + type RawRow = Omit<NarrativeViewInfo, "queryParamsForParentChart"> & { queryParamsForParentChart: JsonString } let rows: RawRow[] diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 90d02c2257..1b0784d563 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -56,7 +56,7 @@ import { import { ARCHVED_THUMBNAIL_FILENAME, ChartConfigType, - ChartViewMetadata, + NarrativeViewInfo, DEFAULT_THUMBNAIL_FILENAME, GrapherInterface, LatestDataInsight, @@ -67,7 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" -import { getChartViewsMetadata } from "../ChartView.js" +import { getNarrativeViewsInfo } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -91,7 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface { linkedIndicators: Record<number, LinkedIndicator> = {} linkedDocuments: Record<string, OwidGdocMinimalPostInterface> = {} latestDataInsights: LatestDataInsight[] = [] - chartViewMetadata?: Record<string, ChartViewMetadata> = {} + narrativeViewsInfo?: Record<string, NarrativeViewInfo> = {} _omittableFields: string[] = [] constructor(id?: string) { @@ -717,12 +717,12 @@ export class GdocBase implements OwidGdocBaseInterface { } } - async loadChartViewMetadata( + async loadNarrativeViewsInfo( knex: db.KnexReadonlyTransaction ): Promise<void> { // TODO: Filter down to only those that are used in the Gdoc - const result = await getChartViewsMetadata(knex) - this.chartViewMetadata = keyBy(result, "name") + const result = await getNarrativeViewsInfo(knex) + this.narrativeViewsInfo = keyBy(result, "name") } async fetchAndEnrichGdoc(): Promise<void> { @@ -870,7 +870,7 @@ export class GdocBase implements OwidGdocBaseInterface { await this.loadImageMetadataFromDB(knex) await this.loadLinkedCharts(knex) await this.loadLinkedIndicators() // depends on linked charts - await this.loadChartViewMetadata(knex) + await this.loadNarrativeViewsInfo(knex) await this._loadSubclassAttachments(knex) await this.validate(knex) } diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 9731db1b70..35187a30f3 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -55,7 +55,7 @@ export interface LinkedChart { } // An object containing metadata needed for embedded narrative charts -export interface ChartViewMetadata { +export interface NarrativeViewInfo { name: string title: string chartConfigId: string diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 3be6749e83..9e94a05ee0 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -332,7 +332,7 @@ export { type OwidGdocContent, type OwidGdocIndexItem, extractGdocIndexItem, - type ChartViewMetadata, + type NarrativeViewInfo, } from "./gdocTypes/Gdoc.js" export { diff --git a/site/gdocs/AttachmentsContext.tsx b/site/gdocs/AttachmentsContext.tsx index 49fc9b30ab..a570a6f7b7 100644 --- a/site/gdocs/AttachmentsContext.tsx +++ b/site/gdocs/AttachmentsContext.tsx @@ -9,7 +9,7 @@ import { LatestDataInsight, OwidGdocHomepageMetadata, DbEnrichedLatestWork, - ChartViewMetadata, + NarrativeViewInfo, } from "@ourworldindata/types" export type Attachments = { @@ -23,7 +23,7 @@ export type Attachments = { latestDataInsights?: LatestDataInsight[] homepageMetadata?: OwidGdocHomepageMetadata latestWorkLinks?: DbEnrichedLatestWork[] - chartViewMetadata?: Record<string, ChartViewMetadata> + narrativeViewsInfo?: Record<string, NarrativeViewInfo> } export const AttachmentsContext = createContext<Attachments>({ @@ -36,5 +36,5 @@ export const AttachmentsContext = createContext<Attachments>({ latestDataInsights: [], homepageMetadata: {}, latestWorkLinks: [], - chartViewMetadata: {}, + narrativeViewsInfo: {}, }) diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index d1502a766d..f0f6137f51 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -93,7 +93,7 @@ export function OwidGdoc({ latestDataInsights: get(props, "latestDataInsights", []), homepageMetadata: get(props, "homepageMetadata", {}), latestWorkLinks: get(props, "latestWorkLinks", []), - chartViewMetadata: get(props, "chartViewMetadata", {}), + narrativeViewsInfo: get(props, "narrativeViewsInfo", {}), }} > <DocumentContext.Provider value={{ isPreviewing }}> diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index 4ca43b37c8..de5604ffdc 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,7 +1,7 @@ import React, { useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" -import { useChartViewMetadata } from "../utils.js" +import { useNarrativeViewsInfo } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" import { AttachmentsContext } from "../../gdocs/AttachmentsContext.js" @@ -20,7 +20,7 @@ export default function NarrativeChart({ const refChartContainer = useRef<HTMLDivElement>(null) useEmbedChart(0, refChartContainer) - const viewMetadata = useChartViewMetadata(d.name) + const viewMetadata = useNarrativeViewsInfo(d.name) if (!viewMetadata) return ( diff --git a/site/gdocs/utils.ts b/site/gdocs/utils.ts index ffd255f73c..78b10b195c 100644 --- a/site/gdocs/utils.ts +++ b/site/gdocs/utils.ts @@ -148,9 +148,9 @@ export function useDonors(): string[] | undefined { return donors } -export const useChartViewMetadata = (name: string) => { - const { chartViewMetadata } = useContext(AttachmentsContext) - return chartViewMetadata?.[name] +export const useNarrativeViewsInfo = (name: string) => { + const { narrativeViewsInfo } = useContext(AttachmentsContext) + return narrativeViewsInfo?.[name] } export function getShortPageCitation( From eab3dac6cacbbd7ebdbcbdc75763118b4a3751f6 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Tue, 17 Dec 2024 16:28:12 +0100 Subject: [PATCH 12/39] enhance: narrative views are reflected as links in gdocs --- db/model/Gdoc/GdocBase.ts | 13 +++++++----- db/model/Link.ts | 20 +++++++++++++++++++ .../types/src/gdocTypes/Gdoc.ts | 1 + 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 1b0784d563..5df9c6164b 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -48,7 +48,7 @@ import { getVariableMetadata, getVariableOfDatapageIfApplicable, } from "../Variable.js" -import { createLinkFromUrl } from "../Link.js" +import { createLinkForNarrativeChart, createLinkFromUrl } from "../Link.js" import { getMultiDimDataPageBySlug, isMultiDimDataPagePublished, @@ -352,6 +352,13 @@ export class GdocBase implements OwidGdocBaseInterface { componentType: block.type, }), ]) + .with({ type: "narrative-chart" }, (block) => [ + createLinkForNarrativeChart({ + name: block.name, + source: this, + componentType: block.type, + }), + ]) .with({ type: "all-charts" }, (block) => block.top.map((item) => createLinkFromUrl({ @@ -580,10 +587,6 @@ export class GdocBase implements OwidGdocBaseInterface { "key-indicator-collection", "list", "missing-data", - - // Open question: there's not a direct link to a chart here, but there is a chart and also a parent chart - "narrative-chart", - "numbered-list", "people", "people-rows", diff --git a/db/model/Link.ts b/db/model/Link.ts index 4468e6832d..c2a62a16cc 100644 --- a/db/model/Link.ts +++ b/db/model/Link.ts @@ -62,3 +62,23 @@ export function createLinkFromUrl({ sourceId: source.id, } satisfies DbInsertPostGdocLink } + +export function createLinkForNarrativeChart({ + name, + source, + componentType, +}: { + name: string + source: GdocBase + componentType: string +}): DbInsertPostGdocLink { + return { + target: name, + linkType: OwidGdocLinkType.NarrativeChart, + queryString: "", + hash: "", + text: "", + componentType, + sourceId: source.id, + } satisfies DbInsertPostGdocLink +} diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 35187a30f3..9f75af5e70 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -281,6 +281,7 @@ export enum OwidGdocLinkType { Url = "url", Grapher = "grapher", Explorer = "explorer", + NarrativeChart = "narrative-chart", } export interface OwidGdocLinkJSON { From 8b1fa6e16861bd272ab834fac7c9ad63a6226636 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Tue, 17 Dec 2024 16:38:01 +0100 Subject: [PATCH 13/39] refactor: filter down `narrativeViewsInfo` --- baker/SiteBaker.tsx | 12 +++++++++--- db/model/Gdoc/GdocBase.ts | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 24ca2315cc..29dc2c2c93 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -348,7 +348,7 @@ export class SiteBaker { _prefetchedAttachmentsCache: PrefetchedAttachments | undefined = undefined private async getPrefetchedGdocAttachments( knex: db.KnexReadonlyTransaction, - picks?: [string[], string[], string[], string[], string[]] + picks?: [string[], string[], string[], string[], string[], string[]] ): Promise<PrefetchedAttachments> { if (!this._prefetchedAttachmentsCache) { console.log("Prefetching attachments...") @@ -490,6 +490,7 @@ export class SiteBaker { imageFilenames, linkedGrapherSlugs, linkedExplorerSlugs, + linkedNarrativeChartNames, ] = picks const linkedDocuments = pick( this._prefetchedAttachmentsCache.linkedDocuments, @@ -538,8 +539,10 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), - narrativeViewsInfo: - this._prefetchedAttachmentsCache.narrativeViewsInfo, // TODO: Filter + narrativeViewsInfo: pick( + this._prefetchedAttachmentsCache.narrativeViewsInfo, + linkedNarrativeChartNames + ), } } return this._prefetchedAttachmentsCache @@ -631,6 +634,7 @@ export class SiteBaker { publishedGdoc.linkedImageFilenames, publishedGdoc.linkedChartSlugs.grapher, publishedGdoc.linkedChartSlugs.explorer, + publishedGdoc.linkedNarrativeChartNames, ]) publishedGdoc.donors = attachments.donors publishedGdoc.linkedAuthors = attachments.linkedAuthors @@ -889,6 +893,7 @@ export class SiteBaker { dataInsight.linkedImageFilenames, dataInsight.linkedChartSlugs.grapher, dataInsight.linkedChartSlugs.explorer, + dataInsight.linkedNarrativeChartNames, ]) dataInsight.linkedDocuments = attachments.linkedDocuments dataInsight.imageMetadata = { @@ -962,6 +967,7 @@ export class SiteBaker { publishedAuthor.linkedImageFilenames, publishedAuthor.linkedChartSlugs.grapher, publishedAuthor.linkedChartSlugs.explorer, + publishedAuthor.linkedNarrativeChartNames, ]) // We don't need these to be attached to the gdoc in the current diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 5df9c6164b..a93e49946d 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -295,6 +295,14 @@ export class GdocBase implements OwidGdocBaseInterface { return { grapher: [...grapher], explorer: [...explorer] } } + get linkedNarrativeChartNames(): string[] { + const filteredLinks = this.links + .filter((link) => link.linkType === "narrative-chart") + .map((link) => link.target) + + return filteredLinks + } + get hasAllChartsBlock(): boolean { let hasAllChartsBlock = false for (const enrichedBlockSource of this.enrichedBlockSources) { @@ -723,8 +731,10 @@ export class GdocBase implements OwidGdocBaseInterface { async loadNarrativeViewsInfo( knex: db.KnexReadonlyTransaction ): Promise<void> { - // TODO: Filter down to only those that are used in the Gdoc - const result = await getNarrativeViewsInfo(knex) + const result = await getNarrativeViewsInfo( + knex, + this.linkedNarrativeChartNames + ) this.narrativeViewsInfo = keyBy(result, "name") } From 3a7ceca3a3f575789ae740adf496623a867e3a7b Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Tue, 17 Dec 2024 18:06:54 +0100 Subject: [PATCH 14/39] refactor: change `linkType` enum --- ...4799588-PostsGdocsLinksAddNarrativeCharts.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts diff --git a/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts b/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts new file mode 100644 index 0000000000..cb1d710cf9 --- /dev/null +++ b/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm" + +export class PostsGdocsLinksAddNarrativeCharts1734454799588 + implements MigrationInterface +{ + public async up(queryRunner: QueryRunner): Promise<void> { + await queryRunner.query(` + ALTER TABLE posts_gdocs_links + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer', 'narrative-chart') NULL`) + } + + public async down(queryRunner: QueryRunner): Promise<void> { + await queryRunner.query(` + ALTER TABLE posts_gdocs_links + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer') NULL`) + } +} From 02717eca2a329f6523ccd8c307c494eeab6261ff Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Wed, 18 Dec 2024 20:28:11 +0100 Subject: [PATCH 15/39] fix: fix error when publishing NarrativeChart with error --- site/gdocs/components/NarrativeChart.tsx | 30 ++++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index de5604ffdc..c5b3fa6f3e 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,12 +1,12 @@ -import React, { useRef } from "react" +import { useContext, useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" import { useNarrativeViewsInfo } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" -import { AttachmentsContext } from "../../gdocs/AttachmentsContext.js" import { BlockErrorFallback } from "./BlockErrorBoundary.js" import SpanElements from "./SpanElements.js" +import { DocumentContext } from "../DocumentContext.js" export default function NarrativeChart({ d, @@ -22,16 +22,21 @@ export default function NarrativeChart({ const viewMetadata = useNarrativeViewsInfo(d.name) - if (!viewMetadata) - return ( - <BlockErrorFallback - className={className} - error={{ - name: "Narrative view not found", - message: `Narrative view with name "${d.name}" couldn't be found.`, - }} - /> - ) + const { isPreviewing } = useContext(DocumentContext) + + if (!viewMetadata) { + if (isPreviewing) { + return ( + <BlockErrorFallback + className={className} + error={{ + name: "Narrative view not found", + message: `Narrative view with name "${d.name}" couldn't be found.`, + }} + /> + ) + } else return null // If not previewing, just don't render anything + } const metadataStringified = JSON.stringify(viewMetadata) @@ -47,7 +52,6 @@ export default function NarrativeChart({ key={metadataStringified} className={cx(GRAPHER_PREVIEW_CLASS, "chart")} data-grapher-view-config={metadataStringified} - // data-grapher-src={isExplorer ? undefined : resolvedUrl} style={{ width: "100%", border: "0px none", From 95b23b83e88198044ff23266b9f22f09d87b3166 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Wed, 18 Dec 2024 22:09:56 +0100 Subject: [PATCH 16/39] enhance: add narrative chart support to MultiEmbedder --- .../grapher/src/core/GrapherConstants.ts | 3 + packages/@ourworldindata/grapher/src/index.ts | 1 + site/multiembedder/MultiEmbedder.tsx | 283 +++++++++++------- 3 files changed, 180 insertions(+), 107 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index 1601f36f65..a4f8a89682 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -5,6 +5,9 @@ import type { GrapherProgrammaticInterface } from "./Grapher" export const GRAPHER_EMBEDDED_FIGURE_ATTR = "data-grapher-src" export const GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR = "data-grapher-config" +export const GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR = + "data-grapher-view-config" + export const GRAPHER_PAGE_BODY_CLASS = "StandaloneGrapherOrExplorerPage" export const GRAPHER_IS_IN_IFRAME_CLASS = "IsInIframe" export const GRAPHER_TIMELINE_CLASS = "timeline-component" diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index 8d52675e86..ac3a8142dc 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -10,6 +10,7 @@ export { ChartDimension } from "./chart/ChartDimension" export { GRAPHER_EMBEDDED_FIGURE_ATTR, GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR, + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, GRAPHER_PAGE_BODY_CLASS, GRAPHER_IS_IN_IFRAME_CLASS, DEFAULT_GRAPHER_WIDTH, diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index 45a41a0394..e06e78a75b 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -10,6 +10,7 @@ import { migrateSelectedEntityNamesParam, SelectionArray, migrateGrapherConfigToLatestVersion, + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, } from "@ourworldindata/grapher" import { fetchText, @@ -21,6 +22,7 @@ import { MultiDimDataPageConfig, extractMultiDimChoicesFromQueryStr, fetchWithRetry, + NarrativeViewInfo, } from "@ourworldindata/utils" import { action } from "mobx" import ReactDOM from "react-dom" @@ -41,6 +43,9 @@ import { } from "../../settings/clientSettings.js" import Bugsnag from "@bugsnag/js" import { embedDynamicCollectionGrapher } from "../collections/DynamicCollection.js" +import { match } from "ts-pattern" + +type EmbedType = "grapher" | "explorer" | "multiDim" | "grapherView" const figuresFromDOM = ( container: HTMLElement | Document = document, @@ -109,10 +114,16 @@ class MultiEmbedder { * Use this when you programmatically create/replace charts. */ observeFigures(container: HTMLElement | Document = document) { - const figures = figuresFromDOM( - container, - GRAPHER_EMBEDDED_FIGURE_ATTR - ).concat(figuresFromDOM(container, EXPLORER_EMBEDDED_FIGURE_SELECTOR)) + const figures = figuresFromDOM(container, GRAPHER_EMBEDDED_FIGURE_ATTR) + .concat( + figuresFromDOM(container, EXPLORER_EMBEDDED_FIGURE_SELECTOR) + ) + .concat( + figuresFromDOM( + container, + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + ) figures.forEach((figure) => { this.figuresObserver?.observe(figure) @@ -127,33 +138,41 @@ class MultiEmbedder { }) } - @action.bound - async renderInteractiveFigure(figure: Element) { - const isExplorer = figure.hasAttribute( + async renderExplorerIntoFigure(figure: Element) { + const explorerUrl = figure.getAttribute( EXPLORER_EMBEDDED_FIGURE_SELECTOR ) - const isMultiDim = figure.hasAttribute("data-is-multi-dim") - const dataSrc = figure.getAttribute( - isExplorer - ? EXPLORER_EMBEDDED_FIGURE_SELECTOR - : GRAPHER_EMBEDDED_FIGURE_ATTR - ) + if (!explorerUrl) return - if (!dataSrc) return + const { fullUrl, queryStr } = Url.fromURL(explorerUrl) - const hasPreview = isExplorer ? false : !!figure.querySelector("img") - if (!shouldProgressiveEmbed() && hasPreview) return - - // Stop observing visibility as soon as possible, that is not before - // shouldProgressiveEmbed gets a chance to reevaluate a possible change - // in screen size on mobile (i.e. after a rotation). Stopping before - // shouldProgressiveEmbed would prevent rendering interactive charts - // when going from portrait to landscape mode (without page reload). - this.figuresObserver?.unobserve(figure) + const html = await fetchText(fullUrl) + const props: ExplorerProps = await buildExplorerProps( + html, + queryStr, + this.selection + ) + if (props.selection) + this.graphersAndExplorersToUpdate.add(props.selection) + ReactDOM.render(<Explorer {...props} />, figure) + } - const { fullUrl, queryStr, queryParams } = Url.fromURL(dataSrc) + private async _renderGrapherComponentIntoFigure( + figure: Element, + { + configUrl, + embedUrl, + additionalConfig, + }: { + configUrl: string + embedUrl?: Url + additionalConfig?: Partial<GrapherProgrammaticInterface> + } + ) { + const { queryStr, queryParams } = embedUrl ?? {} + figure.classList.remove(GRAPHER_PREVIEW_CLASS) const common: GrapherProgrammaticInterface = { isEmbeddedInAnOwidPage: true, queryStr, @@ -162,95 +181,145 @@ class MultiEmbedder { dataApiUrl: DATA_API_URL, } - if (isExplorer) { - const html = await fetchText(fullUrl) - const props: ExplorerProps = await buildExplorerProps( - html, - queryStr, - this.selection - ) - if (props.selection) - this.graphersAndExplorersToUpdate.add(props.selection) - ReactDOM.render(<Explorer {...props} />, figure) - } else { - figure.classList.remove(GRAPHER_PREVIEW_CLASS) - const url = new URL(fullUrl) - const slug = url.pathname.split("/").pop() - let configUrl - if (isMultiDim) { - const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json` - const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then( - (res) => res.json() - ) - const mdimConfig = - MultiDimDataPageConfig.fromObject(mdimJsonConfig) - const dimensions = extractMultiDimChoicesFromQueryStr( - url.search, - mdimConfig - ) - const view = mdimConfig.findViewByDimensions(dimensions) - if (!view) { - throw new Error( - `No view found for dimensions ${JSON.stringify( - dimensions - )}` - ) - } - configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${view.fullConfigId}.config.json` - } else { - configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${slug}.config.json` - } - const fetchedGrapherPageConfig = await fetchWithRetry( - configUrl - ).then((res) => res.json()) - const grapherPageConfig = migrateGrapherConfigToLatestVersion( - fetchedGrapherPageConfig - ) + const fetchedGrapherPageConfig = await fetchWithRetry(configUrl).then( + (res) => res.json() + ) + const grapherPageConfig = migrateGrapherConfigToLatestVersion( + fetchedGrapherPageConfig + ) - const figureConfigAttr = figure.getAttribute( - GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR - ) - const localConfig = figureConfigAttr - ? JSON.parse(figureConfigAttr) - : {} - - // make sure the tab of the active pane is visible - if (figureConfigAttr && !isEmpty(localConfig)) { - const activeTab = queryParams.tab || grapherPageConfig.tab - if (activeTab === GRAPHER_TAB_OPTIONS.chart) - localConfig.hideChartTabs = false - if (activeTab === GRAPHER_TAB_OPTIONS.map) - localConfig.hasMapTab = true - if (activeTab === GRAPHER_TAB_OPTIONS.table) - localConfig.hasTableTab = true + const figureConfigAttr = figure.getAttribute( + GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR + ) + const localConfig = figureConfigAttr ? JSON.parse(figureConfigAttr) : {} + + // make sure the tab of the active pane is visible + if (figureConfigAttr && !isEmpty(localConfig)) { + const activeTab = queryParams?.tab || grapherPageConfig.tab + if (activeTab === GRAPHER_TAB_OPTIONS.chart) + localConfig.hideChartTabs = false + if (activeTab === GRAPHER_TAB_OPTIONS.map) + localConfig.hasMapTab = true + if (activeTab === GRAPHER_TAB_OPTIONS.table) + localConfig.hasTableTab = true + } + + const config = merge( + {}, // merge mutates the first argument + grapherPageConfig, + common, + additionalConfig, + localConfig, + { + manager: { + selection: new SelectionArray( + this.selection.selectedEntityNames + ), + }, } + ) + if (config.manager?.selection) + this.graphersAndExplorersToUpdate.add(config.manager.selection) - const config = merge( - {}, // merge mutates the first argument - grapherPageConfig, - common, - localConfig, - { - manager: { - selection: new SelectionArray( - this.selection.selectedEntityNames - ), - }, - } - ) - if (config.manager?.selection) - this.graphersAndExplorersToUpdate.add(config.manager.selection) + const grapherRef = Grapher.renderGrapherIntoContainer(config, figure) - const grapherRef = Grapher.renderGrapherIntoContainer( - config, - figure - ) + // Special handling for shared collections + if (window.location.pathname.startsWith("/collection/custom")) { + embedDynamicCollectionGrapher(grapherRef, figure) + } + } + async renderGrapherIntoFigure(figure: Element) { + const embedUrlRaw = figure.getAttribute(GRAPHER_EMBEDDED_FIGURE_ATTR) + if (!embedUrlRaw) return + const embedUrl = Url.fromURL(embedUrlRaw) - // Special handling for shared collections - if (window.location.pathname.startsWith("/collection/custom")) { - embedDynamicCollectionGrapher(grapherRef, figure) - } + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${embedUrl.slug}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + embedUrl, + }) + } + async renderMultiDimIntoFigure(figure: Element) { + const embedUrlRaw = figure.getAttribute(GRAPHER_EMBEDDED_FIGURE_ATTR) + if (!embedUrlRaw) return + const embedUrl = Url.fromURL(embedUrlRaw) + + const { queryStr, slug } = embedUrl + + const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json` + const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then((res) => + res.json() + ) + const mdimConfig = MultiDimDataPageConfig.fromObject(mdimJsonConfig) + const dimensions = extractMultiDimChoicesFromQueryStr( + queryStr, + mdimConfig + ) + const view = mdimConfig.findViewByDimensions(dimensions) + if (!view) { + throw new Error( + `No view found for dimensions ${JSON.stringify(dimensions)}` + ) } + + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${view.fullConfigId}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + embedUrl, + }) + } + async renderGrapherViewIntoFigure(figure: Element) { + const viewConfigRaw = figure.getAttribute( + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + if (!viewConfigRaw) return + const viewConfig: NarrativeViewInfo = JSON.parse(viewConfigRaw) + if (!viewConfig) return + + const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` + + await this._renderGrapherComponentIntoFigure(figure, { + configUrl, + additionalConfig: {}, + }) + } + + @action.bound + async renderInteractiveFigure(figure: Element) { + const isExplorer = figure.hasAttribute( + EXPLORER_EMBEDDED_FIGURE_SELECTOR + ) + const isMultiDim = figure.hasAttribute("data-is-multi-dim") + const isGrapherView = figure.hasAttribute( + GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + ) + + const embedType: EmbedType = isExplorer + ? "explorer" + : isMultiDim + ? "multiDim" + : isGrapherView + ? "grapherView" + : "grapher" + + const hasPreview = isExplorer ? false : !!figure.querySelector("img") + if (!shouldProgressiveEmbed() && hasPreview) return + + // Stop observing visibility as soon as possible, that is not before + // shouldProgressiveEmbed gets a chance to reevaluate a possible change + // in screen size on mobile (i.e. after a rotation). Stopping before + // shouldProgressiveEmbed would prevent rendering interactive charts + // when going from portrait to landscape mode (without page reload). + this.figuresObserver?.unobserve(figure) + + await match(embedType) + .with("explorer", () => this.renderExplorerIntoFigure(figure)) + .with("multiDim", () => this.renderMultiDimIntoFigure(figure)) + .with("grapherView", () => this.renderGrapherViewIntoFigure(figure)) + .with("grapher", () => this.renderGrapherIntoFigure(figure)) + .exhaustive() } setUpGlobalEntitySelectorForEmbeds() { From cab124febf4273473fd4284242f7ba71d1f52864 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Wed, 18 Dec 2024 22:35:40 +0100 Subject: [PATCH 17/39] enhance: basic config to hide some grapher elements --- site/multiembedder/MultiEmbedder.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index e06e78a75b..b9bb9cd93e 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -23,6 +23,7 @@ import { extractMultiDimChoicesFromQueryStr, fetchWithRetry, NarrativeViewInfo, + queryParamsToStr, } from "@ourworldindata/utils" import { action } from "mobx" import ReactDOM from "react-dom" @@ -280,9 +281,18 @@ class MultiEmbedder { const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` + const queryStr = queryParamsToStr(viewConfig.queryParamsForParentChart) + await this._renderGrapherComponentIntoFigure(figure, { configUrl, - additionalConfig: {}, + additionalConfig: { + hideRelatedQuestion: true, + hideShareButton: true, // always hidden since the original chart would be shared, not the customized one + hideExploreTheDataButton: false, + manager: { + canonicalUrl: `${BAKED_GRAPHER_URL}/${viewConfig.parentChartSlug}${queryStr}`, + }, + }, }) } From cf94feddedf2212c3f8d3014618de084158b4d5e Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Wed, 18 Dec 2024 22:45:01 +0100 Subject: [PATCH 18/39] fix: correctly generate narrative view query params --- adminSiteServer/apiRouter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 5cf0e042bc..96278efa0f 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -3639,7 +3639,7 @@ const createPatchConfigAndQueryParamsForChartView = async ( ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST), } - const queryParams = grapherConfigToQueryParams(config) + const queryParams = grapherConfigToQueryParams(patchConfigToSave) const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave) return { patchConfig: patchConfigToSave, fullConfig, queryParams } From cc778475c47d3329de8832835ac53f860f63a8d6 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 19 Dec 2024 18:18:10 +0100 Subject: [PATCH 19/39] refactor: use consistent names -- chart views & narrative charts --- adminSiteClient/AdminSidebar.tsx | 2 +- adminSiteClient/ChartEditor.ts | 4 +-- adminSiteClient/ChartViewIndexPage.tsx | 2 +- adminSiteClient/EditorReferencesTab.tsx | 2 +- adminSiteClient/SaveButtons.tsx | 8 ++--- baker/SiteBaker.tsx | 30 +++++++++---------- baker/siteRenderers.tsx | 2 +- ...454799588-PostsGdocsLinksAddChartViews.ts} | 4 +-- db/model/ChartView.ts | 8 ++--- db/model/Gdoc/GdocBase.ts | 27 +++++++---------- db/model/Link.ts | 4 +-- .../grapher/src/core/GrapherConstants.ts | 4 +-- packages/@ourworldindata/grapher/src/index.ts | 2 +- .../types/src/gdocTypes/Gdoc.ts | 4 +-- packages/@ourworldindata/types/src/index.ts | 2 +- site/gdocs/AttachmentsContext.tsx | 6 ++-- site/gdocs/OwidGdoc.tsx | 2 +- site/gdocs/components/NarrativeChart.tsx | 14 +++++---- site/gdocs/utils.ts | 6 ++-- site/multiembedder/MultiEmbedder.tsx | 24 +++++++-------- 20 files changed, 78 insertions(+), 79 deletions(-) rename db/migration/{1734454799588-PostsGdocsLinksAddNarrativeCharts.ts => 1734454799588-PostsGdocsLinksAddChartViews.ts} (85%) diff --git a/adminSiteClient/AdminSidebar.tsx b/adminSiteClient/AdminSidebar.tsx index 3b7ad4c2d0..ec76340cbe 100644 --- a/adminSiteClient/AdminSidebar.tsx +++ b/adminSiteClient/AdminSidebar.tsx @@ -37,7 +37,7 @@ export const AdminSidebar = (): React.ReactElement => ( {chartViewsFeatureEnabled && ( <li> <Link to="/chartViews"> - <FontAwesomeIcon icon={faPanorama} /> Narrative views + <FontAwesomeIcon icon={faPanorama} /> Narrative charts </Link> </li> )} diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index 1a4747fbb3..25b10df424 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -200,7 +200,7 @@ export class ChartEditor extends AbstractChartEditor<ChartEditorManager> { ) } - async saveAsNarrativeView(): Promise<void> { + async saveAsChartView(): Promise<void> { const { patchConfig, grapher } = this const chartJson = omit(patchConfig, CHART_VIEW_PROPS_TO_OMIT) @@ -208,7 +208,7 @@ export class ChartEditor extends AbstractChartEditor<ChartEditorManager> { const suggestedName = grapher.title ? slugify(grapher.title) : undefined const name = prompt( - "Please enter a programmatic name for the narrative view. Note that this name cannot be changed later.", + "Please enter a programmatic name for the narrative chart. Note that this name cannot be changed later.", suggestedName ) diff --git a/adminSiteClient/ChartViewIndexPage.tsx b/adminSiteClient/ChartViewIndexPage.tsx index 4211ba1d15..3bc945314f 100644 --- a/adminSiteClient/ChartViewIndexPage.tsx +++ b/adminSiteClient/ChartViewIndexPage.tsx @@ -135,7 +135,7 @@ export function ChartViewIndexPage() { }, [admin]) return ( - <AdminLayout title="Narrative views"> + <AdminLayout title="Narrative charts"> <main className="ChartViewIndexPage"> <Flex justify="space-between"> <Input diff --git a/adminSiteClient/EditorReferencesTab.tsx b/adminSiteClient/EditorReferencesTab.tsx index 5e91f17300..02ee7aaa2c 100644 --- a/adminSiteClient/EditorReferencesTab.tsx +++ b/adminSiteClient/EditorReferencesTab.tsx @@ -127,7 +127,7 @@ export const ReferencesSection = (props: { const chartViews = !!props.references?.chartViews?.length && ( <> - <p>Narrative views based on this chart</p> + <p>Narrative charts based on this chart</p> <ul className="list-group"> {props.references.chartViews.map((chartView) => ( <li key={chartView.id} className="list-group-item"> diff --git a/adminSiteClient/SaveButtons.tsx b/adminSiteClient/SaveButtons.tsx index ce63127a91..3166add379 100644 --- a/adminSiteClient/SaveButtons.tsx +++ b/adminSiteClient/SaveButtons.tsx @@ -61,8 +61,8 @@ class SaveButtonsForChart extends Component<{ void this.props.editor.saveAsNewGrapher() } - @action.bound onSaveAsNarrativeView() { - void this.props.editor.saveAsNarrativeView() + @action.bound onSaveAsChartView() { + void this.props.editor.saveAsChartView() } @action.bound onPublishToggle() { @@ -120,10 +120,10 @@ class SaveButtonsForChart extends Component<{ <div className="mt-2"> <button className="btn btn-primary" - onClick={this.onSaveAsNarrativeView} + onClick={this.onSaveAsChartView} disabled={isSavingDisabled} > - Save as narrative view + Save as narrative chart </button> </div> )} diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 29dc2c2c93..52f9b4cb51 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -56,7 +56,7 @@ import { grabMetadataForGdocLinkedIndicator, TombstonePageData, gdocUrlRegex, - NarrativeViewInfo, + ChartViewInfo, } from "@ourworldindata/utils" import { execWrapper } from "../db/execWrapper.js" import { countryProfileSpecs } from "../site/countryProfileProjects.js" @@ -110,7 +110,7 @@ import { getTombstones } from "../db/model/GdocTombstone.js" import { bakeAllMultiDimDataPages } from "./MultiDimBaker.js" import { getAllLinkedPublishedMultiDimDataPages } from "../db/model/MultiDimDataPage.js" import { getPublicDonorNames } from "../db/model/Donor.js" -import { getNarrativeViewsInfo } from "../db/model/ChartView.js" +import { getChartViewsInfo } from "../db/model/ChartView.js" type PrefetchedAttachments = { donors: string[] @@ -122,7 +122,7 @@ type PrefetchedAttachments = { explorers: Record<string, LinkedChart> } linkedIndicators: Record<number, LinkedIndicator> - narrativeViewsInfo: Record<string, NarrativeViewInfo> + linkedChartViews: Record<string, ChartViewInfo> } // These aren't all "wordpress" steps @@ -462,10 +462,10 @@ export class SiteBaker { name: `✅ Prefetched ${publishedAuthors.length} authors`, }) - const narrativeViewsInfo = await getNarrativeViewsInfo(knex) - const narrativeViewsInfoByName = keyBy(narrativeViewsInfo, "name") + const chartViewsInfo = await getChartViewsInfo(knex) + const chartViewsInfoByName = keyBy(chartViewsInfo, "name") this.progressBar.tick({ - name: `✅ Prefetched ${narrativeViewsInfo.length} chart views`, + name: `✅ Prefetched ${chartViewsInfo.length} chart views`, }) const prefetchedAttachments = { @@ -478,7 +478,7 @@ export class SiteBaker { graphers: publishedChartsBySlug, }, linkedIndicators: datapageIndicatorsById, - narrativeViewsInfo: narrativeViewsInfoByName, + linkedChartViews: chartViewsInfoByName, } this.progressBar.tick({ name: "✅ Prefetched attachments" }) this._prefetchedAttachmentsCache = prefetchedAttachments @@ -490,7 +490,7 @@ export class SiteBaker { imageFilenames, linkedGrapherSlugs, linkedExplorerSlugs, - linkedNarrativeChartNames, + linkedChartViewNames, ] = picks const linkedDocuments = pick( this._prefetchedAttachmentsCache.linkedDocuments, @@ -539,9 +539,9 @@ export class SiteBaker { this._prefetchedAttachmentsCache.linkedAuthors.filter( (author) => authorNames.includes(author.name) ), - narrativeViewsInfo: pick( - this._prefetchedAttachmentsCache.narrativeViewsInfo, - linkedNarrativeChartNames + linkedChartViews: pick( + this._prefetchedAttachmentsCache.linkedChartViews, + linkedChartViewNames ), } } @@ -634,7 +634,7 @@ export class SiteBaker { publishedGdoc.linkedImageFilenames, publishedGdoc.linkedChartSlugs.grapher, publishedGdoc.linkedChartSlugs.explorer, - publishedGdoc.linkedNarrativeChartNames, + publishedGdoc.linkedChartViewNames, ]) publishedGdoc.donors = attachments.donors publishedGdoc.linkedAuthors = attachments.linkedAuthors @@ -645,7 +645,7 @@ export class SiteBaker { ...attachments.linkedCharts.explorers, } publishedGdoc.linkedIndicators = attachments.linkedIndicators - publishedGdoc.narrativeViewsInfo = attachments.narrativeViewsInfo + publishedGdoc.linkedChartViews = attachments.linkedChartViews // this is a no-op if the gdoc doesn't have an all-chart block if ("loadRelatedCharts" in publishedGdoc) { @@ -893,7 +893,7 @@ export class SiteBaker { dataInsight.linkedImageFilenames, dataInsight.linkedChartSlugs.grapher, dataInsight.linkedChartSlugs.explorer, - dataInsight.linkedNarrativeChartNames, + dataInsight.linkedChartViewNames, ]) dataInsight.linkedDocuments = attachments.linkedDocuments dataInsight.imageMetadata = { @@ -967,7 +967,7 @@ export class SiteBaker { publishedAuthor.linkedImageFilenames, publishedAuthor.linkedChartSlugs.grapher, publishedAuthor.linkedChartSlugs.explorer, - publishedAuthor.linkedNarrativeChartNames, + publishedAuthor.linkedChartViewNames, ]) // We don't need these to be attached to the gdoc in the current diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index ba9d4aecbf..e64743f932 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -441,7 +441,7 @@ ${dataInsights latestDataInsights: get(post, "latestDataInsights", []), homepageMetadata: get(post, "homepageMetadata", {}), latestWorkLinks: get(post, "latestWorkLinks", []), - narrativeViewsInfo: get(post, "narrativeViewsInfo", {}), + linkedChartViews: get(post, "linkedChartViews", {}), }} > <AtomArticleBlocks blocks={post.content.body} /> diff --git a/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts b/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts similarity index 85% rename from db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts rename to db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts index cb1d710cf9..06596eea89 100644 --- a/db/migration/1734454799588-PostsGdocsLinksAddNarrativeCharts.ts +++ b/db/migration/1734454799588-PostsGdocsLinksAddChartViews.ts @@ -1,12 +1,12 @@ import { MigrationInterface, QueryRunner } from "typeorm" -export class PostsGdocsLinksAddNarrativeCharts1734454799588 +export class PostsGdocsLinksAddChartViews1734454799588 implements MigrationInterface { public async up(queryRunner: QueryRunner): Promise<void> { await queryRunner.query(` ALTER TABLE posts_gdocs_links - MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer', 'narrative-chart') NULL`) + MODIFY linkType ENUM ('gdoc', 'url', 'grapher', 'explorer', 'chart-view') NULL`) } public async down(queryRunner: QueryRunner): Promise<void> { diff --git a/db/model/ChartView.ts b/db/model/ChartView.ts index 5a7e609694..2bfd5ad8ed 100644 --- a/db/model/ChartView.ts +++ b/db/model/ChartView.ts @@ -1,11 +1,11 @@ -import { NarrativeViewInfo, JsonString } from "@ourworldindata/types" +import { ChartViewInfo, JsonString } from "@ourworldindata/types" import * as db from "../db.js" -export const getNarrativeViewsInfo = async ( +export const getChartViewsInfo = async ( knex: db.KnexReadonlyTransaction, names?: string[] -): Promise<NarrativeViewInfo[]> => { - type RawRow = Omit<NarrativeViewInfo, "queryParamsForParentChart"> & { +): Promise<ChartViewInfo[]> => { + type RawRow = Omit<ChartViewInfo, "queryParamsForParentChart"> & { queryParamsForParentChart: JsonString } let rows: RawRow[] diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index a93e49946d..173fd06e6f 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -48,7 +48,7 @@ import { getVariableMetadata, getVariableOfDatapageIfApplicable, } from "../Variable.js" -import { createLinkForNarrativeChart, createLinkFromUrl } from "../Link.js" +import { createLinkForChartView, createLinkFromUrl } from "../Link.js" import { getMultiDimDataPageBySlug, isMultiDimDataPagePublished, @@ -56,7 +56,7 @@ import { import { ARCHVED_THUMBNAIL_FILENAME, ChartConfigType, - NarrativeViewInfo, + ChartViewInfo, DEFAULT_THUMBNAIL_FILENAME, GrapherInterface, LatestDataInsight, @@ -67,7 +67,7 @@ import { OwidGdocLinkType, OwidGdocType, } from "@ourworldindata/types" -import { getNarrativeViewsInfo } from "../ChartView.js" +import { getChartViewsInfo } from "../ChartView.js" export class GdocBase implements OwidGdocBaseInterface { id!: string @@ -91,7 +91,7 @@ export class GdocBase implements OwidGdocBaseInterface { linkedIndicators: Record<number, LinkedIndicator> = {} linkedDocuments: Record<string, OwidGdocMinimalPostInterface> = {} latestDataInsights: LatestDataInsight[] = [] - narrativeViewsInfo?: Record<string, NarrativeViewInfo> = {} + linkedChartViews?: Record<string, ChartViewInfo> = {} _omittableFields: string[] = [] constructor(id?: string) { @@ -295,9 +295,9 @@ export class GdocBase implements OwidGdocBaseInterface { return { grapher: [...grapher], explorer: [...explorer] } } - get linkedNarrativeChartNames(): string[] { + get linkedChartViewNames(): string[] { const filteredLinks = this.links - .filter((link) => link.linkType === "narrative-chart") + .filter((link) => link.linkType === OwidGdocLinkType.ChartView) .map((link) => link.target) return filteredLinks @@ -361,7 +361,7 @@ export class GdocBase implements OwidGdocBaseInterface { }), ]) .with({ type: "narrative-chart" }, (block) => [ - createLinkForNarrativeChart({ + createLinkForChartView({ name: block.name, source: this, componentType: block.type, @@ -728,14 +728,9 @@ export class GdocBase implements OwidGdocBaseInterface { } } - async loadNarrativeViewsInfo( - knex: db.KnexReadonlyTransaction - ): Promise<void> { - const result = await getNarrativeViewsInfo( - knex, - this.linkedNarrativeChartNames - ) - this.narrativeViewsInfo = keyBy(result, "name") + async loadChartViewsInfo(knex: db.KnexReadonlyTransaction): Promise<void> { + const result = await getChartViewsInfo(knex, this.linkedChartViewNames) + this.linkedChartViews = keyBy(result, "name") } async fetchAndEnrichGdoc(): Promise<void> { @@ -883,7 +878,7 @@ export class GdocBase implements OwidGdocBaseInterface { await this.loadImageMetadataFromDB(knex) await this.loadLinkedCharts(knex) await this.loadLinkedIndicators() // depends on linked charts - await this.loadNarrativeViewsInfo(knex) + await this.loadChartViewsInfo(knex) await this._loadSubclassAttachments(knex) await this.validate(knex) } diff --git a/db/model/Link.ts b/db/model/Link.ts index c2a62a16cc..bb0d2941cf 100644 --- a/db/model/Link.ts +++ b/db/model/Link.ts @@ -63,7 +63,7 @@ export function createLinkFromUrl({ } satisfies DbInsertPostGdocLink } -export function createLinkForNarrativeChart({ +export function createLinkForChartView({ name, source, componentType, @@ -74,7 +74,7 @@ export function createLinkForNarrativeChart({ }): DbInsertPostGdocLink { return { target: name, - linkType: OwidGdocLinkType.NarrativeChart, + linkType: OwidGdocLinkType.ChartView, queryString: "", hash: "", text: "", diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index a4f8a89682..dc7143bc79 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -5,8 +5,8 @@ import type { GrapherProgrammaticInterface } from "./Grapher" export const GRAPHER_EMBEDDED_FIGURE_ATTR = "data-grapher-src" export const GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR = "data-grapher-config" -export const GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR = - "data-grapher-view-config" +export const GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR = + "data-grapher-chart-view-config" export const GRAPHER_PAGE_BODY_CLASS = "StandaloneGrapherOrExplorerPage" export const GRAPHER_IS_IN_IFRAME_CLASS = "IsInIframe" diff --git a/packages/@ourworldindata/grapher/src/index.ts b/packages/@ourworldindata/grapher/src/index.ts index ac3a8142dc..13ca26a69c 100644 --- a/packages/@ourworldindata/grapher/src/index.ts +++ b/packages/@ourworldindata/grapher/src/index.ts @@ -10,7 +10,7 @@ export { ChartDimension } from "./chart/ChartDimension" export { GRAPHER_EMBEDDED_FIGURE_ATTR, GRAPHER_EMBEDDED_FIGURE_CONFIG_ATTR, - GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, GRAPHER_PAGE_BODY_CLASS, GRAPHER_IS_IN_IFRAME_CLASS, DEFAULT_GRAPHER_WIDTH, diff --git a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts index 9f75af5e70..318c5d1e5b 100644 --- a/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts +++ b/packages/@ourworldindata/types/src/gdocTypes/Gdoc.ts @@ -55,7 +55,7 @@ export interface LinkedChart { } // An object containing metadata needed for embedded narrative charts -export interface NarrativeViewInfo { +export interface ChartViewInfo { name: string title: string chartConfigId: string @@ -281,7 +281,7 @@ export enum OwidGdocLinkType { Url = "url", Grapher = "grapher", Explorer = "explorer", - NarrativeChart = "narrative-chart", + ChartView = "chart-view", } export interface OwidGdocLinkJSON { diff --git a/packages/@ourworldindata/types/src/index.ts b/packages/@ourworldindata/types/src/index.ts index 9e94a05ee0..a8a3f80db5 100644 --- a/packages/@ourworldindata/types/src/index.ts +++ b/packages/@ourworldindata/types/src/index.ts @@ -332,7 +332,7 @@ export { type OwidGdocContent, type OwidGdocIndexItem, extractGdocIndexItem, - type NarrativeViewInfo, + type ChartViewInfo, } from "./gdocTypes/Gdoc.js" export { diff --git a/site/gdocs/AttachmentsContext.tsx b/site/gdocs/AttachmentsContext.tsx index a570a6f7b7..ec1b767fe1 100644 --- a/site/gdocs/AttachmentsContext.tsx +++ b/site/gdocs/AttachmentsContext.tsx @@ -9,7 +9,7 @@ import { LatestDataInsight, OwidGdocHomepageMetadata, DbEnrichedLatestWork, - NarrativeViewInfo, + ChartViewInfo, } from "@ourworldindata/types" export type Attachments = { @@ -23,7 +23,7 @@ export type Attachments = { latestDataInsights?: LatestDataInsight[] homepageMetadata?: OwidGdocHomepageMetadata latestWorkLinks?: DbEnrichedLatestWork[] - narrativeViewsInfo?: Record<string, NarrativeViewInfo> + linkedChartViews?: Record<string, ChartViewInfo> } export const AttachmentsContext = createContext<Attachments>({ @@ -36,5 +36,5 @@ export const AttachmentsContext = createContext<Attachments>({ latestDataInsights: [], homepageMetadata: {}, latestWorkLinks: [], - narrativeViewsInfo: {}, + linkedChartViews: {}, }) diff --git a/site/gdocs/OwidGdoc.tsx b/site/gdocs/OwidGdoc.tsx index f0f6137f51..8c3162a618 100644 --- a/site/gdocs/OwidGdoc.tsx +++ b/site/gdocs/OwidGdoc.tsx @@ -93,7 +93,7 @@ export function OwidGdoc({ latestDataInsights: get(props, "latestDataInsights", []), homepageMetadata: get(props, "homepageMetadata", {}), latestWorkLinks: get(props, "latestWorkLinks", []), - narrativeViewsInfo: get(props, "narrativeViewsInfo", {}), + linkedChartViews: get(props, "linkedChartViews", {}), }} > <DocumentContext.Provider value={{ isPreviewing }}> diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index c5b3fa6f3e..260be94b53 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -1,12 +1,13 @@ import { useContext, useRef } from "react" import { useEmbedChart } from "../../hooks.js" import { EnrichedBlockNarrativeChart } from "@ourworldindata/types" -import { useNarrativeViewsInfo } from "../utils.js" +import { useLinkedChartView } from "../utils.js" import cx from "classnames" import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" import { BlockErrorFallback } from "./BlockErrorBoundary.js" import SpanElements from "./SpanElements.js" import { DocumentContext } from "../DocumentContext.js" +import { GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR } from "@ourworldindata/grapher" export default function NarrativeChart({ d, @@ -20,7 +21,7 @@ export default function NarrativeChart({ const refChartContainer = useRef<HTMLDivElement>(null) useEmbedChart(0, refChartContainer) - const viewMetadata = useNarrativeViewsInfo(d.name) + const viewMetadata = useLinkedChartView(d.name) const { isPreviewing } = useContext(DocumentContext) @@ -30,8 +31,8 @@ export default function NarrativeChart({ <BlockErrorFallback className={className} error={{ - name: "Narrative view not found", - message: `Narrative view with name "${d.name}" couldn't be found.`, + name: "Narrative chart not found", + message: `Narrative chart with name "${d.name}" couldn't be found.`, }} /> ) @@ -51,12 +52,15 @@ export default function NarrativeChart({ <figure key={metadataStringified} className={cx(GRAPHER_PREVIEW_CLASS, "chart")} - data-grapher-view-config={metadataStringified} style={{ width: "100%", border: "0px none", height: d.height, }} + {...{ + [GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR]: + metadataStringified, + }} > {/* <a href={resolvedUrl} target="_blank" rel="noopener"> <GrapherImage slug={resolvedSlug} alt={d.title} /> diff --git a/site/gdocs/utils.ts b/site/gdocs/utils.ts index 78b10b195c..18675be683 100644 --- a/site/gdocs/utils.ts +++ b/site/gdocs/utils.ts @@ -148,9 +148,9 @@ export function useDonors(): string[] | undefined { return donors } -export const useNarrativeViewsInfo = (name: string) => { - const { narrativeViewsInfo } = useContext(AttachmentsContext) - return narrativeViewsInfo?.[name] +export const useLinkedChartView = (name: string) => { + const { linkedChartViews } = useContext(AttachmentsContext) + return linkedChartViews?.[name] } export function getShortPageCitation( diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index b9bb9cd93e..ad90ecb856 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -10,7 +10,7 @@ import { migrateSelectedEntityNamesParam, SelectionArray, migrateGrapherConfigToLatestVersion, - GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, } from "@ourworldindata/grapher" import { fetchText, @@ -22,7 +22,7 @@ import { MultiDimDataPageConfig, extractMultiDimChoicesFromQueryStr, fetchWithRetry, - NarrativeViewInfo, + ChartViewInfo, queryParamsToStr, } from "@ourworldindata/utils" import { action } from "mobx" @@ -46,7 +46,7 @@ import Bugsnag from "@bugsnag/js" import { embedDynamicCollectionGrapher } from "../collections/DynamicCollection.js" import { match } from "ts-pattern" -type EmbedType = "grapher" | "explorer" | "multiDim" | "grapherView" +type EmbedType = "grapher" | "explorer" | "multiDim" | "chartView" const figuresFromDOM = ( container: HTMLElement | Document = document, @@ -122,7 +122,7 @@ class MultiEmbedder { .concat( figuresFromDOM( container, - GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR ) ) @@ -271,12 +271,12 @@ class MultiEmbedder { embedUrl, }) } - async renderGrapherViewIntoFigure(figure: Element) { + async renderChartViewIntoFigure(figure: Element) { const viewConfigRaw = figure.getAttribute( - GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR ) if (!viewConfigRaw) return - const viewConfig: NarrativeViewInfo = JSON.parse(viewConfigRaw) + const viewConfig: ChartViewInfo = JSON.parse(viewConfigRaw) if (!viewConfig) return const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` @@ -302,16 +302,16 @@ class MultiEmbedder { EXPLORER_EMBEDDED_FIGURE_SELECTOR ) const isMultiDim = figure.hasAttribute("data-is-multi-dim") - const isGrapherView = figure.hasAttribute( - GRAPHER_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR + const isChartView = figure.hasAttribute( + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR ) const embedType: EmbedType = isExplorer ? "explorer" : isMultiDim ? "multiDim" - : isGrapherView - ? "grapherView" + : isChartView + ? "chartView" : "grapher" const hasPreview = isExplorer ? false : !!figure.querySelector("img") @@ -327,7 +327,7 @@ class MultiEmbedder { await match(embedType) .with("explorer", () => this.renderExplorerIntoFigure(figure)) .with("multiDim", () => this.renderMultiDimIntoFigure(figure)) - .with("grapherView", () => this.renderGrapherViewIntoFigure(figure)) + .with("chartView", () => this.renderChartViewIntoFigure(figure)) .with("grapher", () => this.renderGrapherIntoFigure(figure)) .exhaustive() } From 073fa23498623f2355077d41202435f4d63b8a50 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 19 Dec 2024 19:56:23 +0100 Subject: [PATCH 20/39] feat: correctly generate query params for chart view --- .../grapher/src/core/Grapher.tsx | 28 +++++++++++++++++++ site/multiembedder/MultiEmbedder.tsx | 8 ++---- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index 94407af5a9..7a0e65d538 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -112,6 +112,7 @@ import { GRAPHER_TAB_QUERY_PARAMS, GrapherTabOption, SeriesName, + ChartViewInfo, } from "@ourworldindata/types" import { BlankOwidTable, @@ -332,6 +333,11 @@ export interface GrapherProgrammaticInterface extends GrapherInterface { isEmbeddedInAnOwidPage?: boolean isEmbeddedInADataPage?: boolean + chartViewInfo?: Pick< + ChartViewInfo, + "parentChartSlug" | "queryParamsForParentChart" + > + manager?: GrapherManager instanceRef?: React.RefObject<Grapher> } @@ -506,6 +512,11 @@ export class Grapher isEmbeddedInAnOwidPage?: boolean = this.props.isEmbeddedInAnOwidPage isEmbeddedInADataPage?: boolean = this.props.isEmbeddedInADataPage + chartViewInfo?: Pick< + ChartViewInfo, + "parentChartSlug" | "queryParamsForParentChart" + > = undefined + selection = this.manager?.selection ?? new SelectionArray( @@ -3521,10 +3532,27 @@ export class Grapher return this.props.manager } + @computed get canonicalUrlIfIsChartView(): string | undefined { + if (!this.chartViewInfo) return undefined + + const { parentChartSlug, queryParamsForParentChart } = + this.chartViewInfo + + const combinedQueryParams = { + ...queryParamsForParentChart, + ...this.changedParams, + } + + return `${this.bakedGrapherURL}/${parentChartSlug}${queryParamsToStr( + combinedQueryParams + )}` + } + // Get the full url representing the canonical location of this grapher state @computed get canonicalUrl(): string | undefined { return ( this.manager?.canonicalUrl ?? + this.canonicalUrlIfIsChartView ?? (this.baseUrl ? this.baseUrl + this.queryStr : undefined) ) } diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index ad90ecb856..3c8d9a7691 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -281,17 +281,13 @@ class MultiEmbedder { const configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/by-uuid/${viewConfig.chartConfigId}.config.json` - const queryStr = queryParamsToStr(viewConfig.queryParamsForParentChart) - await this._renderGrapherComponentIntoFigure(figure, { configUrl, additionalConfig: { hideRelatedQuestion: true, - hideShareButton: true, // always hidden since the original chart would be shared, not the customized one + hideShareButton: true, hideExploreTheDataButton: false, - manager: { - canonicalUrl: `${BAKED_GRAPHER_URL}/${viewConfig.parentChartSlug}${queryStr}`, - }, + chartViewInfo: viewConfig, }, }) } From 0823bfed945bf3b1250f75ee286242b35047e602 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 19 Dec 2024 20:12:11 +0100 Subject: [PATCH 21/39] style: remove unused import --- site/multiembedder/MultiEmbedder.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx index 3c8d9a7691..5d2bcec088 100644 --- a/site/multiembedder/MultiEmbedder.tsx +++ b/site/multiembedder/MultiEmbedder.tsx @@ -23,7 +23,6 @@ import { extractMultiDimChoicesFromQueryStr, fetchWithRetry, ChartViewInfo, - queryParamsToStr, } from "@ourworldindata/utils" import { action } from "mobx" import ReactDOM from "react-dom" From 2ef0506e5382352ab2d72f50e89ae9e1bd631959 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 19 Dec 2024 20:52:18 +0100 Subject: [PATCH 22/39] enhance: enable narrative charts on staging server --- adminSiteClient/ChartViewEditor.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/adminSiteClient/ChartViewEditor.ts b/adminSiteClient/ChartViewEditor.ts index 982f967b83..578ec7028c 100644 --- a/adminSiteClient/ChartViewEditor.ts +++ b/adminSiteClient/ChartViewEditor.ts @@ -5,7 +5,7 @@ import { References, type EditorTab, } from "./AbstractChartEditor.js" -import { ENV } from "../settings/clientSettings.js" +import { BAKED_BASE_URL, ENV } from "../settings/clientSettings.js" import { CHART_VIEW_PROPS_TO_OMIT, CHART_VIEW_PROPS_TO_PERSIST, @@ -16,7 +16,8 @@ import { diffGrapherConfigs, omit, pick } from "@ourworldindata/utils" // Don't yet show chart views in the admin interface // This is low-stakes - if it shows up anyhow (e.g. on staging servers), it's not a big deal. // TODO: Remove this flag once we're launching this feature -export const chartViewsFeatureEnabled = ENV === "development" +export const chartViewsFeatureEnabled = + ENV === "development" || BAKED_BASE_URL.includes("narrative-") export interface Chart { id: number From e272937b46a9884dfee2b0314712ad1132e890ac Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 19 Dec 2024 21:20:39 +0100 Subject: [PATCH 23/39] enhance: show static preview of chart view --- adminSiteClient/ChartViewIndexPage.tsx | 4 ++-- site/gdocs/components/NarrativeChart.tsx | 29 ++++++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/adminSiteClient/ChartViewIndexPage.tsx b/adminSiteClient/ChartViewIndexPage.tsx index 3bc945314f..c0b5d64d3b 100644 --- a/adminSiteClient/ChartViewIndexPage.tsx +++ b/adminSiteClient/ChartViewIndexPage.tsx @@ -7,7 +7,7 @@ import { AdminAppContext } from "./AdminAppContext.js" import { Timeago } from "./Forms.js" import { ColumnsType } from "antd/es/table/InternalTable.js" import { ApiChartViewOverview } from "../adminShared/AdminTypes.js" -import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js" +import { GRAPHER_DYNAMIC_THUMBNAIL_URL } from "../settings/clientSettings.js" import { Link } from "./Link.js" import { buildSearchWordsFromSearchString, @@ -28,7 +28,7 @@ function createColumns( width: 200, render: (chartConfigId) => ( <img - src={`${BAKED_GRAPHER_URL}/by-uuid/${chartConfigId}.svg`} + src={`${GRAPHER_DYNAMIC_THUMBNAIL_URL}/by-uuid/${chartConfigId}.svg`} style={{ maxWidth: 200, maxHeight: 200 }} /> ), diff --git a/site/gdocs/components/NarrativeChart.tsx b/site/gdocs/components/NarrativeChart.tsx index 260be94b53..d2e5336d9b 100644 --- a/site/gdocs/components/NarrativeChart.tsx +++ b/site/gdocs/components/NarrativeChart.tsx @@ -7,7 +7,17 @@ import { GRAPHER_PREVIEW_CLASS } from "../../SiteConstants.js" import { BlockErrorFallback } from "./BlockErrorBoundary.js" import SpanElements from "./SpanElements.js" import { DocumentContext } from "../DocumentContext.js" -import { GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR } from "@ourworldindata/grapher" +import { + DEFAULT_GRAPHER_HEIGHT, + DEFAULT_GRAPHER_WIDTH, + GRAPHER_CHART_VIEW_EMBEDDED_FIGURE_CONFIG_ATTR, +} from "@ourworldindata/grapher" +import { + BAKED_GRAPHER_URL, + GRAPHER_DYNAMIC_THUMBNAIL_URL, +} from "../../../settings/clientSettings.js" +import { queryParamsToStr } from "@ourworldindata/utils" +import InteractionNotice from "../../InteractionNotice.js" export default function NarrativeChart({ d, @@ -41,6 +51,10 @@ export default function NarrativeChart({ const metadataStringified = JSON.stringify(viewMetadata) + const resolvedUrl = `${BAKED_GRAPHER_URL}/${viewMetadata.parentChartSlug}${queryParamsToStr( + viewMetadata.queryParamsForParentChart + )}` + return ( <div className={cx(d.position, className, { @@ -62,10 +76,17 @@ export default function NarrativeChart({ metadataStringified, }} > - {/* <a href={resolvedUrl} target="_blank" rel="noopener"> - <GrapherImage slug={resolvedSlug} alt={d.title} /> + <a href={resolvedUrl} target="_blank" rel="noopener"> + <img + src={`${GRAPHER_DYNAMIC_THUMBNAIL_URL}/by-uuid/${viewMetadata.chartConfigId}.svg`} + alt={viewMetadata.title} + width={DEFAULT_GRAPHER_WIDTH} + height={DEFAULT_GRAPHER_HEIGHT} + loading="lazy" + data-no-lightbox + /> <InteractionNotice /> - </a> */} + </a> </figure> {d.caption ? ( <figcaption> From 4c6ed348eed6b9f20e3eaac3e8beb882d75cb543 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 19 Dec 2024 23:29:24 +0100 Subject: [PATCH 24/39] enhance: make "explore the data" button blue --- .../grapher/src/controls/ActionButtons.scss | 27 +++++++++++++------ .../grapher/src/controls/ActionButtons.tsx | 2 +- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss b/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss index d3f3a464ae..cc0b23bd31 100644 --- a/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss +++ b/packages/@ourworldindata/grapher/src/controls/ActionButtons.scss @@ -26,10 +26,20 @@ $paddingX: 12px; // keep in sync with PADDING_X } } -.ActionButton { - $light-fill: $gray-10; - $hover-fill: $gray-20; - $active-fill: $blue-20; +div.ActionButton { + --light-fill: #{$gray-10}; + --hover-fill: #{$gray-20}; + --active-fill: #{$blue-20}; + --text-color: #{$dark-text}; + + &.ActionButton--exploreData { + --light-fill: #{$blue-20}; + --hover-fill: #{$blue-20}; + --active-fill: #{$blue-10}; + --text-color: #{$blue-90}; + + --hover-decoration: underline; + } height: 100%; border-radius: 4px; @@ -43,12 +53,12 @@ $paddingX: 12px; // keep in sync with PADDING_X height: 100%; width: 100%; cursor: pointer; - color: $dark-text; + color: var(--text-color); font-size: 13px; font-weight: 500; padding: 0 $paddingX; border-radius: inherit; - background-color: $light-fill; + background-color: var(--light-fill); position: relative; letter-spacing: 0.01em; @@ -62,13 +72,14 @@ $paddingX: 12px; // keep in sync with PADDING_X } &:hover { - background-color: $hover-fill; + background-color: var(--hover-fill); + text-decoration: var(--hover-decoration); } &:active, &.active { color: $active-text; - background-color: $active-fill; + background-color: var(--active-fill); } } diff --git a/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx b/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx index dadc693d30..13e0e2f164 100644 --- a/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx +++ b/packages/@ourworldindata/grapher/src/controls/ActionButtons.tsx @@ -305,7 +305,7 @@ export class ActionButtons extends React.Component<{ {this.hasExploreTheDataButton && ( <li style={{ width: this.exploreTheDataButtonWidth }}> <div - className="ActionButton" + className="ActionButton ActionButton--exploreData" style={{ width: "100%" }} > <a From a7d896eab80e8db509fa71b7d69c5b2da8f6fcc9 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 9 Jan 2025 11:37:41 +0100 Subject: [PATCH 25/39] feat: use nice modal for entering narrative chart name, gracefully handle duplicate names --- adminSiteClient/ChartEditor.ts | 32 ++++---- adminSiteClient/SaveButtons.tsx | 129 ++++++++++++++++++++++++++++---- adminSiteServer/apiRouter.ts | 12 +++ 3 files changed, 141 insertions(+), 32 deletions(-) diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index 25b10df424..c059e75ebc 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -26,6 +26,8 @@ import { References, } from "./AbstractChartEditor.js" import { Admin } from "./Admin.js" +import { Form, Input, Modal } from "antd" +import React, { useState } from "react" export interface Log { userId: number @@ -200,22 +202,17 @@ export class ChartEditor extends AbstractChartEditor<ChartEditorManager> { ) } - async saveAsChartView(): Promise<void> { - const { patchConfig, grapher } = this - - const chartJson = omit(patchConfig, CHART_VIEW_PROPS_TO_OMIT) - + openNarrativeChartNameModal(): void { + const { grapher } = this const suggestedName = grapher.title ? slugify(grapher.title) : undefined + } - const name = prompt( - "Please enter a programmatic name for the narrative chart. Note that this name cannot be changed later.", - suggestedName - ) - - if (name === null) return + async saveAsChartView( + name: string + ): Promise<{ success: boolean; errorMsg?: string }> { + const { patchConfig, grapher } = this - // Need to open intermediary tab before AJAX to avoid popup blockers - const w = window.open("/", "_blank") as Window + const chartJson = omit(patchConfig, CHART_VIEW_PROPS_TO_OMIT) const body = { name, @@ -228,11 +225,14 @@ export class ChartEditor extends AbstractChartEditor<ChartEditorManager> { body, "POST" ) - - if (json.success) - w.location.assign( + if (json.success) { + window.open( this.manager.admin.url(`chartViews/${json.chartViewId}/edit`) ) + return { success: true } + } else { + return { success: false, errorMsg: json.errorMsg } + } } publishGrapher(): void { diff --git a/adminSiteClient/SaveButtons.tsx b/adminSiteClient/SaveButtons.tsx index 3166add379..bdae5d157f 100644 --- a/adminSiteClient/SaveButtons.tsx +++ b/adminSiteClient/SaveButtons.tsx @@ -1,8 +1,8 @@ -import { Component } from "react" +import { Component, useEffect, useMemo, useRef, useState } from "react" import { ChartEditor, isChartEditorInstance } from "./ChartEditor.js" -import { action, computed } from "mobx" +import { action, computed, observable } from "mobx" import { observer } from "mobx-react" -import { excludeUndefined, omit } from "@ourworldindata/utils" +import { excludeUndefined, omit, slugify } from "@ourworldindata/utils" import { IndicatorChartEditor, isIndicatorChartEditorInstance, @@ -17,6 +17,7 @@ import { chartViewsFeatureEnabled, isChartViewEditorInstance, } from "./ChartViewEditor.js" +import { Form, Input, InputRef, Modal, Spin } from "antd" @observer export class SaveButtons<Editor extends AbstractChartEditor> extends Component<{ @@ -61,10 +62,6 @@ class SaveButtonsForChart extends Component<{ void this.props.editor.saveAsNewGrapher() } - @action.bound onSaveAsChartView() { - void this.props.editor.saveAsChartView() - } - @action.bound onPublishToggle() { if (this.props.editor.grapher.isPublished) this.props.editor.unpublishGrapher() @@ -79,6 +76,29 @@ class SaveButtonsForChart extends Component<{ ]) } + @computed get initialNarrativeChartName(): string { + return slugify(this.props.editor.grapher.title ?? "") + } + + @observable narrativeChartNameModalOpen: + | "open" + | "open-loading" + | "closed" = "closed" + @observable narrativeChartNameModalError: string | undefined = undefined + + @action.bound async onSubmitNarrativeChartButton(name: string) { + const { editor } = this.props + + this.narrativeChartNameModalOpen = "open-loading" + const res = await editor.saveAsChartView(name) + if (res.success) { + this.narrativeChartNameModalOpen = "closed" + } else { + this.narrativeChartNameModalOpen = "open" + this.narrativeChartNameModalError = res.errorMsg + } + } + render() { const { editingErrors } = this const { editor } = this.props @@ -117,15 +137,30 @@ class SaveButtonsForChart extends Component<{ </button> </div> {chartViewsFeatureEnabled && ( - <div className="mt-2"> - <button - className="btn btn-primary" - onClick={this.onSaveAsChartView} - disabled={isSavingDisabled} - > - Save as narrative chart - </button> - </div> + <> + <div className="mt-2"> + <button + className="btn btn-primary" + onClick={() => { + this.narrativeChartNameModalOpen = "open" + this.narrativeChartNameModalError = + undefined + }} + disabled={isSavingDisabled} + > + Save as narrative chart + </button> + </div> + <NarrativeChartNameModal + open={this.narrativeChartNameModalOpen} + initialName={this.initialNarrativeChartName} + errorMsg={this.narrativeChartNameModalError} + onSubmit={this.onSubmitNarrativeChartButton} + onCancel={() => + (this.narrativeChartNameModalOpen = "closed") + } + /> + </> )} {editingErrors.map((error, i) => ( <div key={i} className="alert alert-danger mt-2"> @@ -238,3 +273,65 @@ class SaveButtonsForChartView extends Component<{ ) } } + +const NarrativeChartNameModal = (props: { + initialName: string + open: "open" | "open-loading" | "closed" + errorMsg?: string + onSubmit: (name: string) => void + onCancel?: () => void +}) => { + const [name, setName] = useState<string>(props.initialName) + const inputField = useRef<InputRef>(null) + const isLoading = useMemo(() => props.open === "open-loading", [props.open]) + const isOpen = useMemo(() => props.open !== "closed", [props.open]) + + useEffect(() => setName(props.initialName), [props.initialName]) + + useEffect(() => { + if (isOpen) { + inputField.current?.focus({ cursor: "all" }) + } + }, [isOpen]) + + return ( + <Modal + title="Save as narrative chart" + open={isOpen} + onOk={() => props.onSubmit(name)} + onCancel={props.onCancel} + onClose={props.onCancel} + okButtonProps={{ disabled: !name || isLoading }} + cancelButtonProps={{ disabled: isLoading }} + > + <div> + <p> + This will create a new narrative chart that is linked to + this chart. Any currently pending changes will be applied to + the narrative chart. + </p> + <p> + Please enter a programmatic name for the narrative chart.{" "} + <i>Note that this name cannot be changed later.</i> + </p> + <Form.Item label="Name"> + <Input + ref={inputField} + onChange={(e) => setName(e.target.value)} + value={name} + disabled={isLoading} + /> + </Form.Item> + {isLoading && <Spin />} + {props.errorMsg && ( + <div + className="alert alert-danger" + style={{ whiteSpace: "pre-wrap" }} + > + {props.errorMsg} + </div> + )} + </div> + </Modal> + ) +} diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 96278efa0f..3b001634de 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -3762,6 +3762,18 @@ postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { throw new JsonError("Invalid request", 400) } + const chartViewWithName = await trx + .table(ChartViewsTableName) + .where({ name }) + .first() + + if (chartViewWithName) { + return { + success: false, + errorMsg: `Narrative chart with name "${name}" already exists`, + } + } + const { patchConfig, fullConfig, queryParams } = await createPatchConfigAndQueryParamsForChartView( trx, From b943f296d4930c935a00a610c91d5e4966832a6c Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 9 Jan 2025 12:33:33 +0100 Subject: [PATCH 26/39] style: fix eslint warnings --- adminSiteClient/ChartEditor.ts | 8 --- adminSiteClient/NarrativeChartNameModal.tsx | 64 ++++++++++++++++++++ adminSiteClient/SaveButtons.tsx | 66 +-------------------- 3 files changed, 66 insertions(+), 72 deletions(-) create mode 100644 adminSiteClient/NarrativeChartNameModal.tsx diff --git a/adminSiteClient/ChartEditor.ts b/adminSiteClient/ChartEditor.ts index c059e75ebc..839efe0794 100644 --- a/adminSiteClient/ChartEditor.ts +++ b/adminSiteClient/ChartEditor.ts @@ -13,7 +13,6 @@ import { getParentVariableIdFromChartConfig, mergeGrapherConfigs, isEmpty, - slugify, omit, CHART_VIEW_PROPS_TO_OMIT, } from "@ourworldindata/utils" @@ -26,8 +25,6 @@ import { References, } from "./AbstractChartEditor.js" import { Admin } from "./Admin.js" -import { Form, Input, Modal } from "antd" -import React, { useState } from "react" export interface Log { userId: number @@ -202,11 +199,6 @@ export class ChartEditor extends AbstractChartEditor<ChartEditorManager> { ) } - openNarrativeChartNameModal(): void { - const { grapher } = this - const suggestedName = grapher.title ? slugify(grapher.title) : undefined - } - async saveAsChartView( name: string ): Promise<{ success: boolean; errorMsg?: string }> { diff --git a/adminSiteClient/NarrativeChartNameModal.tsx b/adminSiteClient/NarrativeChartNameModal.tsx new file mode 100644 index 0000000000..81d61c02bd --- /dev/null +++ b/adminSiteClient/NarrativeChartNameModal.tsx @@ -0,0 +1,64 @@ +import { useEffect, useMemo, useRef, useState } from "react" +import { Form, Input, InputRef, Modal, Spin } from "antd" + +export const NarrativeChartNameModal = (props: { + initialName: string + open: "open" | "open-loading" | "closed" + errorMsg?: string + onSubmit: (name: string) => void + onCancel?: () => void +}) => { + const [name, setName] = useState<string>(props.initialName) + const inputField = useRef<InputRef>(null) + const isLoading = useMemo(() => props.open === "open-loading", [props.open]) + const isOpen = useMemo(() => props.open !== "closed", [props.open]) + + useEffect(() => setName(props.initialName), [props.initialName]) + + useEffect(() => { + if (isOpen) { + inputField.current?.focus({ cursor: "all" }) + } + }, [isOpen]) + + return ( + <Modal + title="Save as narrative chart" + open={isOpen} + onOk={() => props.onSubmit(name)} + onCancel={props.onCancel} + onClose={props.onCancel} + okButtonProps={{ disabled: !name || isLoading }} + cancelButtonProps={{ disabled: isLoading }} + > + <div> + <p> + This will create a new narrative chart that is linked to + this chart. Any currently pending changes will be applied to + the narrative chart. + </p> + <p> + Please enter a programmatic name for the narrative chart.{" "} + <i>Note that this name cannot be changed later.</i> + </p> + <Form.Item label="Name"> + <Input + ref={inputField} + onChange={(e) => setName(e.target.value)} + value={name} + disabled={isLoading} + /> + </Form.Item> + {isLoading && <Spin />} + {props.errorMsg && ( + <div + className="alert alert-danger" + style={{ whiteSpace: "pre-wrap" }} + > + {props.errorMsg} + </div> + )} + </div> + </Modal> + ) +} diff --git a/adminSiteClient/SaveButtons.tsx b/adminSiteClient/SaveButtons.tsx index bdae5d157f..0bdbafca27 100644 --- a/adminSiteClient/SaveButtons.tsx +++ b/adminSiteClient/SaveButtons.tsx @@ -1,4 +1,4 @@ -import { Component, useEffect, useMemo, useRef, useState } from "react" +import { Component } from "react" import { ChartEditor, isChartEditorInstance } from "./ChartEditor.js" import { action, computed, observable } from "mobx" import { observer } from "mobx-react" @@ -17,7 +17,7 @@ import { chartViewsFeatureEnabled, isChartViewEditorInstance, } from "./ChartViewEditor.js" -import { Form, Input, InputRef, Modal, Spin } from "antd" +import { NarrativeChartNameModal } from "./NarrativeChartNameModal.js" @observer export class SaveButtons<Editor extends AbstractChartEditor> extends Component<{ @@ -273,65 +273,3 @@ class SaveButtonsForChartView extends Component<{ ) } } - -const NarrativeChartNameModal = (props: { - initialName: string - open: "open" | "open-loading" | "closed" - errorMsg?: string - onSubmit: (name: string) => void - onCancel?: () => void -}) => { - const [name, setName] = useState<string>(props.initialName) - const inputField = useRef<InputRef>(null) - const isLoading = useMemo(() => props.open === "open-loading", [props.open]) - const isOpen = useMemo(() => props.open !== "closed", [props.open]) - - useEffect(() => setName(props.initialName), [props.initialName]) - - useEffect(() => { - if (isOpen) { - inputField.current?.focus({ cursor: "all" }) - } - }, [isOpen]) - - return ( - <Modal - title="Save as narrative chart" - open={isOpen} - onOk={() => props.onSubmit(name)} - onCancel={props.onCancel} - onClose={props.onCancel} - okButtonProps={{ disabled: !name || isLoading }} - cancelButtonProps={{ disabled: isLoading }} - > - <div> - <p> - This will create a new narrative chart that is linked to - this chart. Any currently pending changes will be applied to - the narrative chart. - </p> - <p> - Please enter a programmatic name for the narrative chart.{" "} - <i>Note that this name cannot be changed later.</i> - </p> - <Form.Item label="Name"> - <Input - ref={inputField} - onChange={(e) => setName(e.target.value)} - value={name} - disabled={isLoading} - /> - </Form.Item> - {isLoading && <Spin />} - {props.errorMsg && ( - <div - className="alert alert-danger" - style={{ whiteSpace: "pre-wrap" }} - > - {props.errorMsg} - </div> - )} - </div> - </Modal> - ) -} From 8a2a6114389eae9a14cd63d37b94b07faf0b17d1 Mon Sep 17 00:00:00 2001 From: Sophia Mersmann <sophia.mersmann1@gmail.com> Date: Thu, 9 Jan 2025 11:13:30 +0100 Subject: [PATCH 27/39] =?UTF-8?q?=F0=9F=90=9D=20(svg=20tester)=20format=20?= =?UTF-8?q?reference=20svgs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devTools/svgTester/update-configs.sh | 52 ++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/devTools/svgTester/update-configs.sh b/devTools/svgTester/update-configs.sh index 6604fc59dc..2bc3cd3c5f 100755 --- a/devTools/svgTester/update-configs.sh +++ b/devTools/svgTester/update-configs.sh @@ -20,29 +20,51 @@ Make sure to run \`make refresh\` and \`make refresh.pageviews\` before running main() { echo "=> Resetting owid-grapher-svgs to origin/master" - cd $SVGS_REPO\ - && git fetch\ - && git checkout -f master\ - && git reset --hard origin/master\ - && git clean -fd\ + cd $SVGS_REPO \ + && git fetch \ + && git checkout -f master \ + && git reset --hard origin/master \ + && git clean -fdx \ && cd - - echo "=> Removing existing configs and reference svgs" - rm -rf $CONFIGS_DIR $REFERENCES_DIR $ALL_VIEWS_DIR - echo "=> Dumping new configs and data" + rm -rf $CONFIGS_DIR node itsJustJavascript/devTools/svgTester/dump-data.js -o $CONFIGS_DIR node itsJustJavascript/devTools/svgTester/dump-chart-ids.js -o $CHART_IDS_FILE - echo "=> Generating reference SVGs" - node itsJustJavascript/devTools/svgTester/export-graphs.js\ - -i $CONFIGS_DIR\ + echo "=> Committing new configs and chart ids" + cd $SVGS_REPO \ + && git add --all \ + && git commit -m "chore: update configs and chart ids" \ + && cd - + + echo "=> Generating reference SVGs (default views)" + rm -rf $REFERENCES_DIR + node itsJustJavascript/devTools/svgTester/export-graphs.js \ + -i $CONFIGS_DIR \ -o $REFERENCES_DIR - node itsJustJavascript/devTools/svgTester/export-graphs.js\ - -i $CONFIGS_DIR\ - -o $ALL_VIEWS_SVG_DIR\ - -f $CHART_IDS_FILE\ + yarn prettier --write --parser html $REFERENCES_DIR + + echo "=> Committing reference SVGs (default views)" + cd $SVGS_REPO \ + && git add --all \ + && git commit -m 'chore: update reference svgs (default views)' \ + && cd - + + echo "=> Generating reference SVGs (all views)" + rm -rf $ALL_VIEWS_DIR + node itsJustJavascript/devTools/svgTester/export-graphs.js \ + -i $CONFIGS_DIR \ + -o $ALL_VIEWS_SVG_DIR \ + -f $CHART_IDS_FILE \ --all-views + yarn prettier --write --parser html $ALL_VIEWS_SVG_DIR + + echo "=> Committing reference SVGs (all views)" + cd $SVGS_REPO \ + && git add --all \ + && git commit -m 'chore: update reference svgs (all views)' \ + && cd - } # show help From d69d129935e65e8ccf4dc7b4558fbd71b583a014 Mon Sep 17 00:00:00 2001 From: Marcel Gerber <mgerber59@gmail.com> Date: Thu, 9 Jan 2025 13:42:35 +0100 Subject: [PATCH 28/39] enhance(admin): don't show slug for narrative charts --- adminSiteClient/EditorTextTab.tsx | 33 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/adminSiteClient/EditorTextTab.tsx b/adminSiteClient/EditorTextTab.tsx index b259827bdd..a8106b6e96 100644 --- a/adminSiteClient/EditorTextTab.tsx +++ b/adminSiteClient/EditorTextTab.tsx @@ -19,6 +19,7 @@ import { } from "./Forms.js" import { AbstractChartEditor } from "./AbstractChartEditor.js" import { ErrorMessages } from "./ChartEditorTypes.js" +import { isChartViewEditorInstance } from "./ChartViewEditor.js" @observer export class EditorTextTab< @@ -74,6 +75,10 @@ export class EditorTextTab< return this.props.errorMessages } + @computed get showChartSlug() { + return !isChartViewEditorInstance(this.props.editor) + } + @computed get showAnyAnnotationFieldInTitleToggle() { const { features } = this.props.editor return ( @@ -139,19 +144,21 @@ export class EditorTextTab< /> )} {this.showAnyAnnotationFieldInTitleToggle && <hr />} - <AutoTextField - label="/grapher" - value={grapher.displaySlug} - onValue={this.onSlug} - isAuto={grapher.slug === undefined} - onToggleAuto={() => - (grapher.slug = - grapher.slug === undefined - ? grapher.displaySlug - : undefined) - } - helpText="Human-friendly URL for this chart" - /> + {this.showChartSlug && ( + <AutoTextField + label="/grapher" + value={grapher.displaySlug} + onValue={this.onSlug} + isAuto={grapher.slug === undefined} + onToggleAuto={() => + (grapher.slug = + grapher.slug === undefined + ? grapher.displaySlug + : undefined) + } + helpText="Human-friendly URL for this chart" + /> + )} <BindAutoStringExt label="Subtitle" readFn={(grapher) => grapher.currentSubtitle} From c10537fda10523c5947c3276111693d5fd9a0740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= <martin.racak@ourworldindata.org> Date: Wed, 8 Jan 2025 10:39:39 +0100 Subject: [PATCH 29/39] Stop adding Max as an extra author --- adminSiteServer/apiRouter.ts | 2 +- db/migrateWpPostsToArchieMl.ts | 11 +---------- .../@ourworldindata/utils/src/metadataHelpers.ts | 5 ----- site/Byline.tsx | 3 --- site/CitationMeta.tsx | 7 +------ site/LongFormPage.tsx | 3 --- site/clientFormatting.tsx | 5 ----- site/formatting.test.ts | 14 +++++++------- site/formatting.tsx | 6 +----- 9 files changed, 11 insertions(+), 45 deletions(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 5cf0e042bc..7937994feb 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -3467,7 +3467,7 @@ getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { > type GdocRecord = Pick<DbRawPostGdoc, "id" | "publishedAt"> - const author = req.query.author || "Max Roser" + const author = req.query.author const gdocs = await db.knexRaw<GdocRecord>( trx, `-- sql diff --git a/db/migrateWpPostsToArchieMl.ts b/db/migrateWpPostsToArchieMl.ts index f9c35c0e84..a0938d6c54 100644 --- a/db/migrateWpPostsToArchieMl.ts +++ b/db/migrateWpPostsToArchieMl.ts @@ -20,7 +20,7 @@ import { adjustHeadingLevels, findMinimumHeadingLevel, } from "./model/Gdoc/htmlToEnriched.js" -import { getPostRelatedCharts, isPostSlugCitable } from "./model/Post.js" +import { getPostRelatedCharts } from "./model/Post.js" import { enrichedBlocksToMarkdown } from "./model/Gdoc/enrichedToMarkdown.js" // slugs from all the linear entries we want to migrate from @edomt @@ -131,15 +131,6 @@ const migrate = async (trx: db.KnexReadWriteTransaction): Promise<void> => { relatedCharts = await getPostRelatedCharts(trx, post.id) } - const shouldIncludeMaxAsAuthor = isPostSlugCitable(post.slug) - if ( - shouldIncludeMaxAsAuthor && - post.authors && - !post.authors.includes("Max Roser") - ) { - post.authors.push("Max Roser") - } - // We don't get the first and last nodes if they are comments. // This can cause issues with the wp:components so here we wrap // everything in a div diff --git a/packages/@ourworldindata/utils/src/metadataHelpers.ts b/packages/@ourworldindata/utils/src/metadataHelpers.ts index 6a99a3d3d2..9fcf200dd4 100644 --- a/packages/@ourworldindata/utils/src/metadataHelpers.ts +++ b/packages/@ourworldindata/utils/src/metadataHelpers.ts @@ -73,16 +73,11 @@ export const getETLPathComponents = (path: string): ETLPathComponents => { export const formatAuthors = ({ authors, - requireMax, forBibtex, }: { authors: string[] - requireMax?: boolean forBibtex?: boolean }): string => { - if (requireMax && !authors.includes("Max Roser")) - authors = [...authors, "Max Roser"] - let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ") if (authorsText.length === 0) authorsText = authors[0] else authorsText += ` and ${last(authors)}` diff --git a/site/Byline.tsx b/site/Byline.tsx index 0042eaff21..738f35769f 100644 --- a/site/Byline.tsx +++ b/site/Byline.tsx @@ -2,11 +2,9 @@ import { formatAuthors } from "./clientFormatting.js" export const Byline = ({ authors, - withMax, override, }: { authors: string[] - withMax: boolean override?: string }) => { return ( @@ -20,7 +18,6 @@ export const Byline = ({ ) : ( <a href="/team">{`by ${formatAuthors({ authors, - requireMax: withMax, })}`}</a> )} </div> diff --git a/site/CitationMeta.tsx b/site/CitationMeta.tsx index c312f4db6e..6e5c9a4600 100644 --- a/site/CitationMeta.tsx +++ b/site/CitationMeta.tsx @@ -8,12 +8,7 @@ export const CitationMeta = (props: { date: Date canonicalUrl: string }) => { - const { title, date, canonicalUrl } = props - let { authors } = props - - if (authors.indexOf("Max Roser") === -1) - authors = authors.concat(["Max Roser"]) - + const { authors, title, date, canonicalUrl } = props return ( <Fragment> <meta name="citation_title" content={title} /> diff --git a/site/LongFormPage.tsx b/site/LongFormPage.tsx index 63f8f0e0ba..8bbcb5e2bd 100644 --- a/site/LongFormPage.tsx +++ b/site/LongFormPage.tsx @@ -95,13 +95,11 @@ export const LongFormPage = (props: { const citationText = `${formatAuthors({ authors: citationAuthors, - requireMax: true, })} (${citationPublishedYear}) - "${citationTitle}". Published online at OurWorldinData.org. Retrieved from: '${citationCanonicalUrl}' [Online Resource]` const bibtex = `@article{owid${citationSlug.replace(/-/g, "")}, author = {${formatAuthors({ authors: citationAuthors, - requireMax: true, forBibtex: true, })}}, title = {${citationTitle}}, @@ -169,7 +167,6 @@ export const LongFormPage = (props: { {!formattingOptions.hideAuthors && ( <Byline authors={post.authors} - withMax={withCitation} override={post.byline} /> )} diff --git a/site/clientFormatting.tsx b/site/clientFormatting.tsx index d6fb816761..71596ab8e6 100644 --- a/site/clientFormatting.tsx +++ b/site/clientFormatting.tsx @@ -2,16 +2,11 @@ import { last } from "@ourworldindata/utils" export const formatAuthors = ({ authors, - requireMax, forBibtex, }: { authors: string[] - requireMax?: boolean forBibtex?: boolean }) => { - if (requireMax && !authors.includes("Max Roser")) - authors = [...authors, "Max Roser"] - let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ") if (authorsText.length === 0) authorsText = authors[0] else authorsText += ` and ${last(authors)}` diff --git a/site/formatting.test.ts b/site/formatting.test.ts index 61848ac98b..ae7e7e50b9 100644 --- a/site/formatting.test.ts +++ b/site/formatting.test.ts @@ -162,17 +162,17 @@ describe(formatAuthors, () => { "Author 1, Author 2 and Author 3" ) - expect(formatAuthors({ authors, requireMax: true })).toEqual( - "Author 1, Author 2, Author 3 and Max Roser" - ) - expect(formatAuthors({ authors: ["Author 1"] })).toEqual("Author 1") expect(formatAuthors({ authors: ["Author 1", "Author 2"] })).toEqual( "Author 1 and Author 2" ) - expect( - formatAuthors({ authors, requireMax: true, forBibtex: true }) - ).toEqual("Author 1 and Author 2 and Author 3 and Max Roser") + expect(formatAuthors({ authors, forBibtex: true })).toEqual( + "Author 1 and Author 2 and Author 3" + ) + + expect(formatAuthors({ authors, forBibtex: false })).toEqual( + "Author 1, Author 2 and Author 3" + ) }) }) diff --git a/site/formatting.tsx b/site/formatting.tsx index 8123e7ca8b..7c10b9856d 100644 --- a/site/formatting.tsx +++ b/site/formatting.tsx @@ -336,11 +336,7 @@ const addPostHeader = (cheerioEl: CheerioStatic, post: FormattedPost) => { ReactDOMServer.renderToStaticMarkup( <div className="article-meta"> {post.excerpt && <div className="excerpt">{post.excerpt}</div>} - <Byline - authors={post.authors} - withMax={false} - override={post.byline} - /> + <Byline authors={post.authors} override={post.byline} /> <div className="published-updated"> <time>{publishedDate}</time> From 34fb547c398218a4981bbb0bdc5235930bfb2a02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= <martin.racak@ourworldindata.org> Date: Wed, 8 Jan 2025 11:00:11 +0100 Subject: [PATCH 30/39] Fix date in CitationMeta It should be the same as we display to the reader in the post, which is the publication date. --- site/gdocs/OwidGdocPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/gdocs/OwidGdocPage.tsx b/site/gdocs/OwidGdocPage.tsx index 9f4525b363..ed80dd3699 100644 --- a/site/gdocs/OwidGdocPage.tsx +++ b/site/gdocs/OwidGdocPage.tsx @@ -84,7 +84,7 @@ export default function OwidGdocPage({ debug?: boolean isPreviewing?: boolean }) { - const { content, createdAt, updatedAt } = gdoc + const { content, createdAt, publishedAt } = gdoc const pageDesc = getPageDesc(gdoc) const featuredImageFilename = getFeaturedImageFilename(gdoc) @@ -125,7 +125,7 @@ export default function OwidGdocPage({ <CitationMeta title={content.title || ""} authors={content.authors} - date={updatedAt || createdAt} + date={publishedAt || createdAt} canonicalUrl={canonicalUrl} /> )} From 01013524c1603399596121e310694ef79bbedf45 Mon Sep 17 00:00:00 2001 From: Sophia Mersmann <sophia.mersmann1@gmail.com> Date: Thu, 9 Jan 2025 15:28:24 +0100 Subject: [PATCH 31/39] =?UTF-8?q?=F0=9F=94=A8=20stop=20prettifying=20SVGs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- devTools/svgTester/update-configs.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/devTools/svgTester/update-configs.sh b/devTools/svgTester/update-configs.sh index 2bc3cd3c5f..95298b748d 100755 --- a/devTools/svgTester/update-configs.sh +++ b/devTools/svgTester/update-configs.sh @@ -43,7 +43,6 @@ main() { node itsJustJavascript/devTools/svgTester/export-graphs.js \ -i $CONFIGS_DIR \ -o $REFERENCES_DIR - yarn prettier --write --parser html $REFERENCES_DIR echo "=> Committing reference SVGs (default views)" cd $SVGS_REPO \ @@ -58,7 +57,6 @@ main() { -o $ALL_VIEWS_SVG_DIR \ -f $CHART_IDS_FILE \ --all-views - yarn prettier --write --parser html $ALL_VIEWS_SVG_DIR echo "=> Committing reference SVGs (all views)" cd $SVGS_REPO \ From 98d7908cf164c22d86ff77387553432e2fa05eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= <martin.racak@ourworldindata.org> Date: Thu, 9 Jan 2025 15:47:16 +0100 Subject: [PATCH 32/39] Add formatting for numbers in data catalog --- site/DataCatalog/DataCatalog.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/site/DataCatalog/DataCatalog.tsx b/site/DataCatalog/DataCatalog.tsx index e68f8d47a2..a2b5738a1c 100644 --- a/site/DataCatalog/DataCatalog.tsx +++ b/site/DataCatalog/DataCatalog.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useReducer, useRef, useState } from "react" import * as React from "react" import cx from "classnames" import { + commafyNumber, countriesByName, Country, Region, @@ -419,7 +420,7 @@ const DataCatalogRibbon = ({ <div className="data-catalog-ribbon__header"> <h2 className="body-1-regular">{result.title}</h2> <span className="data-catalog-ribbon__hit-count body-2-semibold"> - {result.nbHits}{" "} + {commafyNumber(result.nbHits)}{" "} {result.nbHits === 1 ? "chart" : "charts"} <FontAwesomeIcon icon={faArrowRight} /> </span> @@ -455,7 +456,7 @@ const DataCatalogRibbon = ({ > {result.nbHits === 1 ? `See 1 chart` - : `See ${result.nbHits} charts`} + : `See ${commafyNumber(result.nbHits)} charts`} <FontAwesomeIcon icon={faArrowRight} /> </button> </div> @@ -562,7 +563,8 @@ const DataCatalogResults = ({ <div className="span-cols-12 col-start-2 data-catalog-search-hits"> {nbHits && ( <p className="data-catalog-search-list__results-count body-3-medium"> - {nbHits} {nbHits === 1 ? "indicator" : "indicators"} + {commafyNumber(nbHits)}{" "} + {nbHits === 1 ? "indicator" : "indicators"} </p> )} <ul className="data-catalog-search-list grid grid-cols-4 grid-sm-cols-1"> @@ -673,7 +675,7 @@ const TopicsRefinementList = ({ onClick={() => addTopic(facetName)} > <span className="body-3-medium"> - {facetName} ({count}) + {facetName} ({commafyNumber(count)}) </span> </button> </li> From 04d8c524bc2dda3e6b5c0e2caeabecb47c14c658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= <martin.racak@ourworldindata.org> Date: Thu, 9 Jan 2025 15:47:35 +0100 Subject: [PATCH 33/39] Fix data catalog canonical URL --- site/DataCatalog/DataCatalogPage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/DataCatalog/DataCatalogPage.tsx b/site/DataCatalog/DataCatalogPage.tsx index 20a4b53cc5..c227886d74 100644 --- a/site/DataCatalog/DataCatalogPage.tsx +++ b/site/DataCatalog/DataCatalogPage.tsx @@ -20,7 +20,7 @@ export const DataCatalogPage = (props: { return ( <Html> <Head - canonicalUrl={`${baseUrl}/charts`} + canonicalUrl={`${baseUrl}/data`} pageTitle="Data Catalog" pageDesc="Explore Our World in Data's extensive collection of charts. Use the search bar to find specific data visualizations or browse by topic. Filter by country or subject area to discover insights on global issues supported by reliable data." baseUrl={baseUrl} From b0c7b829c3d2056b691cb1f88991bbaa7a982581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ra=C4=8D=C3=A1k?= <rakyi@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:34:22 +0100 Subject: [PATCH 34/39] Update owid-logo.svg (#4425) --- public/owid-logo.svg | 2 +- site/gdocs/pages/GdocPost.tsx | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/public/owid-logo.svg b/public/owid-logo.svg index 38643f2e38..5c7ea55398 100644 --- a/public/owid-logo.svg +++ b/public/owid-logo.svg @@ -1 +1 @@ -<svg xmlns="http://www.w3.org/2000/svg" width="104" height="57" fill="none"><path fill="#002147" d="M.576 0h102.848v54.522H.576z"/><path fill="#CE261E" d="M.576 54.522h102.848V57H.576z"/><path fill="#fff" d="M24.811 16.985c0 .93-.147 1.787-.442 2.567a5.878 5.878 0 0 1-1.25 2.004 5.623 5.623 0 0 1-1.942 1.31c-.752.306-1.585.46-2.498.46-.914 0-1.747-.154-2.498-.46a5.642 5.642 0 0 1-1.935-1.31 5.878 5.878 0 0 1-1.249-2.004c-.295-.78-.442-1.636-.442-2.567 0-.931.147-1.784.442-2.559a5.861 5.861 0 0 1 1.25-2.012 5.584 5.584 0 0 1 1.933-1.319c.752-.312 1.585-.468 2.499-.468.913 0 1.746.156 2.498.468a5.566 5.566 0 0 1 1.943 1.319 5.862 5.862 0 0 1 1.249 2.012c.295.775.442 1.628.442 2.559Zm-1.726 0c0-.764-.104-1.449-.312-2.056a4.302 4.302 0 0 0-.885-1.535 3.814 3.814 0 0 0-1.388-.972c-.544-.225-1.15-.338-1.821-.338-.665 0-1.27.113-1.813.338a3.898 3.898 0 0 0-1.397.972 4.304 4.304 0 0 0-.884 1.535c-.209.607-.313 1.292-.313 2.056 0 .763.104 1.448.313 2.056.208.6.503 1.113.884 1.535a3.95 3.95 0 0 0 1.397.963c.543.22 1.148.33 1.813.33.67 0 1.277-.11 1.821-.33a3.864 3.864 0 0 0 1.388-.963c.382-.422.676-.934.885-1.535.208-.608.312-1.293.312-2.056Zm10.86-2.576v8.786h-.92c-.22 0-.359-.107-.417-.32l-.121-.946c-.382.422-.81.763-1.284 1.023-.474.255-1.018.382-1.63.382-.48 0-.905-.078-1.276-.234a2.57 2.57 0 0 1-.919-.677 2.904 2.904 0 0 1-.564-1.05 4.788 4.788 0 0 1-.182-1.361v-5.603h1.544v5.603c0 .665.15 1.18.451 1.544.306.364.772.546 1.396.546.457 0 .882-.107 1.276-.32.399-.22.766-.521 1.101-.903v-6.47h1.544Zm3.913 1.76c.278-.6.619-1.07 1.024-1.405.404-.34.899-.511 1.483-.511.185 0 .361.02.529.06.173.04.327.104.46.191l-.113 1.154c-.035.144-.121.216-.26.216-.081 0-.2-.017-.356-.052a2.448 2.448 0 0 0-.529-.052c-.278 0-.526.04-.746.122-.214.08-.408.202-.581.364a2.576 2.576 0 0 0-.46.59 5.146 5.146 0 0 0-.364.798v5.551h-1.553V14.41h.885c.168 0 .283.031.347.095.063.064.107.174.13.33l.104 1.335Zm25.154-5.403-3.877 12.43h-1.51l-3.148-9.481a9.62 9.62 0 0 1-.087-.295c-.023-.104-.05-.214-.078-.33a13.113 13.113 0 0 1-.174.625l-3.166 9.48h-1.509l-3.877-12.43h1.396c.15 0 .275.038.373.113.104.075.17.17.2.287l2.567 8.639c.04.156.078.324.113.503.04.18.078.37.113.572.04-.202.08-.393.121-.572a5.98 5.98 0 0 1 .148-.503l2.923-8.64a.619.619 0 0 1 .2-.268.564.564 0 0 1 .372-.13h.486a.56.56 0 0 1 .364.112c.093.075.162.17.208.287l2.915 8.639c.052.15.098.312.139.486.046.173.09.355.13.546.029-.19.06-.373.095-.546a7.69 7.69 0 0 1 .113-.486l2.576-8.64a.55.55 0 0 1 .191-.277.59.59 0 0 1 .373-.121h1.31Zm4.207 3.504c.642 0 1.22.107 1.735.32.514.215.954.518 1.318.912.364.393.642.87.833 1.43.196.556.295 1.178.295 1.866 0 .694-.099 1.318-.295 1.873a3.953 3.953 0 0 1-.833 1.423 3.719 3.719 0 0 1-1.318.91c-.515.209-1.093.313-1.735.313-.648 0-1.232-.104-1.752-.313a3.718 3.718 0 0 1-1.319-.91 4.057 4.057 0 0 1-.841-1.423c-.19-.555-.286-1.18-.286-1.873 0-.688.095-1.31.286-1.865.197-.561.477-1.038.841-1.431a3.716 3.716 0 0 1 1.319-.911c.52-.214 1.104-.321 1.752-.321Zm0 7.841c.867 0 1.515-.29 1.943-.867.428-.584.642-1.397.642-2.438 0-1.046-.214-1.862-.642-2.446-.428-.584-1.076-.876-1.943-.876-.44 0-.824.075-1.154.226-.324.15-.595.367-.815.65a2.984 2.984 0 0 0-.486 1.05c-.104.41-.156.876-.156 1.396 0 1.041.214 1.854.642 2.438.434.578 1.09.867 1.969.867Zm7.605-5.942c.277-.6.619-1.07 1.023-1.405.405-.34.9-.511 1.484-.511.185 0 .361.02.529.06.173.04.326.104.46.191l-.113 1.154c-.035.144-.122.216-.26.216-.082 0-.2-.017-.356-.052a2.449 2.449 0 0 0-.53-.052c-.277 0-.526.04-.745.122-.214.08-.408.202-.581.364-.168.156-.321.353-.46.59a5.146 5.146 0 0 0-.364.798v5.551h-1.553V14.41h.885c.167 0 .283.031.347.095.063.064.107.174.13.33l.104 1.335Zm6.45-5.75v12.776h-1.545V10.42h1.544Zm8.228 6.08c-.29-.387-.602-.656-.937-.807a2.64 2.64 0 0 0-1.128-.234c-.815 0-1.442.292-1.882.876-.44.584-.66 1.417-.66 2.498 0 .573.05 1.064.148 1.475.098.405.243.74.434 1.006.19.26.425.451.703.573.277.121.592.182.945.182.509 0 .951-.116 1.327-.347a3.786 3.786 0 0 0 1.05-.98v-4.242Zm1.544-6.08v12.776h-.92c-.22 0-.358-.107-.416-.32l-.139-1.067a4.787 4.787 0 0 1-1.284 1.092c-.48.278-1.035.417-1.665.417-.503 0-.96-.096-1.37-.287a2.882 2.882 0 0 1-1.05-.858c-.29-.376-.512-.845-.668-1.405-.156-.561-.234-1.206-.234-1.935 0-.647.087-1.249.26-1.804a4.45 4.45 0 0 1 .746-1.457 3.43 3.43 0 0 1 1.188-.963 3.52 3.52 0 0 1 1.596-.356c.538 0 .998.093 1.38.278.387.18.73.43 1.032.755v-4.866h1.544Zm-63.493 23.99v8.786H26.01V34.41h1.544Zm.33-2.759c0 .15-.032.292-.096.425a1.178 1.178 0 0 1-.242.347 1.087 1.087 0 0 1-.356.235 1.055 1.055 0 0 1-.425.086 1.054 1.054 0 0 1-.763-.32 1.265 1.265 0 0 1-.235-.348 1.055 1.055 0 0 1-.086-.425c0-.15.029-.292.086-.425.058-.139.136-.257.235-.355a1.07 1.07 0 0 1 .338-.243c.133-.058.274-.087.425-.087.15 0 .292.029.425.087.139.058.257.139.355.243.105.098.186.216.243.355a.973.973 0 0 1 .096.425Zm3.664 4.034c.191-.214.393-.408.607-.581.214-.174.44-.321.677-.443a3.537 3.537 0 0 1 1.64-.39c.48 0 .901.08 1.266.243.37.156.676.381.92.676.248.29.436.64.563 1.05.127.41.19.864.19 1.362v5.594h-1.552v-5.594c0-.665-.153-1.18-.46-1.544-.3-.37-.76-.555-1.379-.555-.456 0-.884.11-1.283.33-.394.219-.758.517-1.093.893v6.47H30.09V34.41h.928c.22 0 .356.107.408.32l.121.955Zm23.792 1.3c0 .932-.147 1.779-.442 2.542a5.636 5.636 0 0 1-1.249 1.96 5.603 5.603 0 0 1-1.943 1.267c-.752.295-1.584.442-2.498.442h-4.64v-12.43h4.64c.914 0 1.746.15 2.498.452a5.445 5.445 0 0 1 1.943 1.266 5.636 5.636 0 0 1 1.249 1.96c.295.764.442 1.61.442 2.542Zm-1.726 0c0-.763-.104-1.445-.312-2.046a4.314 4.314 0 0 0-.885-1.527 3.773 3.773 0 0 0-1.387-.954 4.822 4.822 0 0 0-1.822-.33h-2.958v9.706h2.958c.67 0 1.278-.11 1.822-.33a3.822 3.822 0 0 0 1.387-.945c.382-.416.677-.925.885-1.526.208-.602.312-1.284.312-2.047Zm8.328 2.256c-.711.023-1.318.081-1.821.173-.498.087-.905.203-1.223.347-.313.145-.541.316-.686.512a1.116 1.116 0 0 0-.208.66c0 .23.038.43.113.598.075.168.176.306.303.416.134.104.287.182.46.235.18.046.37.069.573.069.271 0 .52-.026.746-.078.225-.058.436-.139.633-.243.202-.104.393-.229.572-.373.185-.145.365-.31.538-.494V39.24Zm-4.987-3.591a5.016 5.016 0 0 1 1.57-1.05 4.847 4.847 0 0 1 1.864-.346c.492 0 .929.08 1.31.242.382.162.703.388.963.677.26.29.457.639.59 1.05.133.41.2.861.2 1.353v5.62h-.686a.708.708 0 0 1-.347-.07c-.08-.051-.144-.15-.19-.294l-.174-.833a9.396 9.396 0 0 1-.677.573c-.22.162-.45.3-.694.416-.242.11-.503.194-.78.252a4.005 4.005 0 0 1-.911.095c-.341 0-.662-.046-.963-.139a2.327 2.327 0 0 1-.79-.433 2.1 2.1 0 0 1-.528-.72A2.613 2.613 0 0 1 56.52 41c0-.347.095-.68.286-.998.19-.324.5-.61.928-.858.428-.25.986-.451 1.674-.608.688-.161 1.533-.254 2.533-.277v-.685c0-.683-.147-1.197-.442-1.544-.295-.353-.726-.53-1.293-.53-.381 0-.702.05-.963.148-.254.093-.477.2-.667.32-.185.117-.347.224-.486.322a.695.695 0 0 1-.4.139.46.46 0 0 1-.268-.078.82.82 0 0 1-.19-.209l-.278-.494Zm11.427 7.685c-.694 0-1.229-.194-1.605-.581-.37-.388-.555-.946-.555-1.674V35.7h-1.058a.358.358 0 0 1-.234-.078c-.064-.058-.095-.145-.095-.26v-.616l1.44-.182.355-2.715a.367.367 0 0 1 .113-.208.347.347 0 0 1 .243-.087h.78v3.027h2.542v1.12h-2.542v5.273c0 .37.09.645.27.824.178.18.41.269.693.269.162 0 .3-.02.416-.061.122-.046.226-.095.313-.148.087-.052.159-.098.217-.138.063-.046.118-.07.164-.07.081 0 .154.05.217.148l.451.737a2.933 2.933 0 0 1-.963.59 3.33 3.33 0 0 1-1.162.208Zm8.282-4.094c-.711.023-1.318.081-1.822.173-.497.087-.904.203-1.222.347-.313.145-.541.316-.686.512a1.116 1.116 0 0 0-.208.66c0 .23.038.43.113.598.075.168.176.306.303.416.133.104.287.182.46.235.18.046.37.069.573.069.271 0 .52-.026.746-.078.225-.058.436-.139.633-.243.202-.104.393-.229.572-.373.185-.145.365-.31.538-.494V39.24Zm-4.987-3.591a5.018 5.018 0 0 1 1.57-1.05 4.847 4.847 0 0 1 1.864-.346c.492 0 .928.08 1.31.242.382.162.703.388.963.677.26.29.457.639.59 1.05.133.41.2.861.2 1.353v5.62h-.686a.707.707 0 0 1-.347-.07c-.08-.051-.144-.15-.19-.294l-.174-.833a9.396 9.396 0 0 1-.677.573c-.22.162-.45.3-.694.416-.243.11-.503.194-.78.252a4.004 4.004 0 0 1-.911.095c-.341 0-.662-.046-.963-.139a2.327 2.327 0 0 1-.79-.433 2.1 2.1 0 0 1-.528-.72A2.613 2.613 0 0 1 71.242 41c0-.347.095-.68.286-.998.19-.324.5-.61.928-.858.428-.25.986-.451 1.674-.608.688-.161 1.533-.254 2.533-.277v-.685c0-.683-.147-1.197-.442-1.544-.295-.353-.726-.53-1.293-.53-.381 0-.702.05-.963.148a3.42 3.42 0 0 0-.668.32c-.184.117-.346.224-.485.322a.695.695 0 0 1-.4.139.461.461 0 0 1-.268-.078.817.817 0 0 1-.19-.209l-.278-.494Z"/></svg> \ No newline at end of file +<svg xmlns="http://www.w3.org/2000/svg" width="104" height="57" fill="none"><path fill="#002147" d="M0 0h104v57H0z"/><path fill="#CE261E" d="M0 53h104v4H0z"/><path fill="#fff" d="M22.088 16.92c0 1.018-.17 1.962-.509 2.832A6.708 6.708 0 0 1 20.148 22a6.539 6.539 0 0 1-2.22 1.488c-.864.352-1.821.528-2.872.528-1.05 0-2.01-.177-2.882-.529a6.644 6.644 0 0 1-2.228-1.488 6.708 6.708 0 0 1-1.432-2.246 7.727 7.727 0 0 1-.509-2.832c0-1.018.17-1.958.51-2.822a6.687 6.687 0 0 1 1.43-2.256 6.527 6.527 0 0 1 2.23-1.479c.87-.358 1.831-.537 2.881-.537 1.05 0 2.008.179 2.873.537a6.539 6.539 0 0 1 2.219 1.488 6.582 6.582 0 0 1 1.431 2.247c.34.864.51 1.804.51 2.822Zm-2.651 0c0-.762-.103-1.443-.308-2.045a4.272 4.272 0 0 0-.864-1.545 3.734 3.734 0 0 0-1.383-.97c-.538-.224-1.147-.336-1.826-.336-.678 0-1.29.112-1.834.336a3.824 3.824 0 0 0-1.393.97 4.38 4.38 0 0 0-.875 1.545c-.205.602-.307 1.284-.307 2.045 0 .762.102 1.446.307 2.054a4.4 4.4 0 0 0 .875 1.537 3.87 3.87 0 0 0 1.393.96c.544.224 1.155.335 1.834.335.68 0 1.288-.111 1.826-.335a3.786 3.786 0 0 0 1.383-.96c.378-.423.666-.935.864-1.537.205-.608.308-1.292.308-2.054Zm6.346-2.909v6.26c0 .601.138 1.068.413 1.401.282.326.701.49 1.258.49.41 0 .795-.09 1.153-.27a4.132 4.132 0 0 0 1.019-.758v-7.123h2.372v9.85h-1.45c-.308 0-.51-.144-.605-.432l-.164-.787a7.444 7.444 0 0 1-.634.566c-.218.167-.451.31-.701.432a3.79 3.79 0 0 1-.797.269 3.78 3.78 0 0 1-.913.106c-.538 0-1.015-.09-1.431-.27a2.916 2.916 0 0 1-1.038-.777 3.45 3.45 0 0 1-.643-1.18 5.169 5.169 0 0 1-.212-1.518v-6.259h2.373Zm8.365 9.849v-9.849h1.392c.244 0 .413.045.51.135.096.09.16.243.192.46l.144 1.19c.352-.607.765-1.087 1.239-1.44a2.609 2.609 0 0 1 1.595-.527c.486 0 .89.112 1.21.336l-.307 1.776c-.02.115-.061.198-.125.25a.445.445 0 0 1-.26.067c-.096 0-.227-.023-.393-.067a2.638 2.638 0 0 0-.663-.068c-.493 0-.916.138-1.268.413-.353.269-.65.666-.894 1.19v6.135h-2.373Zm9.636-13.88h2.171c.224 0 .41.054.557.162a.735.735 0 0 1 .308.423l2.363 8.304c.057.205.109.429.154.672.05.237.099.49.144.758.05-.268.105-.521.163-.758.064-.243.131-.467.202-.672l2.728-8.304a.92.92 0 0 1 .298-.403.87.87 0 0 1 .557-.183h.759a.91.91 0 0 1 .557.164.79.79 0 0 1 .307.422l2.71 8.304c.14.41.262.864.364 1.363.045-.25.09-.49.135-.72.051-.23.102-.445.153-.643l2.364-8.304a.775.775 0 0 1 .297-.413.903.903 0 0 1 .558-.173h2.027l-4.314 13.882h-2.334l-3.036-9.485-.124-.393a15.356 15.356 0 0 1-.116-.452c-.038.16-.08.31-.125.451a8.546 8.546 0 0 1-.115.394l-3.064 9.485h-2.335L43.784 9.979Zm24.696 3.878c.736 0 1.402.118 1.998.355a4.31 4.31 0 0 1 1.537 1.008c.422.435.749.966.98 1.593.23.628.345 1.328.345 2.103 0 .78-.115 1.485-.345 2.112a4.56 4.56 0 0 1-.98 1.603c-.423.442-.935.78-1.537 1.018-.596.237-1.262.355-1.998.355-.737 0-1.406-.118-2.008-.355a4.343 4.343 0 0 1-1.547-1.018 4.676 4.676 0 0 1-.99-1.603c-.23-.627-.345-1.331-.345-2.112 0-.775.115-1.475.346-2.103a4.573 4.573 0 0 1 .99-1.593 4.4 4.4 0 0 1 1.546-1.008c.602-.237 1.271-.355 2.008-.355Zm0 8.323c.82 0 1.424-.275 1.815-.826.397-.55.596-1.357.596-2.419s-.199-1.872-.596-2.429c-.39-.557-.996-.835-1.816-.835-.832 0-1.447.282-1.844.845-.397.557-.596 1.363-.596 2.42 0 1.055.199 1.861.596 2.418.397.55 1.012.826 1.844.826Zm6.328 1.679v-9.849H76.2c.244 0 .413.045.51.135.096.09.16.243.192.46l.144 1.19c.352-.607.765-1.087 1.239-1.44a2.609 2.609 0 0 1 1.595-.527c.486 0 .89.112 1.21.336l-.307 1.776c-.02.115-.061.198-.125.25a.445.445 0 0 1-.26.067 1.62 1.62 0 0 1-.394-.067 2.637 2.637 0 0 0-.662-.068c-.493 0-.916.138-1.268.413-.353.269-.65.666-.894 1.19v6.135h-2.373Zm9.984-14.265v14.266H82.42V9.595h2.372Zm9.242 14.265c-.308 0-.51-.143-.606-.431l-.192-.95a5.89 5.89 0 0 1-.643.623 3.996 3.996 0 0 1-.73.48 3.513 3.513 0 0 1-.836.308 3.8 3.8 0 0 1-.96.115 3.263 3.263 0 0 1-1.48-.336 3.359 3.359 0 0 1-1.163-.97c-.32-.429-.57-.957-.749-1.584-.173-.627-.26-1.344-.26-2.15 0-.73.1-1.408.298-2.035a4.953 4.953 0 0 1 .855-1.632c.372-.461.817-.82 1.336-1.075.519-.263 1.101-.394 1.748-.394.55 0 1.022.09 1.412.269.39.172.74.406 1.047.7V9.596h2.373v14.266h-1.45Zm-3.161-1.737c.493 0 .913-.102 1.258-.307a3.74 3.74 0 0 0 .98-.873v-4.417c-.269-.326-.563-.556-.883-.69a2.559 2.559 0 0 0-1.019-.202c-.358 0-.685.067-.98.201-.288.135-.534.34-.74.615-.198.268-.352.614-.46 1.037-.11.416-.164.908-.164 1.478 0 .576.045 1.066.135 1.469.096.396.23.723.403.979.173.25.384.432.634.547.25.109.529.163.836.163ZM26.148 33.211v9.85h-2.373v-9.85h2.373Zm.355-2.87c0 .205-.041.397-.125.576-.083.18-.195.336-.336.47a1.592 1.592 0 0 1-.48.327 1.543 1.543 0 0 1-.596.115c-.205 0-.4-.038-.586-.115a1.63 1.63 0 0 1-.797-.797 1.54 1.54 0 0 1 0-1.171 1.57 1.57 0 0 1 .327-.48c.134-.135.29-.24.47-.317.186-.083.381-.125.586-.125.212 0 .41.042.596.125.185.077.346.182.48.317a1.494 1.494 0 0 1 .461 1.075Zm1.95 12.719v-9.849h1.45c.308 0 .51.144.606.432l.163.778c.199-.205.407-.39.625-.557.224-.166.457-.31.7-.432.25-.122.516-.214.798-.278a4.17 4.17 0 0 1 .922-.096c.538 0 1.016.093 1.432.278.416.18.762.435 1.037.768.282.327.493.72.634 1.18.148.455.221.958.221 1.508v6.269h-2.373v-6.269c0-.602-.14-1.066-.422-1.392-.276-.333-.692-.5-1.25-.5a2.47 2.47 0 0 0-1.152.28 4.134 4.134 0 0 0-1.018.758v7.123h-2.373Zm26.335-6.94c0 1.018-.17 1.952-.509 2.803a6.44 6.44 0 0 1-1.431 2.199 6.404 6.404 0 0 1-2.22 1.43c-.864.34-1.821.509-2.871.509h-5.303V29.179h5.303c1.05 0 2.007.173 2.872.519a6.413 6.413 0 0 1 2.219 1.43 6.339 6.339 0 0 1 1.431 2.189c.34.851.51 1.785.51 2.803Zm-2.641 0c0-.761-.103-1.443-.308-2.045-.205-.608-.496-1.12-.874-1.536a3.734 3.734 0 0 0-1.383-.97c-.538-.223-1.147-.335-1.825-.335h-2.7v9.773h2.7c.678 0 1.287-.112 1.825-.337a3.786 3.786 0 0 0 1.383-.96c.378-.422.67-.934.874-1.535.205-.608.308-1.293.308-2.055Zm11.865 6.94h-1.066c-.224 0-.4-.031-.529-.095-.128-.07-.224-.208-.288-.413l-.211-.7c-.25.223-.496.422-.74.595a4.486 4.486 0 0 1-.74.422 3.798 3.798 0 0 1-.816.26 4.916 4.916 0 0 1-.96.086c-.417 0-.801-.055-1.154-.164a2.592 2.592 0 0 1-.912-.509 2.317 2.317 0 0 1-.586-.835 2.96 2.96 0 0 1-.212-1.161 2.3 2.3 0 0 1 .289-1.095c.198-.364.525-.69.98-.979.454-.294 1.06-.538 1.815-.73.756-.191 1.694-.3 2.815-.326v-.576c0-.66-.141-1.146-.423-1.46-.275-.32-.679-.48-1.21-.48-.385 0-.705.046-.96.135-.257.09-.481.192-.673.308a36.94 36.94 0 0 1-.52.297c-.16.09-.335.135-.527.135a.687.687 0 0 1-.413-.125.983.983 0 0 1-.28-.307l-.431-.759c1.133-1.037 2.5-1.555 4.102-1.555.576 0 1.088.096 1.537.288.454.185.839.448 1.152.787.314.333.551.733.711 1.2.167.467.25.98.25 1.536v6.22Zm-4.611-1.477c.243 0 .467-.023.672-.068a2.46 2.46 0 0 0 .577-.201 3.01 3.01 0 0 0 .528-.327c.173-.134.346-.29.519-.47v-1.66c-.692.031-1.271.092-1.739.181-.461.084-.832.193-1.114.327-.282.134-.484.291-.605.47a1.06 1.06 0 0 0-.173.586c0 .416.121.714.365.893.25.179.573.269.97.269Zm9.797 1.632c-.851 0-1.508-.24-1.969-.72-.461-.487-.692-1.156-.692-2.007v-5.5h-.999a.498.498 0 0 1-.336-.126c-.09-.083-.135-.208-.135-.374v-.94l1.586-.26.5-2.688a.484.484 0 0 1 .172-.298.585.585 0 0 1 .355-.105h1.23v3.1h2.594v1.69H68.91v5.338c0 .307.077.547.23.72.154.173.36.26.615.26.148 0 .27-.017.366-.049.102-.038.188-.077.259-.115.077-.038.144-.074.202-.106a.31.31 0 0 1 .172-.057c.071 0 .129.019.173.057a.677.677 0 0 1 .144.154l.711 1.152a3.592 3.592 0 0 1-1.19.653c-.45.147-.913.22-1.394.22Zm11.532-.155h-1.066c-.224 0-.4-.031-.529-.095-.128-.07-.224-.208-.288-.413l-.211-.7c-.25.223-.496.422-.74.595a4.49 4.49 0 0 1-.74.422 3.8 3.8 0 0 1-.816.26 4.916 4.916 0 0 1-.96.086 3.88 3.88 0 0 1-1.153-.164 2.592 2.592 0 0 1-.913-.509 2.317 2.317 0 0 1-.586-.835 2.963 2.963 0 0 1-.211-1.161c0-.371.096-.736.288-1.095.198-.364.525-.69.98-.979.455-.294 1.06-.538 1.815-.73.756-.191 1.694-.3 2.815-.326v-.576c0-.66-.14-1.146-.423-1.46-.275-.32-.678-.48-1.21-.48-.384 0-.704.046-.96.135a3.65 3.65 0 0 0-.673.308 30.79 30.79 0 0 1-.519.297c-.16.09-.336.135-.528.135a.687.687 0 0 1-.413-.125.982.982 0 0 1-.279-.307l-.432-.759c1.133-1.037 2.5-1.555 4.102-1.555.576 0 1.088.096 1.537.288a3.19 3.19 0 0 1 1.153.787c.313.333.55.733.71 1.2.167.467.25.98.25 1.536v6.22Zm-4.61-1.477c.242 0 .467-.023.672-.068a2.46 2.46 0 0 0 .576-.201 3.01 3.01 0 0 0 .528-.327c.173-.134.346-.29.52-.47v-1.66c-.693.031-1.272.092-1.74.181-.46.084-.832.193-1.114.327-.282.134-.484.291-.605.47-.115.18-.173.375-.173.586 0 .416.121.714.365.893.25.179.573.269.97.269Z"/></svg> \ No newline at end of file diff --git a/site/gdocs/pages/GdocPost.tsx b/site/gdocs/pages/GdocPost.tsx index 48ddb81c05..64468b8090 100644 --- a/site/gdocs/pages/GdocPost.tsx +++ b/site/gdocs/pages/GdocPost.tsx @@ -179,6 +179,8 @@ export function GdocPost({ src={`${BAKED_BASE_URL}/owid-logo.svg`} className="img-raw" alt="Our World in Data logo" + width={104} + height={57} /> <h3>Reuse this work freely</h3> </> From 83b0f958166d6876ba5c0ecc3a58449e3bc93adf Mon Sep 17 00:00:00 2001 From: Sophia Mersmann <sophia.mersmann1@gmail.com> Date: Thu, 9 Jan 2025 18:00:53 +0100 Subject: [PATCH 35/39] =?UTF-8?q?=E2=9C=A8=20(line)=20deduplicate=20line?= =?UTF-8?q?=20labels=20(#4420)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grapher/src/lineCharts/LineChart.tsx | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx index 207959ea00..3369bbbfa2 100644 --- a/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx +++ b/packages/@ourworldindata/grapher/src/lineCharts/LineChart.tsx @@ -1363,13 +1363,21 @@ export class LineChart // Order of the legend items on a line chart should visually correspond // to the order of the lines as the approach the legend @computed get lineLegendSeries(): LineLabelSeries[] { - // If there are any projections, ignore non-projection legends - // Bit of a hack - let seriesToShow = this.series - if (seriesToShow.some((series) => !!series.isProjection)) - seriesToShow = seriesToShow.filter((series) => series.isProjection) + // If there are any projections, ignore non-projection legends (bit of a hack) + let series = this.series + if (series.some((series) => !!series.isProjection)) + series = series.filter((series) => series.isProjection) + + // Deduplicate series by seriesName to avoid showing the same label multiple times + const deduplicatedSeries: LineChartSeries[] = [] + const seriesGroupedByName = groupBy(series, "seriesName") + for (const duplicates of Object.values(seriesGroupedByName)) { + // keep only the label for the series with the most recent data + // (series are sorted by time, so we can just take the last one) + deduplicatedSeries.push(last(duplicates)!) + } - return seriesToShow.map((series) => { + return deduplicatedSeries.map((series) => { const { seriesName, color } = series const lastValue = last(series.points)!.y return { From 436c2a91df29a77bb96a973b1ee0e1bbacb63730 Mon Sep 17 00:00:00 2001 From: Daniel Bachler <daniel@danielbachler.de> Date: Wed, 18 Dec 2024 21:11:26 +0100 Subject: [PATCH 36/39] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20first=20round=20o?= =?UTF-8?q?f=20apiRouter=20refactor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRouter.ts | 3866 ---------------------- adminSiteServer/apiRoutes/bulkUpdates.ts | 256 ++ adminSiteServer/apiRoutes/chartViews.ts | 290 ++ adminSiteServer/apiRoutes/charts.ts | 801 +++++ adminSiteServer/apiRoutes/datasets.ts | 417 +++ adminSiteServer/apiRoutes/explorer.ts | 37 + adminSiteServer/apiRoutes/gdocs.ts | 283 ++ adminSiteServer/apiRoutes/images.ts | 252 ++ adminSiteServer/apiRoutes/mdims.ts | 34 + adminSiteServer/apiRoutes/misc.ts | 183 + adminSiteServer/apiRoutes/posts.ts | 220 ++ adminSiteServer/apiRoutes/redirects.ts | 152 + adminSiteServer/apiRoutes/routeUtils.ts | 51 + adminSiteServer/apiRoutes/suggest.ts | 71 + adminSiteServer/apiRoutes/tagGraph.ts | 60 + adminSiteServer/apiRoutes/tags.ts | 269 ++ adminSiteServer/apiRoutes/users.ts | 118 + adminSiteServer/apiRoutes/variables.ts | 547 +++ adminSiteServer/getLogsByChartId.ts | 34 + 19 files changed, 4075 insertions(+), 3866 deletions(-) create mode 100644 adminSiteServer/apiRoutes/bulkUpdates.ts create mode 100644 adminSiteServer/apiRoutes/chartViews.ts create mode 100644 adminSiteServer/apiRoutes/charts.ts create mode 100644 adminSiteServer/apiRoutes/datasets.ts create mode 100644 adminSiteServer/apiRoutes/explorer.ts create mode 100644 adminSiteServer/apiRoutes/gdocs.ts create mode 100644 adminSiteServer/apiRoutes/images.ts create mode 100644 adminSiteServer/apiRoutes/mdims.ts create mode 100644 adminSiteServer/apiRoutes/misc.ts create mode 100644 adminSiteServer/apiRoutes/posts.ts create mode 100644 adminSiteServer/apiRoutes/redirects.ts create mode 100644 adminSiteServer/apiRoutes/routeUtils.ts create mode 100644 adminSiteServer/apiRoutes/suggest.ts create mode 100644 adminSiteServer/apiRoutes/tagGraph.ts create mode 100644 adminSiteServer/apiRoutes/tags.ts create mode 100644 adminSiteServer/apiRoutes/users.ts create mode 100644 adminSiteServer/apiRoutes/variables.ts create mode 100644 adminSiteServer/getLogsByChartId.ts diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 7937994feb..48ae2b306e 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -1,3872 +1,6 @@ /* eslint @typescript-eslint/no-unused-vars: [ "warn", { argsIgnorePattern: "^(res|req)$" } ] */ -import * as lodash from "lodash" -import * as db from "../db/db.js" -import { - UNCATEGORIZED_TAG_ID, - BAKE_ON_CHANGE, - BAKED_BASE_URL, - ADMIN_BASE_URL, - DATA_API_URL, - FEATURE_FLAGS, -} from "../settings/serverSettings.js" -import { - CLOUDFLARE_IMAGES_URL, - FeatureFlagFeature, -} from "../settings/clientSettings.js" -import { expectInt, isValidSlug } from "../serverUtils/serverUtil.js" -import { - OldChartFieldList, - assignTagsForCharts, - getChartConfigById, - getChartSlugById, - getGptTopicSuggestions, - getRedirectsByChartId, - oldChartFieldList, - setChartTags, - getParentByChartConfig, - getPatchConfigByChartId, - isInheritanceEnabledForChart, - getParentByChartId, -} from "../db/model/Chart.js" -import { Request } from "./authentication.js" -import { - getMergedGrapherConfigForVariable, - fetchS3MetadataByPath, - fetchS3DataValuesByPath, - searchVariables, - getGrapherConfigsForVariable, - updateGrapherConfigAdminOfVariable, - updateGrapherConfigETLOfVariable, - updateAllChartsThatInheritFromIndicator, - updateAllMultiDimViewsThatInheritFromIndicator, - getAllChartsForIndicator, -} from "../db/model/Variable.js" -import { updateExistingFullConfig } from "../db/model/ChartConfigs.js" -import { getCanonicalUrl } from "@ourworldindata/components" -import { - GDOCS_BASE_URL, - camelCaseProperties, - GdocsContentSource, - isEmpty, - JsonError, - OwidGdocPostInterface, - parseIntOrUndefined, - DbRawPostWithGdocPublishStatus, - OwidVariableWithSource, - TaggableType, - DbChartTagJoin, - pick, - Json, - checkIsGdocPostExcludingFragments, - checkIsPlainObjectWithGuard, - mergeGrapherConfigs, - diffGrapherConfigs, - omitUndefinedValues, - getParentVariableIdFromChartConfig, - omit, - gdocUrlRegex, -} from "@ourworldindata/utils" -import { applyPatch } from "../adminShared/patchHelper.js" -import { - OperationContext, - parseToOperation, -} from "../adminShared/SqlFilterSExpression.js" -import { - BulkChartEditResponseRow, - BulkGrapherConfigResponse, - chartBulkUpdateAllowedColumnNamesAndTypes, - GrapherConfigPatch, - variableAnnotationAllowedColumnNamesAndTypes, - VariableAnnotationsResponseRow, -} from "../adminShared/AdminSessionTypes.js" -import { - DbPlainDatasetTag, - GrapherInterface, - OwidGdocType, - DbPlainUser, - UsersTableName, - DbPlainTag, - DbRawVariable, - parseOriginsRow, - PostsTableName, - DbRawPost, - DbPlainChartSlugRedirect, - DbPlainChart, - DbInsertChartRevision, - serializeChartConfig, - DbRawOrigin, - DbRawPostGdoc, - PostsGdocsXImagesTableName, - PostsGdocsLinksTableName, - PostsGdocsTableName, - DbPlainDataset, - DbInsertUser, - FlatTagGraph, - DbRawChartConfig, - parseChartConfig, - MultiDimDataPageConfigRaw, - R2GrapherConfigDirectory, - ChartConfigsTableName, - Base64String, - DbPlainChartView, - ChartViewsTableName, - DbInsertChartView, - PostsGdocsComponentsTableName, - CHART_VIEW_PROPS_TO_PERSIST, - CHART_VIEW_PROPS_TO_OMIT, - DbEnrichedImage, - JsonString, -} from "@ourworldindata/types" -import { uuidv7 } from "uuidv7" -import { - migrateGrapherConfigToLatestVersion, - getVariableDataRoute, - getVariableMetadataRoute, - defaultGrapherConfig, - grapherConfigToQueryParams, -} from "@ourworldindata/grapher" -import { getDatasetById, setTagsForDataset } from "../db/model/Dataset.js" -import { getUserById, insertUser, updateUser } from "../db/model/User.js" -import { GdocPost } from "../db/model/Gdoc/GdocPost.js" -import { - syncDatasetToGitRepo, - removeDatasetFromGitRepo, -} from "./gitDataExport.js" -import { denormalizeLatestCountryData } from "../baker/countryProfiles.js" -import { - indexIndividualGdocPost, - removeIndividualGdocPostFromIndex, -} from "../baker/algolia/utils/pages.js" -import { ChartViewMinimalInformation } from "../adminSiteClient/ChartEditor.js" -import { DeployQueueServer } from "../baker/DeployQueueServer.js" import { FunctionalRouter } from "./FunctionalRouter.js" -import Papa from "papaparse" -import { - setTagsForPost, - getTagsByPostId, - getWordpressPostReferencesByChartId, - getGdocsPostReferencesByChartId, -} from "../db/model/Post.js" -import { - checkHasChanges, - checkIsLightningUpdate, - GdocPublishingAction, - getPublishingAction, -} from "../adminSiteClient/gdocsDeploy.js" -import { createGdocAndInsertOwidGdocPostContent } from "../db/model/Gdoc/archieToGdoc.js" -import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js" -import { - getRouteWithROTransaction, - deleteRouteWithRWTransaction, - putRouteWithRWTransaction, - postRouteWithRWTransaction, - patchRouteWithRWTransaction, - getRouteNonIdempotentWithRWTransaction, -} from "./functionalRouterHelpers.js" -import { getPublishedLinksTo } from "../db/model/Link.js" -import { - getChainedRedirect, - getRedirectById, - getRedirects, - redirectWithSourceExists, -} from "../db/model/Redirect.js" -import { getMinimalGdocPostsByIds } from "../db/model/Gdoc/GdocBase.js" -import { - GdocLinkUpdateMode, - createOrLoadGdocById, - gdocFromJSON, - getAllGdocIndexItemsOrderedByUpdatedAt, - getAndLoadGdocById, - getGdocBaseObjectById, - setLinksForGdoc, - setTagsForGdoc, - addImagesToContentGraph, - updateGdocContentOnly, - upsertGdoc, -} from "../db/model/Gdoc/GdocFactory.js" -import { match } from "ts-pattern" -import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js" -import { GdocHomepage } from "../db/model/Gdoc/GdocHomepage.js" -import { GdocAbout } from "../db/model/Gdoc/GdocAbout.js" -import { GdocAuthor } from "../db/model/Gdoc/GdocAuthor.js" -import path from "path" -import { - deleteGrapherConfigFromR2, - deleteGrapherConfigFromR2ByUUID, - saveGrapherConfigToR2ByUUID, -} from "./chartConfigR2Helpers.js" -import { createMultiDimConfig } from "./multiDim.js" -import { isMultiDimDataPagePublished } from "../db/model/MultiDimDataPage.js" -import { - retrieveChartConfigFromDbAndSaveToR2, - saveNewChartConfigInDbAndR2, - updateChartConfigInDbAndR2, -} from "./chartConfigHelpers.js" -import { ApiChartViewOverview } from "../adminShared/AdminTypes.js" -import { References } from "../adminSiteClient/AbstractChartEditor.js" -import { - deleteFromCloudflare, - fetchGptGeneratedAltText, - processImageContent, - uploadToCloudflare, - validateImagePayload, -} from "./imagesHelpers.js" -import pMap from "p-map" const apiRouter = new FunctionalRouter() - -// Call this to trigger build and deployment of static charts on change -const triggerStaticBuild = async (user: DbPlainUser, commitMessage: string) => { - if (!BAKE_ON_CHANGE) { - console.log( - "Not triggering static build because BAKE_ON_CHANGE is false" - ) - return - } - - return new DeployQueueServer().enqueueChange({ - timeISOString: new Date().toISOString(), - authorName: user.fullName, - authorEmail: user.email, - message: commitMessage, - }) -} - -const enqueueLightningChange = async ( - user: DbPlainUser, - commitMessage: string, - slug: string -) => { - if (!BAKE_ON_CHANGE) { - console.log( - "Not triggering static build because BAKE_ON_CHANGE is false" - ) - return - } - - return new DeployQueueServer().enqueueChange({ - timeISOString: new Date().toISOString(), - authorName: user.fullName, - authorEmail: user.email, - message: commitMessage, - slug, - }) -} - -async function getLogsByChartId( - knex: db.KnexReadonlyTransaction, - chartId: number -): Promise< - { - userId: number - config: Json - userName: string - createdAt: Date - }[] -> { - const logs = await db.knexRaw<{ - userId: number - config: string - userName: string - createdAt: Date - }>( - knex, - `SELECT userId, config, fullName as userName, l.createdAt - FROM chart_revisions l - LEFT JOIN users u on u.id = userId - WHERE chartId = ? - ORDER BY l.id DESC - LIMIT 50`, - [chartId] - ) - return logs.map((log) => ({ - ...log, - config: JSON.parse(log.config), - })) -} - -const getReferencesByChartId = async ( - chartId: number, - knex: db.KnexReadonlyTransaction -): Promise<References> => { - const postsWordpressPromise = getWordpressPostReferencesByChartId( - chartId, - knex - ) - const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex) - const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>( - knex, - `SELECT DISTINCT - explorerSlug - FROM - explorer_charts - WHERE - chartId = ?`, - [chartId] - ) - const chartViewsPromise = db.knexRaw<ChartViewMinimalInformation>( - knex, - `-- sql - SELECT cv.id, cv.name, cc.full ->> "$.title" AS title - FROM chart_views cv - JOIN chart_configs cc ON cc.id = cv.chartConfigId - WHERE cv.parentChartId = ?`, - [chartId] - ) - const [postsWordpress, postsGdocs, explorerSlugs, chartViews] = - await Promise.all([ - postsWordpressPromise, - postGdocsPromise, - explorerSlugsPromise, - chartViewsPromise, - ]) - - return { - postsGdocs, - postsWordpress, - explorers: explorerSlugs.map( - (row: { explorerSlug: string }) => row.explorerSlug - ), - chartViews, - } -} - -const expectChartById = async ( - knex: db.KnexReadonlyTransaction, - chartId: any -): Promise<GrapherInterface> => { - const chart = await getChartConfigById(knex, expectInt(chartId)) - if (chart) return chart.config - - throw new JsonError(`No chart found for id ${chartId}`, 404) -} - -const expectPatchConfigByChartId = async ( - knex: db.KnexReadonlyTransaction, - chartId: any -): Promise<GrapherInterface> => { - const patchConfig = await getPatchConfigByChartId(knex, expectInt(chartId)) - if (!patchConfig) { - throw new JsonError(`No chart found for id ${chartId}`, 404) - } - return patchConfig -} - -const saveNewChart = async ( - knex: db.KnexReadWriteTransaction, - { - config, - user, - // new charts inherit by default - shouldInherit = true, - }: { config: GrapherInterface; user: DbPlainUser; shouldInherit?: boolean } -): Promise<{ - chartConfigId: Base64String - patchConfig: GrapherInterface - fullConfig: GrapherInterface -}> => { - // grab the parent of the chart if inheritance should be enabled - const parent = shouldInherit - ? await getParentByChartConfig(knex, config) - : undefined - - // compute patch and full configs - const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) - const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) - - // insert patch & full configs into the chart_configs table - // We can't quite use `saveNewChartConfigInDbAndR2` here, because - // we need to update the chart id in the config after inserting it. - const chartConfigId = uuidv7() as Base64String - await db.knexRaw( - knex, - `-- sql - INSERT INTO chart_configs (id, patch, full) - VALUES (?, ?, ?) - `, - [ - chartConfigId, - serializeChartConfig(patchConfig), - serializeChartConfig(fullConfig), - ] - ) - - // add a new chart to the charts table - const result = await db.knexRawInsert( - knex, - `-- sql - INSERT INTO charts (configId, isInheritanceEnabled, lastEditedAt, lastEditedByUserId) - VALUES (?, ?, ?, ?) - `, - [chartConfigId, shouldInherit, new Date(), user.id] - ) - - // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true - const chartId = result.insertId - patchConfig.id = chartId - fullConfig.id = chartId - await db.knexRaw( - knex, - `-- sql - UPDATE chart_configs cc - JOIN charts c ON c.configId = cc.id - SET - cc.patch=JSON_SET(cc.patch, '$.id', ?), - cc.full=JSON_SET(cc.full, '$.id', ?) - WHERE c.id = ? - `, - [chartId, chartId, chartId] - ) - - await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId) - - return { chartConfigId, patchConfig, fullConfig } -} - -const updateExistingChart = async ( - knex: db.KnexReadWriteTransaction, - params: { - config: GrapherInterface - user: DbPlainUser - chartId: number - // if undefined, keep inheritance as is. - // if true or false, enable or disable inheritance - shouldInherit?: boolean - } -): Promise<{ - chartConfigId: Base64String - patchConfig: GrapherInterface - fullConfig: GrapherInterface -}> => { - const { config, user, chartId } = params - - // make sure that the id of the incoming config matches the chart id - config.id = chartId - - // if inheritance is enabled, grab the parent from its config - const shouldInherit = - params.shouldInherit ?? - (await isInheritanceEnabledForChart(knex, chartId)) - const parent = shouldInherit - ? await getParentByChartConfig(knex, config) - : undefined - - // compute patch and full configs - const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) - const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) - - const chartConfigIdRow = await db.knexRawFirst< - Pick<DbPlainChart, "configId"> - >(knex, `SELECT configId FROM charts WHERE id = ?`, [chartId]) - - if (!chartConfigIdRow) - throw new JsonError(`No chart config found for id ${chartId}`, 404) - - const now = new Date() - - const { chartConfigId } = await updateChartConfigInDbAndR2( - knex, - chartConfigIdRow.configId as Base64String, - patchConfig, - fullConfig - ) - - // update charts row - await db.knexRaw( - knex, - `-- sql - UPDATE charts - SET isInheritanceEnabled=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=? - WHERE id = ? - `, - [shouldInherit, now, now, user.id, chartId] - ) - - return { chartConfigId, patchConfig, fullConfig } -} - -const saveGrapher = async ( - knex: db.KnexReadWriteTransaction, - { - user, - newConfig, - existingConfig, - shouldInherit, - referencedVariablesMightChange = true, - }: { - user: DbPlainUser - newConfig: GrapherInterface - existingConfig?: GrapherInterface - // if undefined, keep inheritance as is. - // if true or false, enable or disable inheritance - shouldInherit?: boolean - // if the variables a chart uses can change then we need - // to update the latest country data which takes quite a long time (hundreds of ms) - referencedVariablesMightChange?: boolean - } -) => { - // Try to migrate the new config to the latest version - newConfig = migrateGrapherConfigToLatestVersion(newConfig) - - // Slugs need some special logic to ensure public urls remain consistent whenever possible - async function isSlugUsedInRedirect() { - const rows = await db.knexRaw<DbPlainChartSlugRedirect>( - knex, - `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`, - // -1 is a placeholder ID that will never exist; but we cannot use NULL because - // in that case we would always get back an empty resultset - [existingConfig ? existingConfig.id : -1, newConfig.slug] - ) - return rows.length > 0 - } - - async function isSlugUsedInOtherGrapher() { - const rows = await db.knexRaw<Pick<DbPlainChart, "id">>( - knex, - `-- sql - SELECT c.id - FROM charts c - JOIN chart_configs cc ON cc.id = c.configId - WHERE - c.id != ? - AND cc.full ->> "$.isPublished" = "true" - AND cc.slug = ? - `, - // -1 is a placeholder ID that will never exist; but we cannot use NULL because - // in that case we would always get back an empty resultset - [existingConfig ? existingConfig.id : -1, newConfig.slug] - ) - return rows.length > 0 - } - - // When a chart is published, check for conflicts - if (newConfig.isPublished) { - if (!isValidSlug(newConfig.slug)) - throw new JsonError(`Invalid chart slug ${newConfig.slug}`) - else if (await isSlugUsedInRedirect()) - throw new JsonError( - `This chart slug was previously used by another chart: ${newConfig.slug}` - ) - else if (await isSlugUsedInOtherGrapher()) - throw new JsonError( - `This chart slug is in use by another published chart: ${newConfig.slug}` - ) - else if ( - existingConfig && - existingConfig.isPublished && - existingConfig.slug !== newConfig.slug - ) { - // Changing slug of an existing chart, delete any old redirect and create new one - await db.knexRaw( - knex, - `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`, - [existingConfig.id, existingConfig.slug] - ) - await db.knexRaw( - knex, - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [existingConfig.id, existingConfig.slug] - ) - // When we rename grapher configs, make sure to delete the old one (the new one will be saved below) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${existingConfig.slug}.json` - ) - } - } - - if (existingConfig) - // Bump chart version, very important for cachebusting - newConfig.version = existingConfig.version! + 1 - else if (newConfig.version) - // If a chart is republished, we want to keep incrementing the old version number, - // otherwise it can lead to clients receiving cached versions of the old data. - newConfig.version += 1 - else newConfig.version = 1 - - // add the isPublished field if is missing - if (newConfig.isPublished === undefined) { - newConfig.isPublished = false - } - - // Execute the actual database update or creation - let chartId: number - let chartConfigId: Base64String - let patchConfig: GrapherInterface - let fullConfig: GrapherInterface - if (existingConfig) { - chartId = existingConfig.id! - const configs = await updateExistingChart(knex, { - config: newConfig, - user, - chartId, - shouldInherit, - }) - chartConfigId = configs.chartConfigId - patchConfig = configs.patchConfig - fullConfig = configs.fullConfig - } else { - const configs = await saveNewChart(knex, { - config: newConfig, - user, - shouldInherit, - }) - chartConfigId = configs.chartConfigId - patchConfig = configs.patchConfig - fullConfig = configs.fullConfig - chartId = fullConfig.id! - } - - // Record this change in version history - const chartRevisionLog = { - chartId: chartId as number, - userId: user.id, - config: serializeChartConfig(patchConfig), - createdAt: new Date(), - updatedAt: new Date(), - } satisfies DbInsertChartRevision - await db.knexRaw( - knex, - `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, - [ - chartRevisionLog.chartId, - chartRevisionLog.userId, - chartRevisionLog.config, - chartRevisionLog.createdAt, - chartRevisionLog.updatedAt, - ] - ) - - // Remove any old dimensions and store the new ones - // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json - await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [ - chartId, - ]) - - const newDimensions = fullConfig.dimensions ?? [] - for (const [i, dim] of newDimensions.entries()) { - await db.knexRaw( - knex, - `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?, ?, ?, ?)`, - [chartId, dim.variableId, dim.property, i] - ) - } - - // So we can generate country profiles including this chart data - if (fullConfig.isPublished && referencedVariablesMightChange) - // TODO: remove this ad hoc knex transaction context when we switch the function to knex - await denormalizeLatestCountryData( - knex, - newDimensions.map((d) => d.variableId) - ) - - if (fullConfig.isPublished) { - await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId, { - directory: R2GrapherConfigDirectory.publishedGrapherBySlug, - filename: `${fullConfig.slug}.json`, - }) - } - - if ( - fullConfig.isPublished && - (!existingConfig || !existingConfig.isPublished) - ) { - // Newly published, set publication info - await db.knexRaw( - knex, - `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `, - [new Date(), user.id, chartId] - ) - await triggerStaticBuild(user, `Publishing chart ${fullConfig.slug}`) - } else if ( - !fullConfig.isPublished && - existingConfig && - existingConfig.isPublished - ) { - // Unpublishing chart, delete any existing redirects to it - await db.knexRaw( - knex, - `DELETE FROM chart_slug_redirects WHERE chart_id = ?`, - [existingConfig.id] - ) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${existingConfig.slug}.json` - ) - await triggerStaticBuild(user, `Unpublishing chart ${fullConfig.slug}`) - } else if (fullConfig.isPublished) - await triggerStaticBuild(user, `Updating chart ${fullConfig.slug}`) - - return { - chartId, - savedPatch: patchConfig, - } -} - -async function updateGrapherConfigsInR2( - knex: db.KnexReadonlyTransaction, - updatedCharts: { chartConfigId: string; isPublished: boolean }[], - updatedMultiDimViews: { chartConfigId: string; isPublished: boolean }[] -) { - const idsToUpdate = [ - ...updatedCharts.filter(({ isPublished }) => isPublished), - ...updatedMultiDimViews, - ].map(({ chartConfigId }) => chartConfigId) - const builder = knex<DbRawChartConfig>(ChartConfigsTableName) - .select("id", "full", "fullMd5") - .whereIn("id", idsToUpdate) - for await (const { id, full, fullMd5 } of builder.stream()) { - await saveGrapherConfigToR2ByUUID(id, full, fullMd5) - } -} - -getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 - const charts = await db.knexRaw<OldChartFieldList>( - trx, - `-- sql - SELECT ${oldChartFieldList}, - round(views_365d / 365, 1) as pageviewsPerDay - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN analytics_pageviews on (analytics_pageviews.url = CONCAT("https://ourworldindata.org/grapher/", chart_configs.slug) AND chart_configs.full ->> '$.isPublished' = "true" ) - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - ORDER BY charts.lastEditedAt DESC LIMIT ? - `, - [limit] - ) - - await assignTagsForCharts(trx, charts) - - return { charts } -}) - -getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 - - // note: this query is extended from OldChart.listFields. - const charts = await db.knexRaw( - trx, - `-- sql - SELECT - charts.id, - chart_configs.full->>"$.version" AS version, - CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url, - CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl, - chart_configs.full->>"$.slug" AS slug, - chart_configs.full->>"$.title" AS title, - chart_configs.full->>"$.subtitle" AS subtitle, - chart_configs.full->>"$.sourceDesc" AS sourceDesc, - chart_configs.full->>"$.note" AS note, - chart_configs.chartType AS type, - chart_configs.full->>"$.internalNotes" AS internalNotes, - chart_configs.full->>"$.variantName" AS variantName, - chart_configs.full->>"$.isPublished" AS isPublished, - chart_configs.full->>"$.tab" AS tab, - chart_configs.chartType IS NOT NULL AS hasChartTab, - JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab, - chart_configs.full->>"$.originUrl" AS originUrl, - charts.lastEditedAt, - charts.lastEditedByUserId, - lastEditedByUser.fullName AS lastEditedBy, - charts.publishedAt, - charts.publishedByUserId, - publishedByUser.fullName AS publishedBy - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - ORDER BY charts.lastEditedAt DESC - LIMIT ? - `, - [limit] - ) - // note: retrieving references is VERY slow. - // await Promise.all( - // charts.map(async (chart: any) => { - // const references = await getReferencesByChartId(chart.id) - // chart.references = references.length - // ? references.map((ref) => ref.url) - // : "" - // }) - // ) - // await Chart.assignTagsForCharts(charts) - res.setHeader("Content-disposition", "attachment; filename=charts.csv") - res.setHeader("content-type", "text/csv") - const csv = Papa.unparse(charts) - return csv -}) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.config.json", - async (req, res, trx) => expectChartById(trx, req.params.chartId) -) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.parent.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const parent = await getParentByChartId(trx, chartId) - const isInheritanceEnabled = await isInheritanceEnabledForChart( - trx, - chartId - ) - return omitUndefinedValues({ - variableId: parent?.variableId, - config: parent?.config, - isActive: isInheritanceEnabled, - }) - } -) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.patchConfig.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const config = await expectPatchConfigByChartId(trx, chartId) - return config - } -) - -getRouteWithROTransaction( - apiRouter, - "/editorData/namespaces.json", - async (req, res, trx) => { - const rows = await db.knexRaw<{ - name: string - description?: string - isArchived: boolean - }>( - trx, - `SELECT DISTINCT - namespace AS name, - namespaces.description AS description, - namespaces.isArchived AS isArchived - FROM active_datasets - JOIN namespaces ON namespaces.name = active_datasets.namespace` - ) - - return { - namespaces: lodash - .sortBy(rows, (row) => row.description) - .map((namespace) => ({ - ...namespace, - isArchived: !!namespace.isArchived, - })), - } - } -) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.logs.json", - async (req, res, trx) => ({ - logs: await getLogsByChartId( - trx, - parseInt(req.params.chartId as string) - ), - }) -) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.references.json", - async (req, res, trx) => { - const references = { - references: await getReferencesByChartId( - parseInt(req.params.chartId as string), - trx - ), - } - return references - } -) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.redirects.json", - async (req, res, trx) => ({ - redirects: await getRedirectsByChartId( - trx, - parseInt(req.params.chartId as string) - ), - }) -) - -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.pageviews.json", - async (req, res, trx) => { - const slug = await getChartSlugById( - trx, - parseInt(req.params.chartId as string) - ) - if (!slug) return {} - - const pageviewsByUrl = await db.knexRawFirst( - trx, - `-- sql - SELECT * - FROM - analytics_pageviews - WHERE - url = ?`, - [`https://ourworldindata.org/grapher/${slug}`] - ) - - return { - pageviews: pageviewsByUrl ?? undefined, - } - } -) - -getRouteWithROTransaction( - apiRouter, - "/editorData/variables.json", - async (req, res, trx) => { - const datasets = [] - const rows = await db.knexRaw< - Pick<DbRawVariable, "name" | "id"> & { - datasetId: number - datasetName: string - datasetVersion: string - } & Pick< - DbPlainDataset, - "namespace" | "isPrivate" | "nonRedistributable" - > - >( - trx, - `-- sql - SELECT - v.name, - v.id, - d.id as datasetId, - d.name as datasetName, - d.version as datasetVersion, - d.namespace, - d.isPrivate, - d.nonRedistributable - FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id - ORDER BY d.updatedAt DESC - ` - ) - - let dataset: - | { - id: number - name: string - version: string - namespace: string - isPrivate: boolean - nonRedistributable: boolean - variables: { id: number; name: string }[] - } - | undefined - for (const row of rows) { - if (!dataset || row.datasetName !== dataset.name) { - if (dataset) datasets.push(dataset) - - dataset = { - id: row.datasetId, - name: row.datasetName, - version: row.datasetVersion, - namespace: row.namespace, - isPrivate: !!row.isPrivate, - nonRedistributable: !!row.nonRedistributable, - variables: [], - } - } - - dataset.variables.push({ - id: row.id, - name: row.name ?? "", - }) - } - - if (dataset) datasets.push(dataset) - - return { datasets: datasets } - } -) - -apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { - const variableStr = req.params.variableStr as string - if (!variableStr) throw new JsonError("No variable id given") - if (variableStr.includes("+")) - throw new JsonError( - "Requesting multiple variables at the same time is no longer supported" - ) - const variableId = parseInt(variableStr) - if (isNaN(variableId)) throw new JsonError("Invalid variable id") - return await fetchS3DataValuesByPath( - getVariableDataRoute(DATA_API_URL, variableId) + "?nocache" - ) -}) - -apiRouter.get( - "/data/variables/metadata/:variableStr.json", - async (req, res) => { - const variableStr = req.params.variableStr as string - if (!variableStr) throw new JsonError("No variable id given") - if (variableStr.includes("+")) - throw new JsonError( - "Requesting multiple variables at the same time is no longer supported" - ) - const variableId = parseInt(variableStr) - if (isNaN(variableId)) throw new JsonError("Invalid variable id") - return await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" - ) - } -) - -postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { - let shouldInherit: boolean | undefined - if (req.query.inheritance) { - shouldInherit = req.query.inheritance === "enable" - } - - try { - const { chartId } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - shouldInherit, - }) - - return { success: true, chartId: chartId } - } catch (err) { - return { success: false, error: String(err) } - } -}) - -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/setTags", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - - await setChartTags(trx, chartId, req.body.tags) - - return { success: true } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - let shouldInherit: boolean | undefined - if (req.query.inheritance) { - shouldInherit = req.query.inheritance === "enable" - } - - const existingConfig = await expectChartById(trx, req.params.chartId) - - try { - const { chartId, savedPatch } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - existingConfig, - shouldInherit, - }) - - const logs = await getLogsByChartId( - trx, - existingConfig.id as number - ) - return { - success: true, - chartId, - savedPatch, - newLog: logs[0], - } - } catch (err) { - return { - success: false, - error: String(err), - } - } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - const chart = await expectChartById(trx, req.params.chartId) - if (chart.slug) { - const links = await getPublishedLinksTo(trx, [chart.slug]) - if (links.length) { - const sources = links.map((link) => link.sourceSlug).join(", ") - throw new Error( - `Cannot delete chart in-use in the following published documents: ${sources}` - ) - } - } - - await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ - chart.id, - ]) - await db.knexRaw( - trx, - `DELETE FROM chart_slug_redirects WHERE chart_id=?`, - [chart.id] - ) - - const row = await db.knexRawFirst<Pick<DbPlainChart, "configId">>( - trx, - `SELECT configId FROM charts WHERE id = ?`, - [chart.id] - ) - if (!row || !row.configId) - throw new JsonError(`No chart config found for id ${chart.id}`, 404) - if (row) { - await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) - await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ - row.configId, - ]) - } - - if (chart.isPublished) - await triggerStaticBuild( - res.locals.user, - `Deleting chart ${chart.slug}` - ) - - await deleteGrapherConfigFromR2ByUUID(row.configId) - if (chart.isPublished) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${chart.slug}.json` - ) - - return { success: true } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/multi-dim/:slug", - async (req, res, trx) => { - const { slug } = req.params - if (!isValidSlug(slug)) { - throw new JsonError(`Invalid multi-dim slug ${slug}`) - } - const rawConfig = req.body as MultiDimDataPageConfigRaw - const id = await createMultiDimConfig(trx, slug, rawConfig) - if ( - FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && - (await isMultiDimDataPagePublished(trx, slug)) - ) { - await triggerStaticBuild( - res.locals.user, - `Publishing multidimensional chart ${slug}` - ) - } - return { success: true, id } - } -) - -getRouteWithROTransaction(apiRouter, "/users.json", async (req, res, trx) => ({ - users: await trx - .select( - "id" satisfies keyof DbPlainUser, - "email" satisfies keyof DbPlainUser, - "fullName" satisfies keyof DbPlainUser, - "isActive" satisfies keyof DbPlainUser, - "isSuperuser" satisfies keyof DbPlainUser, - "createdAt" satisfies keyof DbPlainUser, - "updatedAt" satisfies keyof DbPlainUser, - "lastLogin" satisfies keyof DbPlainUser, - "lastSeen" satisfies keyof DbPlainUser - ) - .from<DbPlainUser>(UsersTableName) - .orderBy("lastSeen", "desc"), -})) - -getRouteWithROTransaction( - apiRouter, - "/users/:userId.json", - async (req, res, trx) => { - const id = parseIntOrUndefined(req.params.userId) - if (!id) throw new JsonError("No user id given") - const user = await getUserById(trx, id) - return { user } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/users/:userId", - async (req, res, trx) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = expectInt(req.params.userId) - await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) - - return { success: true } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/users/:userId", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = parseIntOrUndefined(req.params.userId) - const user = - userId !== undefined ? await getUserById(trx, userId) : null - if (!user) throw new JsonError("No such user", 404) - - user.fullName = req.body.fullName - user.isActive = req.body.isActive - - await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/users/add", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const { email, fullName } = req.body - - await insertUser(trx, { - email, - fullName, - }) - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images").where({ id: imageId }).update({ userId }) - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images") - .where({ id: imageId, userId }) - .update({ userId: null }) - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables.json", - async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 - const query = req.query.search as string - return await searchVariables(query, limit, trx) - } -) - -getRouteWithROTransaction( - apiRouter, - "/chart-bulk-update", - async ( - req, - res, - trx - ): Promise<BulkGrapherConfigResponse<BulkChartEditResponseRow>> => { - const context: OperationContext = { - grapherConfigFieldName: "chart_configs.full", - whitelistedColumnNamesAndTypes: - chartBulkUpdateAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined - - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql - SELECT - charts.id as id, - chart_configs.full as config, - charts.createdAt as createdAt, - charts.updatedAt as updatedAt, - charts.lastEditedAt as lastEditedAt, - charts.publishedAt as publishedAt, - lastEditedByUser.fullName as lastEditedByUser, - publishedByUser.fullName as publishedByUser - FROM charts - LEFT JOIN chart_configs ON chart_configs.id = charts.configId - LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId - WHERE ${whereClause} - ORDER BY charts.id DESC - LIMIT 50 - OFFSET ${offset.toString()} - ` - ) - - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql - SELECT count(*) as count - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - WHERE ${whereClause} - ` - ) - return { rows: results, numTotalRows: resultCount[0].count } - } -) - -patchRouteWithRWTransaction( - apiRouter, - "/chart-bulk-update", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const chartIds = new Set(patchesList.map((patch) => patch.id)) - - const configsAndIds = await db.knexRaw< - Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] } - >( - trx, - `-- sql - SELECT c.id, cc.full as config - FROM charts c - JOIN chart_configs cc ON cc.id = c.configId - WHERE c.id IN (?) - `, - [[...chartIds.values()]] - ) - const configMap = new Map<number, GrapherInterface>( - configsAndIds.map((item: any) => [ - item.id, - // make sure that the id is set, otherwise the update behaviour is weird - // TODO: discuss if this has unintended side effects - item.config ? { ...JSON.parse(item.config), id: item.id } : {}, - ]) - ) - const oldValuesConfigMap = new Map(configMap) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } - - for (const [id, newConfig] of configMap.entries()) { - await saveGrapher(trx, { - user: res.locals.user, - newConfig, - existingConfig: oldValuesConfigMap.get(id), - referencedVariablesMightChange: false, - }) - } - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/variable-annotations", - async ( - req, - res, - trx - ): Promise<BulkGrapherConfigResponse<VariableAnnotationsResponseRow>> => { - const context: OperationContext = { - grapherConfigFieldName: "grapherConfigAdmin", - whitelistedColumnNamesAndTypes: - variableAnnotationAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined - - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql - SELECT - variables.id as id, - variables.name as name, - chart_configs.patch as config, - d.name as datasetname, - namespaces.name as namespacename, - variables.createdAt as createdAt, - variables.updatedAt as updatedAt, - variables.description as description - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ORDER BY variables.id DESC - LIMIT 50 - OFFSET ${offset.toString()} - ` - ) - - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql - SELECT count(*) as count - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ` - ) - return { rows: results, numTotalRows: resultCount[0].count } - } -) - -patchRouteWithRWTransaction( - apiRouter, - "/variable-annotations", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const variableIds = new Set(patchesList.map((patch) => patch.id)) - - const configsAndIds = await db.knexRaw< - Pick<DbRawVariable, "id"> & { - grapherConfigAdmin: DbRawChartConfig["patch"] - } - >( - trx, - `-- sql - SELECT v.id, cc.patch AS grapherConfigAdmin - FROM variables v - LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id - WHERE v.id IN (?) - `, - [[...variableIds.values()]] - ) - const configMap = new Map( - configsAndIds.map((item: any) => [ - item.id, - item.grapherConfigAdmin - ? JSON.parse(item.grapherConfigAdmin) - : {}, - ]) - ) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } - - for (const [variableId, newConfig] of configMap.entries()) { - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) continue - await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) - } - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables.usages.json", - async (req, res, trx) => { - const query = `-- sql - SELECT - variableId, - COUNT(DISTINCT chartId) AS usageCount - FROM - chart_dimensions - GROUP BY - variableId - ORDER BY - usageCount DESC` - - const rows = await db.knexRaw(trx, query) - - return rows - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigETL/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.etl?.patchConfig ?? {} - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigAdmin/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.admin?.patchConfig ?? {} - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables/mergedGrapherConfig/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const config = await getMergedGrapherConfigForVariable(trx, variableId) - return config ?? {} - } -) - -// Used in VariableEditPage -getRouteWithROTransaction( - apiRouter, - "/variables/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" - ) - - // XXX: Patch shortName onto the end of catalogPath when it's missing, - // a temporary hack since our S3 metadata is out of date with our DB. - // See: https://github.com/owid/etl/issues/2135 - if (variable.catalogPath && !variable.catalogPath.includes("#")) { - variable.catalogPath += `#${variable.shortName}` - } - - const rawCharts = await db.knexRaw< - OldChartFieldList & { - isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] - config: DbRawChartConfig["full"] - } - >( - trx, - `-- sql - SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - JOIN chart_dimensions cd ON cd.chartId = charts.id - WHERE cd.variableId = ? - GROUP BY charts.id - `, - [variableId] - ) - - // check for parent indicators - const charts = rawCharts.map((chart) => { - const parentIndicatorId = getParentVariableIdFromChartConfig( - parseChartConfig(chart.config) - ) - const hasParentIndicator = parentIndicatorId !== undefined - return omit({ ...chart, hasParentIndicator }, "config") - }) - - await assignTagsForCharts(trx, charts) - - const variableWithConfigs = await getGrapherConfigsForVariable( - trx, - variableId - ) - const grapherConfigETL = variableWithConfigs?.etl?.patchConfig - const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig - const mergedGrapherConfig = - variableWithConfigs?.admin?.fullConfig ?? - variableWithConfigs?.etl?.fullConfig - - // add the variable's display field to the merged grapher config - if (mergedGrapherConfig) { - const [varDims, otherDims] = lodash.partition( - mergedGrapherConfig.dimensions ?? [], - (dim) => dim.variableId === variableId - ) - const varDimsWithDisplay = varDims.map((dim) => ({ - display: variable.display, - ...dim, - })) - mergedGrapherConfig.dimensions = [ - ...varDimsWithDisplay, - ...otherDims, - ] - } - - const variableWithCharts: OwidVariableWithSource & { - charts: Record<string, any> - grapherConfig: GrapherInterface | undefined - grapherConfigETL: GrapherInterface | undefined - grapherConfigAdmin: GrapherInterface | undefined - } = { - ...variable, - charts, - grapherConfig: mergedGrapherConfig, - grapherConfigETL, - grapherConfigAdmin, - } - - return { - variable: variableWithCharts, - } /*, vardata: await getVariableData([variableId]) }*/ - } -) - -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigETLOfVariable(trx, variable, validConfig) - - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } - - return { success: true, savedPatch } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - // no-op if the variable doesn't have an ETL config - if (!variable.etl) return { success: true } - - const now = new Date() - - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql - UPDATE variables - SET grapherConfigIdETL = NULL - WHERE id = ? - `, - [variableId] - ) - - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql - DELETE FROM chart_configs - WHERE id = ? - `, - [variable.etl.configId] - ) - - // update admin config if there is one - if (variable.admin) { - await updateExistingFullConfig(trx, { - configId: variable.admin.configId, - config: variable.admin.patchConfig, - updatedAt: now, - }) - } - - const updates = { - patchConfigAdmin: variable.admin?.patchConfig, - updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( - trx, - variableId, - updates - ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } - - return { success: true } - } -) - -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) - - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } - - return { success: true, savedPatch } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - - // no-op if the variable doesn't have an admin-authored config - if (!variable.admin) return { success: true } - - const now = new Date() - - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql - UPDATE variables - SET grapherConfigIdAdmin = NULL - WHERE id = ? - `, - [variableId] - ) - - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql - DELETE FROM chart_configs - WHERE id = ? - `, - [variable.admin.configId] - ) - - const updates = { - patchConfigETL: variable.etl?.patchConfig, - updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( - trx, - variableId, - updates - ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables/:variableId/charts.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const charts = await getAllChartsForIndicator(trx, variableId) - return charts.map((chart) => ({ - id: chart.chartId, - title: chart.config.title, - variantName: chart.config.variantName, - isChild: chart.isChild, - isInheritanceEnabled: chart.isInheritanceEnabled, - isPublished: chart.isPublished, - })) - } -) - -getRouteWithROTransaction( - apiRouter, - "/datasets.json", - async (req, res, trx) => { - const datasets = await db.knexRaw<Record<string, any>>( - trx, - `-- sql - WITH variable_counts AS ( - SELECT - v.datasetId, - COUNT(DISTINCT cd.chartId) as numCharts - FROM chart_dimensions cd - JOIN variables v ON cd.variableId = v.id - GROUP BY v.datasetId - ) - SELECT - ad.id, - ad.namespace, - ad.name, - d.shortName, - ad.description, - ad.dataEditedAt, - du.fullName AS dataEditedByUserName, - ad.metadataEditedAt, - mu.fullName AS metadataEditedByUserName, - ad.isPrivate, - ad.nonRedistributable, - d.version, - vc.numCharts - FROM active_datasets ad - LEFT JOIN variable_counts vc ON ad.id = vc.datasetId - JOIN users du ON du.id=ad.dataEditedByUserId - JOIN users mu ON mu.id=ad.metadataEditedByUserId - JOIN datasets d ON d.id=ad.id - ORDER BY ad.dataEditedAt DESC - ` - ) - - const tags = await db.knexRaw< - Pick<DbPlainTag, "id" | "name"> & - Pick<DbPlainDatasetTag, "datasetId"> - >( - trx, - `-- sql - SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt - JOIN tags t ON dt.tagId = t.id - ` - ) - const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) - for (const dataset of datasets) { - dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => - lodash.omit(t, "datasetId") - ) - } - /*LEFT JOIN variables AS v ON v.datasetId=d.id - GROUP BY d.id*/ - - return { datasets: datasets } - } -) - -getRouteWithROTransaction( - apiRouter, - "/datasets/:datasetId.json", - async (req: Request, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await db.knexRawFirst<Record<string, any>>( - trx, - `-- sql - SELECT d.id, - d.namespace, - d.name, - d.shortName, - d.version, - d.description, - d.updatedAt, - d.dataEditedAt, - d.dataEditedByUserId, - du.fullName AS dataEditedByUserName, - d.metadataEditedAt, - d.metadataEditedByUserId, - mu.fullName AS metadataEditedByUserName, - d.isPrivate, - d.isArchived, - d.nonRedistributable, - d.updatePeriodDays - FROM datasets AS d - JOIN users du ON du.id=d.dataEditedByUserId - JOIN users mu ON mu.id=d.metadataEditedByUserId - WHERE d.id = ? - `, - [datasetId] - ) - - if (!dataset) - throw new JsonError(`No dataset by id '${datasetId}'`, 404) - - const zipFile = await db.knexRawFirst<{ filename: string }>( - trx, - `SELECT filename FROM dataset_files WHERE datasetId=?`, - [datasetId] - ) - if (zipFile) dataset.zipFile = zipFile - - const variables = await db.knexRaw< - Pick< - DbRawVariable, - "id" | "name" | "description" | "display" | "catalogPath" - > - >( - trx, - `-- sql - SELECT - v.id, - v.name, - v.description, - v.display, - v.catalogPath - FROM - variables AS v - WHERE - v.datasetId = ? - `, - [datasetId] - ) - - for (const v of variables) { - v.display = JSON.parse(v.display) - } - - dataset.variables = variables - - // add all origins - const origins: DbRawOrigin[] = await db.knexRaw<DbRawOrigin>( - trx, - `-- sql - SELECT DISTINCT - o.* - FROM - origins_variables AS ov - JOIN origins AS o ON ov.originId = o.id - JOIN variables AS v ON ov.variableId = v.id - WHERE - v.datasetId = ? - `, - [datasetId] - ) - - const parsedOrigins = origins.map(parseOriginsRow) - - dataset.origins = parsedOrigins - - const sources = await db.knexRaw<{ - id: number - name: string - description: string - }>( - trx, - ` - SELECT s.id, s.name, s.description - FROM sources AS s - WHERE s.datasetId = ? - ORDER BY s.id ASC - `, - [datasetId] - ) - - // expand description of sources and add to dataset as variableSources - dataset.variableSources = sources.map((s: any) => { - return { - id: s.id, - name: s.name, - ...JSON.parse(s.description), - } - }) - - const charts = await db.knexRaw<OldChartFieldList>( - trx, - `-- sql - SELECT ${oldChartFieldList} - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN chart_dimensions AS cd ON cd.chartId = charts.id - JOIN variables AS v ON cd.variableId = v.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE v.datasetId = ? - GROUP BY charts.id - `, - [datasetId] - ) - - dataset.charts = charts - - await assignTagsForCharts(trx, charts) - - const tags = await db.knexRaw<{ id: number; name: string }>( - trx, - ` - SELECT t.id, t.name - FROM tags t - JOIN dataset_tags dt ON dt.tagId = t.id - WHERE dt.datasetId = ? - `, - [datasetId] - ) - dataset.tags = tags - - const availableTags = await db.knexRaw<{ - id: number - name: string - parentName: string - }>( - trx, - ` - SELECT t.id, t.name, p.name AS parentName - FROM tags AS t - JOIN tags AS p ON t.parentId=p.id - ` - ) - dataset.availableTags = availableTags - - return { dataset: dataset } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - // Only updates `nonRedistributable` and `tags`, other fields come from ETL - // and are not editable - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - const newDataset = (req.body as { dataset: any }).dataset - await db.knexRaw( - trx, - ` - UPDATE datasets - SET - nonRedistributable=?, - metadataEditedAt=?, - metadataEditedByUserId=? - WHERE id=? - `, - [ - newDataset.nonRedistributable, - new Date(), - res.locals.user.id, - datasetId, - ] - ) - - const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) - await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ - datasetId, - ]) - if (tagRows.length) - for (const tagRow of tagRows) { - await db.knexRaw( - trx, - `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, - tagRow - ) - } - - try { - await syncDatasetToGitRepo(trx, datasetId, { - oldDatasetName: dataset.name, - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue - } - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setArchived", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ - datasetId, - ]) - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setTags", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - await setTagsForDataset(trx, datasetId, req.body.tagIds) - - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - await db.knexRaw( - trx, - `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, - [datasetId] - ) - await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) - - try { - await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err: any) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue - } - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/charts", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - if (req.body.republish) { - await db.knexRaw( - trx, - `-- sql - UPDATE chart_configs cc - JOIN charts c ON c.configId = cc.id - SET - cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), - cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) - WHERE c.id IN ( - SELECT DISTINCT chart_dimensions.chartId - FROM chart_dimensions - JOIN variables ON variables.id = chart_dimensions.variableId - WHERE variables.datasetId = ? - )`, - [datasetId] - ) - } - - await triggerStaticBuild( - res.locals.user, - `Republishing all charts in dataset ${dataset.name} (${dataset.id})` - ) - - return { success: true } - } -) - -// Get a list of redirects that map old slugs to charts -getRouteWithROTransaction( - apiRouter, - "/redirects.json", - async (req, res, trx) => ({ - redirects: await db.knexRaw( - trx, - `-- sql - SELECT - r.id, - r.slug, - r.chart_id as chartId, - chart_configs.slug AS chartSlug - FROM chart_slug_redirects AS r - JOIN charts ON charts.id = r.chart_id - JOIN chart_configs ON chart_configs.id = charts.configId - ORDER BY r.id DESC - ` - ), - }) -) - -getRouteWithROTransaction( - apiRouter, - "/site-redirects.json", - async (req, res, trx) => ({ redirects: await getRedirects(trx) }) -) - -postRouteWithRWTransaction( - apiRouter, - "/site-redirects/new", - async (req: Request, res, trx) => { - const { source, target } = req.body - const sourceAsUrl = new URL(source, "https://ourworldindata.org") - if (sourceAsUrl.pathname === "/") - throw new JsonError("Cannot redirect from /", 400) - if (await redirectWithSourceExists(trx, source)) { - throw new JsonError( - `Redirect with source ${source} already exists`, - 400 - ) - } - const chainedRedirect = await getChainedRedirect(trx, source, target) - if (chainedRedirect) { - throw new JsonError( - "Creating this redirect would create a chain, redirect from " + - `${chainedRedirect.source} to ${chainedRedirect.target} ` + - "already exists. " + - (target === chainedRedirect.source - ? `Please create the redirect from ${source} to ` + - `${chainedRedirect.target} directly instead.` - : `Please delete the existing redirect and create a ` + - `new redirect from ${chainedRedirect.source} to ` + - `${target} instead.`), - 400 - ) - } - const { insertId: id } = await db.knexRawInsert( - trx, - `INSERT INTO redirects (source, target) VALUES (?, ?)`, - [source, target] - ) - await triggerStaticBuild( - res.locals.user, - `Creating redirect id=${id} source=${source} target=${target}` - ) - return { success: true, redirect: { id, source, target } } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/site-redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const redirect = await getRedirectById(trx, id) - if (!redirect) { - throw new JsonError(`No redirect found for id ${id}`, 404) - } - await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` - ) - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/tags/:tagId.json", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) as number | null - - // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff - // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag - // every time we create a new chart etcs - const uncategorized = tagId === UNCATEGORIZED_TAG_ID - - // TODO: when we have types for our endpoints, make tag of that type instead of any - const tag: any = await db.knexRawFirst< - Pick< - DbPlainTag, - | "id" - | "name" - | "specialType" - | "updatedAt" - | "parentId" - | "slug" - > - >( - trx, - `-- sql - SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug - FROM tags t LEFT JOIN tags p ON t.parentId=p.id - WHERE t.id = ? - `, - [tagId] - ) - - // Datasets tagged with this tag - const datasets = await db.knexRaw< - Pick< - DbPlainDataset, - | "id" - | "namespace" - | "name" - | "description" - | "createdAt" - | "updatedAt" - | "dataEditedAt" - | "isPrivate" - | "nonRedistributable" - > & { dataEditedByUserName: string } - >( - trx, - `-- sql - SELECT - d.id, - d.namespace, - d.name, - d.description, - d.createdAt, - d.updatedAt, - d.dataEditedAt, - du.fullName AS dataEditedByUserName, - d.isPrivate, - d.nonRedistributable - FROM active_datasets d - JOIN users du ON du.id=d.dataEditedByUserId - LEFT JOIN dataset_tags dt ON dt.datasetId = d.id - WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} - ORDER BY d.dataEditedAt DESC - `, - uncategorized ? [] : [tagId] - ) - tag.datasets = datasets - - // The other tags for those datasets - if (tag.datasets.length) { - if (uncategorized) { - for (const dataset of tag.datasets) dataset.tags = [] - } else { - const datasetTags = await db.knexRaw<{ - datasetId: number - id: number - name: string - }>( - trx, - `-- sql - SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt - JOIN tags t ON dt.tagId = t.id - WHERE dt.datasetId IN (?) - `, - [tag.datasets.map((d: any) => d.id)] - ) - const tagsByDatasetId = lodash.groupBy( - datasetTags, - (t) => t.datasetId - ) - for (const dataset of tag.datasets) { - dataset.tags = tagsByDatasetId[dataset.id].map((t) => - lodash.omit(t, "datasetId") - ) - } - } - } - - // Charts using datasets under this tag - const charts = await db.knexRaw<OldChartFieldList>( - trx, - `-- sql - SELECT ${oldChartFieldList} FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - LEFT JOIN chart_tags ct ON ct.chartId=charts.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"} - GROUP BY charts.id - ORDER BY charts.updatedAt DESC - `, - uncategorized ? [] : [tagId] - ) - tag.charts = charts - - await assignTagsForCharts(trx, charts) - - // Subcategories - const children = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql - SELECT t.id, t.name FROM tags t - WHERE t.parentId = ? - `, - [tag.id] - ) - tag.children = children - - // Possible parents to choose from - const possibleParents = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql - SELECT t.id, t.name FROM tags t - WHERE t.parentId IS NULL - ` - ) - tag.possibleParents = possibleParents - - return { - tag, - } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/tags/:tagId", - async (req: Request, res, trx) => { - const tagId = expectInt(req.params.tagId) - const tag = (req.body as { tag: any }).tag - await db.knexRaw( - trx, - `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, - [tag.name, new Date(), tag.slug, tagId] - ) - if (tag.slug) { - // See if there's a published gdoc with a matching slug. - // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, - // where the page for the topic is just an article. - const gdoc = await db.knexRaw<Pick<DbRawPostGdoc, "slug">>( - trx, - `-- sql - SELECT slug FROM posts_gdocs pg - WHERE EXISTS ( - SELECT 1 - FROM posts_gdocs_x_tags gt - WHERE pg.id = gt.gdocId AND gt.tagId = ? - ) AND pg.published = TRUE AND pg.slug = ?`, - [tagId, tag.slug] - ) - if (!gdoc.length) { - return { - success: true, - tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. - -Are you sure you haven't made a typo?`, - } - } - } - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/tags/new", - async (req: Request, res, trx) => { - const tag = req.body - function validateTag( - tag: unknown - ): tag is { name: string; slug: string | null } { - return ( - checkIsPlainObjectWithGuard(tag) && - typeof tag.name === "string" && - (tag.slug === null || - (typeof tag.slug === "string" && tag.slug !== "")) - ) - } - if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) - - const conflictingTag = await db.knexRawFirst<{ - name: string - slug: string | null - }>( - trx, - `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, - [tag.name, tag.slug] - ) - if (conflictingTag) - throw new JsonError( - conflictingTag.name === tag.name - ? `Tag with name ${tag.name} already exists` - : `Tag with slug ${tag.slug} already exists`, - 400 - ) - - const now = new Date() - const result = await db.knexRawInsert( - trx, - `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, - // parentId will be deprecated soon once we migrate fully to the tag graph - [tag.name, tag.slug, now, now] - ) - return { success: true, tagId: result.insertId } - } -) - -getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { - return { tags: await db.getMinimalTagsWithIsTopic(trx) } -}) - -deleteRouteWithRWTransaction( - apiRouter, - "/tags/:tagId/delete", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) - - await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/redirects/new", - async (req: Request, res, trx) => { - const chartId = expectInt(req.params.chartId) - const fields = req.body as { slug: string } - const result = await db.knexRawInsert( - trx, - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [chartId, fields.slug] - ) - const redirectId = result.insertId - const redirect = await db.knexRaw<DbPlainChartSlugRedirect>( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [redirectId] - ) - return { success: true, redirect: redirect } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - const redirect = await db.knexRawFirst<DbPlainChartSlugRedirect>( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [id] - ) - - if (!redirect) - throw new JsonError(`No redirect found for id ${id}`, 404) - - await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [ - id, - ]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect from ${redirect.slug}` - ) - - return { success: true } - } -) - -getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { - const raw_rows = await db.knexRaw( - trx, - `-- sql - WITH - posts_tags_aggregated AS ( - SELECT - post_id, - IF( - COUNT(tags.id) = 0, - JSON_ARRAY(), - JSON_ARRAYAGG(JSON_OBJECT("id", tags.id, "name", tags.name)) - ) AS tags - FROM - post_tags - LEFT JOIN tags ON tags.id = post_tags.tag_id - GROUP BY - post_id - ), - post_gdoc_slug_successors AS ( - SELECT - posts.id, - IF( - COUNT(gdocSlugSuccessor.id) = 0, - JSON_ARRAY(), - JSON_ARRAYAGG( - JSON_OBJECT("id", gdocSlugSuccessor.id, "published", gdocSlugSuccessor.published) - ) - ) AS gdocSlugSuccessors - FROM - posts - LEFT JOIN posts_gdocs gdocSlugSuccessor ON gdocSlugSuccessor.slug = posts.slug - GROUP BY - posts.id - ) - SELECT - posts.id AS id, - posts.title AS title, - posts.type AS TYPE, - posts.slug AS slug, - STATUS, - updated_at_in_wordpress, - posts.authors, - posts_tags_aggregated.tags AS tags, - gdocSuccessorId, - gdocSuccessor.published AS isGdocSuccessorPublished, - -- posts can either have explict successors via the gdocSuccessorId column - -- or implicit successors if a gdoc has been created that uses the same slug - -- as a Wp post (the gdoc one wins once it is published) - post_gdoc_slug_successors.gdocSlugSuccessors AS gdocSlugSuccessors - FROM - posts - LEFT JOIN post_gdoc_slug_successors ON post_gdoc_slug_successors.id = posts.id - LEFT JOIN posts_gdocs gdocSuccessor ON gdocSuccessor.id = posts.gdocSuccessorId - LEFT JOIN posts_tags_aggregated ON posts_tags_aggregated.post_id = posts.id - ORDER BY - updated_at_in_wordpress DESC`, - [] - ) - const rows = raw_rows.map((row: any) => ({ - ...row, - tags: JSON.parse(row.tags), - isGdocSuccessorPublished: !!row.isGdocSuccessorPublished, - gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors), - authors: JSON.parse(row.authors), - })) - - return { posts: rows } -}) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/setTags", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - - await setTagsForPost(trx, postId, req.body.tagIds) - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/posts/:postId.json", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx - .table(PostsTableName) - .where({ id: postId }) - .select("*") - .first()) as DbRawPost | undefined - return camelCaseProperties({ ...post }) - } -) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/createGdoc", - async (req: Request, res, trx) => { - const postId = expectInt(req.params.postId) - const allowRecreate = !!req.body.allowRecreate - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined - - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!allowRecreate && existingGdocId) - throw new JsonError("A gdoc already exists for this post", 400) - if (allowRecreate && existingGdocId && post.isGdocPublished) { - throw new JsonError( - "A gdoc already exists for this post and it is already published", - 400 - ) - } - if (post.archieml === null) - throw new JsonError( - `ArchieML was not present for post with id ${postId}`, - 500 - ) - const tagsByPostId = await getTagsByPostId(trx) - const tags = tagsByPostId.get(postId) || [] - const archieMl = JSON.parse( - // Google Docs interprets ®ion in grapher URLS as ®ion - // So we escape them here - post.archieml.replaceAll("&", "&") - ) as OwidGdocPostInterface - const gdocId = await createGdocAndInsertOwidGdocPostContent( - archieMl.content, - post.gdocSuccessorId - ) - // If we did not yet have a gdoc associated with this post, we need to register - // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise - // we don't need to make changes to the DB (only the gdoc regeneration was required) - if (!existingGdocId) { - post.gdocSuccessorId = gdocId - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", gdocId) - - const gdoc = new GdocPost(gdocId) - gdoc.slug = post.slug - gdoc.content.title = post.title - gdoc.content.type = archieMl.content.type || OwidGdocType.Article - gdoc.published = false - gdoc.createdAt = new Date() - gdoc.publishedAt = post.published_at - await upsertGdoc(trx, gdoc) - await setTagsForGdoc(trx, gdocId, tags) - } - return { googleDocsId: gdocId } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/unlinkGdoc", - async (req: Request, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined - - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!existingGdocId) - throw new JsonError("No gdoc exists for this post", 400) - if (existingGdocId && post.isGdocPublished) { - throw new JsonError( - "The GDoc is already published - you can't unlink it", - 400 - ) - } - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", null) - - await trx - .table(PostsGdocsTableName) - .where({ id: existingGdocId }) - .delete() - - return { success: true } - } -) - -getRouteWithROTransaction( - apiRouter, - "/sources/:sourceId.json", - async (req: Request, res, trx) => { - const sourceId = expectInt(req.params.sourceId) - - const source = await db.knexRawFirst<Record<string, any>>( - trx, - ` - SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace - FROM sources AS s - JOIN active_datasets AS d ON d.id=s.datasetId - WHERE s.id=?`, - [sourceId] - ) - if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) - source.variables = await db.knexRaw( - trx, - `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, - [sourceId] - ) - - return { source: source } - } -) - -apiRouter.get("/deploys.json", async () => ({ - deploys: await new DeployQueueServer().getDeploys(), -})) - -apiRouter.put("/deploy", async (req, res) => { - return triggerStaticBuild(res.locals.user, "Manually triggered deploy") -}) - -getRouteWithROTransaction(apiRouter, "/gdocs", (req, res, trx) => { - return getAllGdocIndexItemsOrderedByUpdatedAt(trx) -}) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/gdocs/:id", - async (req, res, trx) => { - const id = req.params.id - const contentSource = req.query.contentSource as - | GdocsContentSource - | undefined - - try { - // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published - const gdoc = await getAndLoadGdocById(trx, id, contentSource) - - if (!gdoc.published) { - await updateGdocContentOnly(trx, id, gdoc) - } - - res.set("Cache-Control", "no-store") - res.send(gdoc) - } catch (error) { - console.error("Error fetching gdoc", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } - } -) - -/** - * Handles all four `GdocPublishingAction` cases - * - SavingDraft (no action) - * - Publishing (index and bake) - * - Updating (index and bake (potentially via lightning deploy)) - * - Unpublishing (remove from index and bake) - */ -async function indexAndBakeGdocIfNeccesary( - trx: db.KnexReadWriteTransaction, - user: Required<DbInsertUser>, - prevGdoc: - | GdocPost - | GdocDataInsight - | GdocHomepage - | GdocAbout - | GdocAuthor, - nextGdoc: GdocPost | GdocDataInsight | GdocHomepage | GdocAbout | GdocAuthor -) { - const prevJson = prevGdoc.toJSON() - const nextJson = nextGdoc.toJSON() - const hasChanges = checkHasChanges(prevGdoc, nextGdoc) - const action = getPublishingAction(prevJson, nextJson) - const isGdocPost = checkIsGdocPostExcludingFragments(nextJson) - - await match(action) - .with(GdocPublishingAction.SavingDraft, lodash.noop) - .with(GdocPublishingAction.Publishing, async () => { - if (isGdocPost) { - await indexIndividualGdocPost( - nextJson, - trx, - // If the gdoc is being published for the first time, prevGdoc.slug will be undefined - // In that case, we pass nextJson.slug to see if it has any page views (i.e. from WP) - prevGdoc.slug || nextJson.slug - ) - } - await triggerStaticBuild(user, `${action} ${nextJson.slug}`) - }) - .with(GdocPublishingAction.Updating, async () => { - if (isGdocPost) { - await indexIndividualGdocPost(nextJson, trx, prevGdoc.slug) - } - if (checkIsLightningUpdate(prevJson, nextJson, hasChanges)) { - await enqueueLightningChange( - user, - `Lightning update ${nextJson.slug}`, - nextJson.slug - ) - } else { - await triggerStaticBuild(user, `${action} ${nextJson.slug}`) - } - }) - .with(GdocPublishingAction.Unpublishing, async () => { - if (isGdocPost) { - await removeIndividualGdocPostFromIndex(nextJson) - } - await triggerStaticBuild(user, `${action} ${nextJson.slug}`) - }) - .exhaustive() -} - -/** - * Only supports creating a new empty Gdoc or updating an existing one. Does not - * support creating a new Gdoc from an existing one. Relevant updates will - * trigger a deploy. - */ -putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { - const { id } = req.params - - if (isEmpty(req.body)) { - return createOrLoadGdocById(trx, id) - } - - const prevGdoc = await getAndLoadGdocById(trx, id) - if (!prevGdoc) throw new JsonError(`No Google Doc with id ${id} found`) - - const nextGdoc = gdocFromJSON(req.body) - await nextGdoc.loadState(trx) - - await addImagesToContentGraph(trx, nextGdoc) - - await setLinksForGdoc( - trx, - nextGdoc.id, - nextGdoc.links, - nextGdoc.published - ? GdocLinkUpdateMode.DeleteAndInsert - : GdocLinkUpdateMode.DeleteOnly - ) - - await upsertGdoc(trx, nextGdoc) - - await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc) - - return nextGdoc -}) - -async function validateTombstoneRelatedLinkUrl( - trx: db.KnexReadonlyTransaction, - relatedLink?: string -) { - if (!relatedLink || !relatedLink.startsWith(GDOCS_BASE_URL)) return - const id = relatedLink.match(gdocUrlRegex)?.[1] - if (!id) { - throw new JsonError(`Invalid related link: ${relatedLink}`) - } - const [gdoc] = await getMinimalGdocPostsByIds(trx, [id]) - if (!gdoc) { - throw new JsonError(`Google Doc with ID ${id} not found`) - } - if (!gdoc.published) { - throw new JsonError(`Google Doc with ID ${id} is not published`) - } -} - -deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { - const { id } = req.params - - const gdoc = await getGdocBaseObjectById(trx, id, false) - if (!gdoc) throw new JsonError(`No Google Doc with id ${id} found`) - - const gdocSlug = getCanonicalUrl("", gdoc) - const { tombstone } = req.body - - if (tombstone) { - await validateTombstoneRelatedLinkUrl(trx, tombstone.relatedLinkUrl) - const slug = gdocSlug.replace("/", "") - const { relatedLinkThumbnail } = tombstone - if (relatedLinkThumbnail) { - const thumbnailExists = await db.checkIsImageInDB( - trx, - relatedLinkThumbnail - ) - if (!thumbnailExists) { - throw new JsonError( - `Image with filename "${relatedLinkThumbnail}" not found` - ) - } - } - await trx - .table("posts_gdocs_tombstones") - .insert({ ...tombstone, gdocId: id, slug }) - await trx - .table("redirects") - .insert({ source: gdocSlug, target: `/deleted${gdocSlug}` }) - } - - await trx - .table("posts") - .where({ gdocSuccessorId: gdoc.id }) - .update({ gdocSuccessorId: null }) - - await trx.table(PostsGdocsLinksTableName).where({ sourceId: id }).delete() - await trx.table(PostsGdocsXImagesTableName).where({ gdocId: id }).delete() - await trx.table(PostsGdocsTableName).where({ id }).delete() - await trx - .table(PostsGdocsComponentsTableName) - .where({ gdocId: id }) - .delete() - if (gdoc.published && checkIsGdocPostExcludingFragments(gdoc)) { - await removeIndividualGdocPostFromIndex(gdoc) - } - if (gdoc.published) { - if (!tombstone && gdocSlug && gdocSlug !== "/") { - // Assets have TTL of one week in Cloudflare. Add a redirect to make sure - // the page is no longer accessible. - // https://developers.cloudflare.com/pages/configuration/serving-pages/#asset-retention - console.log(`Creating redirect for "${gdocSlug}" to "/"`) - await db.knexRawInsert( - trx, - `INSERT INTO redirects (source, target, ttl) - VALUES (?, ?, DATE_ADD(NOW(), INTERVAL 8 DAY))`, - [gdocSlug, "/"] - ) - } - await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`) - } - return {} -}) - -postRouteWithRWTransaction( - apiRouter, - "/gdocs/:gdocId/setTags", - async (req, res, trx) => { - const { gdocId } = req.params - const { tagIds } = req.body - const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ - id: id, - })) - - await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) - - return { success: true } - } -) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/images.json", - async (_, res, trx) => { - try { - const images = await db.getCloudflareImages(trx) - res.set("Cache-Control", "no-store") - res.send({ images }) - } catch (error) { - console.error("Error fetching images", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } - } -) - -postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { - const { filename, type, content } = validateImagePayload(req.body) - - const { asBlob, dimensions, hash } = await processImageContent( - content, - type - ) - - const collision = await trx<DbEnrichedImage>("images") - .where({ - hash, - replacedBy: null, - }) - .first() - - if (collision) { - return { - success: false, - error: `An image with this content already exists (filename: ${collision.filename})`, - } - } - - const preexisting = await trx<DbEnrichedImage>("images") - .where("filename", "=", filename) - .first() - - if (preexisting) { - return { - success: false, - error: "An image with this filename already exists", - } - } - - const cloudflareId = await uploadToCloudflare(filename, asBlob) - - if (!cloudflareId) { - return { - success: false, - error: "Failed to upload image", - } - } - - await trx<DbEnrichedImage>("images").insert({ - filename, - originalWidth: dimensions.width, - originalHeight: dimensions.height, - cloudflareId, - updatedAt: new Date().getTime(), - userId: res.locals.user.id, - hash, - }) - - const image = await db.getCloudflareImage(trx, filename) - - return { - success: true, - image, - } -}) - -/** - * Similar to the POST route, but for updating an existing image. - * Creates a new image entry in the database and uploads the new image to Cloudflare. - * The old image is marked as replaced by the new image. - */ -putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { - const { type, content } = validateImagePayload(req.body) - const { asBlob, dimensions, hash } = await processImageContent( - content, - type - ) - const collision = await trx<DbEnrichedImage>("images") - .where({ - hash, - replacedBy: null, - }) - .first() - - if (collision) { - return { - success: false, - error: `An exact copy of this image already exists (filename: ${collision.filename})`, - } - } - - const { id } = req.params - - const image = await trx<DbEnrichedImage>("images") - .where("id", "=", id) - .first() - - if (!image) { - throw new JsonError(`No image found for id ${id}`, 404) - } - - const originalCloudflareId = image.cloudflareId - const originalFilename = image.filename - const originalAltText = image.defaultAlt - - if (!originalCloudflareId) { - throw new JsonError( - `Image with id ${id} has no associated Cloudflare image`, - 400 - ) - } - - const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob) - - if (!newCloudflareId) { - throw new JsonError("Failed to upload image", 500) - } - - const [newImageId] = await trx<DbEnrichedImage>("images").insert({ - filename: originalFilename, - originalWidth: dimensions.width, - originalHeight: dimensions.height, - cloudflareId: newCloudflareId, - updatedAt: new Date().getTime(), - userId: res.locals.user.id, - defaultAlt: originalAltText, - hash, - version: image.version + 1, - }) - - await trx<DbEnrichedImage>("images").where("id", "=", id).update({ - replacedBy: newImageId, - }) - - const updated = await db.getCloudflareImage(trx, originalFilename) - - await triggerStaticBuild( - res.locals.user, - `Updating image "${originalFilename}"` - ) - - return { - success: true, - image: updated, - } -}) - -// Update alt text via patch -patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { - const { id } = req.params - - const image = await trx<DbEnrichedImage>("images") - .where("id", "=", id) - .first() - - if (!image) { - throw new JsonError(`No image found for id ${id}`, 404) - } - - const patchableImageProperties = ["defaultAlt"] as const - const patch = lodash.pick(req.body, patchableImageProperties) - - if (Object.keys(patch).length === 0) { - throw new JsonError("No patchable properties provided", 400) - } - - await trx("images").where({ id }).update(patch) - - const updated = await trx<DbEnrichedImage>("images") - .where("id", "=", id) - .first() - - return { - success: true, - image: updated, - } -}) - -deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { - const { id } = req.params - - const image = await trx<DbEnrichedImage>("images") - .where("id", "=", id) - .first() - - if (!image) { - throw new JsonError(`No image found for id ${id}`, 404) - } - if (!image.cloudflareId) { - throw new JsonError(`Image does not have a cloudflare ID`, 400) - } - - const replacementChain = await db.selectReplacementChainForImage(trx, id) - - await pMap( - replacementChain, - async (image) => { - if (image.cloudflareId) { - await deleteFromCloudflare(image.cloudflareId) - } - }, - { concurrency: 5 } - ) - - // There's an ON DELETE CASCADE which will delete the replacements - await trx("images").where({ id }).delete() - - return { - success: true, - } -}) - -getRouteWithROTransaction(apiRouter, "/images/usage", async (_, __, trx) => { - const usage = await db.getImageUsage(trx) - - return { - success: true, - usage, - } -}) - -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, - async ( - req: Request, - res, - trx - ): Promise<Record<"topics", DbChartTagJoin[]>> => { - const chartId = parseIntOrUndefined(req.params.chartId) - if (!chartId) throw new JsonError(`Invalid chart ID`, 400) - - const topics = await getGptTopicSuggestions(trx, chartId) - - if (!topics.length) - throw new JsonError( - `No GPT topic suggestions found for chart ${chartId}`, - 404 - ) - - return { - topics, - } - } -) - -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-alt-text/:imageId`, - async ( - req: Request, - res, - trx - ): Promise<{ - success: boolean - altText: string | null - }> => { - const imageId = parseIntOrUndefined(req.params.imageId) - if (!imageId) throw new JsonError(`Invalid image ID`, 400) - const image = await trx<DbEnrichedImage>("images") - .where("id", imageId) - .first() - if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) - - const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` - let altText: string | null = "" - try { - altText = await fetchGptGeneratedAltText(src) - } catch (error) { - console.error( - `Error fetching GPT alt text for image ${imageId}`, - error - ) - throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) - } - - if (!altText) { - throw new JsonError(`Unable to generate alt text for image`, 404) - } - - return { success: true, altText } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - async (req, res, trx) => { - const { slug } = req.params - const { tagIds } = req.body - const explorer = await trx.table("explorers").where({ slug }).first() - if (!explorer) - throw new JsonError(`No explorer found for slug ${slug}`, 404) - - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - for (const tagId of tagIds) { - await trx - .table("explorer_tags") - .insert({ explorerSlug: slug, tagId }) - } - - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - async (req: Request, res, trx) => { - const { slug } = req.params - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - return { success: true } - } -) - -// Get an ArchieML output of all the work produced by an author. This includes -// gdoc articles, gdoc modular/linear topic pages and wordpress modular topic -// pages. Data insights are excluded. This is used to manually populate the -// [.secondary] section of the {.research-and-writing} block of author pages -// using the alternate template, which highlights topics rather than articles. -getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { - type WordpressPageRecord = { - isWordpressPage: number - } & Record< - "slug" | "title" | "subtitle" | "thumbnail" | "authors" | "publishedAt", - string - > - type GdocRecord = Pick<DbRawPostGdoc, "id" | "publishedAt"> - - const author = req.query.author - const gdocs = await db.knexRaw<GdocRecord>( - trx, - `-- sql - SELECT id, publishedAt - FROM posts_gdocs - WHERE JSON_CONTAINS(content->'$.authors', '"${author}"') - AND type NOT IN ("data-insight", "fragment") - AND published = 1 - ` - ) - - // type: page - const wpModularTopicPages = await db.knexRaw<WordpressPageRecord>( - trx, - `-- sql - SELECT - wpApiSnapshot->>"$.slug" as slug, - wpApiSnapshot->>"$.title.rendered" as title, - wpApiSnapshot->>"$.excerpt.rendered" as subtitle, - TRUE as isWordpressPage, - wpApiSnapshot->>"$.authors_name" as authors, - wpApiSnapshot->>"$.featured_media_paths.medium_large" as thumbnail, - wpApiSnapshot->>"$.date" as publishedAt - FROM posts p - WHERE wpApiSnapshot->>"$.content" LIKE '%topic-page%' - AND JSON_CONTAINS(wpApiSnapshot->'$.authors_name', '"${author}"') - AND wpApiSnapshot->>"$.status" = 'publish' - AND NOT EXISTS ( - SELECT 1 FROM posts_gdocs pg - WHERE pg.slug = p.slug - AND pg.content->>'$.type' LIKE '%topic-page' - ) - ` - ) - - const isWordpressPage = ( - post: WordpressPageRecord | GdocRecord - ): post is WordpressPageRecord => - (post as WordpressPageRecord).isWordpressPage === 1 - - function* generateProperty(key: string, value: string) { - yield `${key}: ${value}\n` - } - - const sortByDateDesc = ( - a: GdocRecord | WordpressPageRecord, - b: GdocRecord | WordpressPageRecord - ): number => { - if (!a.publishedAt || !b.publishedAt) return 0 - return ( - new Date(b.publishedAt).getTime() - - new Date(a.publishedAt).getTime() - ) - } - - function* generateAllWorkArchieMl() { - for (const post of [...gdocs, ...wpModularTopicPages].sort( - sortByDateDesc - )) { - if (isWordpressPage(post)) { - yield* generateProperty( - "url", - `https://ourworldindata.org/${post.slug}` - ) - yield* generateProperty("title", post.title) - yield* generateProperty("subtitle", post.subtitle) - yield* generateProperty( - "authors", - JSON.parse(post.authors).join(", ") - ) - const parsedPath = path.parse(post.thumbnail) - yield* generateProperty( - "filename", - // /app/uploads/2021/09/reducing-fertilizer-768x301.png -> reducing-fertilizer.png - path.format({ - name: parsedPath.name.replace(/-\d+x\d+$/, ""), - ext: parsedPath.ext, - }) - ) - yield "\n" - } else { - // this is a gdoc - yield* generateProperty( - "url", - `https://docs.google.com/document/d/${post.id}/edit` - ) - yield "\n" - } - } - } - - res.type("text/plain") - return [...generateAllWorkArchieMl()].join("") -}) - -getRouteWithROTransaction( - apiRouter, - "/flatTagGraph.json", - async (req, res, trx) => { - const flatTagGraph = await db.getFlatTagGraph(trx) - return flatTagGraph - } -) - -postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { - const tagGraph = req.body?.tagGraph as unknown - if (!tagGraph) { - throw new JsonError("No tagGraph provided", 400) - } - - function validateFlatTagGraph( - tagGraph: Record<any, any> - ): tagGraph is FlatTagGraph { - if (lodash.isObject(tagGraph)) { - for (const [key, value] of Object.entries(tagGraph)) { - if (!lodash.isString(key) && isNaN(Number(key))) { - return false - } - if (!lodash.isArray(value)) { - return false - } - for (const tag of value) { - if ( - !( - checkIsPlainObjectWithGuard(tag) && - lodash.isNumber(tag.weight) && - lodash.isNumber(tag.parentId) && - lodash.isNumber(tag.childId) - ) - ) { - return false - } - } - } - } - - return true - } - const isValid = validateFlatTagGraph(tagGraph) - if (!isValid) { - throw new JsonError("Invalid tag graph provided", 400) - } - await db.updateTagGraph(trx, tagGraph) - res.send({ success: true }) -}) - -const createPatchConfigAndQueryParamsForChartView = async ( - knex: db.KnexReadonlyTransaction, - parentChartId: number, - config: GrapherInterface -) => { - const parentChartConfig = await expectChartById(knex, parentChartId) - - config = omit(config, CHART_VIEW_PROPS_TO_OMIT) - - const patchToParentChart = diffGrapherConfigs(config, parentChartConfig) - - const fullConfigIncludingDefaults = mergeGrapherConfigs( - defaultGrapherConfig, - config - ) - const patchConfigToSave = { - ...patchToParentChart, - - // We want to make sure we're explicitly persisting some props like entity selection - // always, so they never change when the parent chart changes. - // For this, we need to ensure we include the default layer, so that we even - // persist these props when they are the same as the default. - ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST), - } - - const queryParams = grapherConfigToQueryParams(config) - - const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave) - return { patchConfig: patchConfigToSave, fullConfig, queryParams } -} - -getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { - type ChartViewRow = Pick<DbPlainChartView, "id" | "name" | "updatedAt"> & { - lastEditedByUser: string - chartConfigId: string - title: string - parentChartId: number - parentTitle: string - } - - const rows: ChartViewRow[] = await db.knexRaw( - trx, - `-- sql - SELECT - cv.id, - cv.name, - cv.updatedAt, - u.fullName as lastEditedByUser, - cv.chartConfigId, - cc.full ->> "$.title" as title, - cv.parentChartId, - pcc.full ->> "$.title" as parentTitle - FROM chart_views cv - JOIN chart_configs cc ON cv.chartConfigId = cc.id - JOIN charts pc ON cv.parentChartId = pc.id - JOIN chart_configs pcc ON pc.configId = pcc.id - JOIN users u ON cv.lastEditedByUserId = u.id - ORDER BY cv.updatedAt DESC - ` - ) - - const chartViews: ApiChartViewOverview[] = rows.map((row) => ({ - id: row.id, - name: row.name, - updatedAt: row.updatedAt?.toISOString() ?? null, - lastEditedByUser: row.lastEditedByUser, - chartConfigId: row.chartConfigId, - title: row.title, - parent: { - id: row.parentChartId, - title: row.parentTitle, - }, - })) - - return { chartViews } -}) - -getRouteWithROTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - type ChartViewRow = Pick< - DbPlainChartView, - "id" | "name" | "updatedAt" - > & { - lastEditedByUser: string - chartConfigId: string - configFull: JsonString - configPatch: JsonString - parentChartId: number - parentConfigFull: JsonString - queryParamsForParentChart: JsonString - } - - const row = await db.knexRawFirst<ChartViewRow>( - trx, - `-- sql - SELECT - cv.id, - cv.name, - cv.updatedAt, - u.fullName as lastEditedByUser, - cv.chartConfigId, - cc.full as configFull, - cc.patch as configPatch, - cv.parentChartId, - pcc.full as parentConfigFull, - cv.queryParamsForParentChart - FROM chart_views cv - JOIN chart_configs cc ON cv.chartConfigId = cc.id - JOIN charts pc ON cv.parentChartId = pc.id - JOIN chart_configs pcc ON pc.configId = pcc.id - JOIN users u ON cv.lastEditedByUserId = u.id - WHERE cv.id = ? - `, - [id] - ) - - if (!row) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const chartView = { - ...row, - configFull: parseChartConfig(row.configFull), - configPatch: parseChartConfig(row.configPatch), - parentConfigFull: parseChartConfig(row.parentConfigFull), - queryParamsForParentChart: JSON.parse( - row.queryParamsForParentChart - ), - } - - return chartView - } -) - -postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { - const { name, parentChartId } = req.body as Pick< - DbPlainChartView, - "name" | "parentChartId" - > - const rawConfig = req.body.config as GrapherInterface - if (!name || !parentChartId || !rawConfig) { - throw new JsonError("Invalid request", 400) - } - - const { patchConfig, fullConfig, queryParams } = - await createPatchConfigAndQueryParamsForChartView( - trx, - parentChartId, - rawConfig - ) - - const { chartConfigId } = await saveNewChartConfigInDbAndR2( - trx, - undefined, - patchConfig, - fullConfig - ) - - // insert into chart_views - const insertRow: DbInsertChartView = { - name, - parentChartId, - lastEditedByUserId: res.locals.user.id, - chartConfigId: chartConfigId, - queryParamsForParentChart: JSON.stringify(queryParams), - } - const result = await trx.table(ChartViewsTableName).insert(insertRow) - const [resultId] = result - - return { chartViewId: resultId, success: true } -}) - -putRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const rawConfig = req.body.config as GrapherInterface - if (!rawConfig) { - throw new JsonError("Invalid request", 400) - } - - const existingRow: Pick< - DbPlainChartView, - "chartConfigId" | "parentChartId" - > = await trx(ChartViewsTableName) - .select("parentChartId", "chartConfigId") - .where({ id }) - .first() - - if (!existingRow) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const { patchConfig, fullConfig, queryParams } = - await createPatchConfigAndQueryParamsForChartView( - trx, - existingRow.parentChartId, - rawConfig - ) - - await updateChartConfigInDbAndR2( - trx, - existingRow.chartConfigId as Base64String, - patchConfig, - fullConfig - ) - - // update chart_views - await trx - .table(ChartViewsTableName) - .where({ id }) - .update({ - updatedAt: new Date(), - lastEditedByUserId: res.locals.user.id, - queryParamsForParentChart: JSON.stringify(queryParams), - }) - - return { success: true } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - const chartConfigId: string | undefined = await trx(ChartViewsTableName) - .select("chartConfigId") - .where({ id }) - .first() - .then((row) => row?.chartConfigId) - - if (!chartConfigId) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - await trx.table(ChartViewsTableName).where({ id }).delete() - - await deleteGrapherConfigFromR2ByUUID(chartConfigId) - - await trx - .table(ChartConfigsTableName) - .where({ id: chartConfigId }) - .delete() - - return { success: true } - } -) - export { apiRouter } diff --git a/adminSiteServer/apiRoutes/bulkUpdates.ts b/adminSiteServer/apiRoutes/bulkUpdates.ts new file mode 100644 index 0000000000..4dbb3cc902 --- /dev/null +++ b/adminSiteServer/apiRoutes/bulkUpdates.ts @@ -0,0 +1,256 @@ +import { + DbPlainChart, + DbRawChartConfig, + GrapherInterface, + DbRawVariable, +} from "@ourworldindata/types" +import { parseIntOrUndefined } from "@ourworldindata/utils" +import { + BulkGrapherConfigResponse, + BulkChartEditResponseRow, + chartBulkUpdateAllowedColumnNamesAndTypes, + GrapherConfigPatch, + VariableAnnotationsResponseRow, + variableAnnotationAllowedColumnNamesAndTypes, +} from "../../adminShared/AdminSessionTypes.js" +import { applyPatch } from "../../adminShared/patchHelper.js" +import { + OperationContext, + parseToOperation, +} from "../../adminShared/SqlFilterSExpression.js" +import { + getGrapherConfigsForVariable, + updateGrapherConfigAdminOfVariable, +} from "../../db/model/Variable.js" +import { + getRouteWithROTransaction, + patchRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import { saveGrapher } from "./charts.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import { apiRouter } from "../apiRouter.js" + +getRouteWithROTransaction( + apiRouter, + "/chart-bulk-update", + async ( + req, + res, + trx + ): Promise<BulkGrapherConfigResponse<BulkChartEditResponseRow>> => { + const context: OperationContext = { + grapherConfigFieldName: "chart_configs.full", + whitelistedColumnNamesAndTypes: + chartBulkUpdateAllowedColumnNamesAndTypes, + } + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined + + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql + SELECT + charts.id as id, + chart_configs.full as config, + charts.createdAt as createdAt, + charts.updatedAt as updatedAt, + charts.lastEditedAt as lastEditedAt, + charts.publishedAt as publishedAt, + lastEditedByUser.fullName as lastEditedByUser, + publishedByUser.fullName as publishedByUser + FROM charts + LEFT JOIN chart_configs ON chart_configs.id = charts.configId + LEFT JOIN users lastEditedByUser ON lastEditedByUser.id=charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id=charts.publishedByUserId + WHERE ${whereClause} + ORDER BY charts.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` + ) + + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql + SELECT count(*) as count + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + WHERE ${whereClause} + ` + ) + return { rows: results, numTotalRows: resultCount[0].count } + } +) + +patchRouteWithRWTransaction( + apiRouter, + "/chart-bulk-update", + async (req, res, trx) => { + const patchesList = req.body as GrapherConfigPatch[] + const chartIds = new Set(patchesList.map((patch) => patch.id)) + + const configsAndIds = await db.knexRaw< + Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] } + >( + trx, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE c.id IN (?) + `, + [[...chartIds.values()]] + ) + const configMap = new Map<number, GrapherInterface>( + configsAndIds.map((item: any) => [ + item.id, + // make sure that the id is set, otherwise the update behaviour is weird + // TODO: discuss if this has unintended side effects + item.config ? { ...JSON.parse(item.config), id: item.id } : {}, + ]) + ) + const oldValuesConfigMap = new Map(configMap) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) + } + + for (const [id, newConfig] of configMap.entries()) { + await saveGrapher(trx, { + user: res.locals.user, + newConfig, + existingConfig: oldValuesConfigMap.get(id), + referencedVariablesMightChange: false, + }) + } + + return { success: true } + } +) + +getRouteWithROTransaction( + apiRouter, + "/variable-annotations", + async ( + req, + res, + trx + ): Promise<BulkGrapherConfigResponse<VariableAnnotationsResponseRow>> => { + const context: OperationContext = { + grapherConfigFieldName: "grapherConfigAdmin", + whitelistedColumnNamesAndTypes: + variableAnnotationAllowedColumnNamesAndTypes, + } + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined + + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql + SELECT + variables.id as id, + variables.name as name, + chart_configs.patch as config, + d.name as datasetname, + namespaces.name as namespacename, + variables.createdAt as createdAt, + variables.updatedAt as updatedAt, + variables.description as description + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ORDER BY variables.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` + ) + + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql + SELECT count(*) as count + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ` + ) + return { rows: results, numTotalRows: resultCount[0].count } + } +) + +patchRouteWithRWTransaction( + apiRouter, + "/variable-annotations", + async (req, res, trx) => { + const patchesList = req.body as GrapherConfigPatch[] + const variableIds = new Set(patchesList.map((patch) => patch.id)) + + const configsAndIds = await db.knexRaw< + Pick<DbRawVariable, "id"> & { + grapherConfigAdmin: DbRawChartConfig["patch"] + } + >( + trx, + `-- sql + SELECT v.id, cc.patch AS grapherConfigAdmin + FROM variables v + LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id + WHERE v.id IN (?) + `, + [[...variableIds.values()]] + ) + const configMap = new Map( + configsAndIds.map((item: any) => [ + item.id, + item.grapherConfigAdmin + ? JSON.parse(item.grapherConfigAdmin) + : {}, + ]) + ) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) + } + + for (const [variableId, newConfig] of configMap.entries()) { + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) continue + await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) + } + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/chartViews.ts b/adminSiteServer/apiRoutes/chartViews.ts new file mode 100644 index 0000000000..c0013b57ef --- /dev/null +++ b/adminSiteServer/apiRoutes/chartViews.ts @@ -0,0 +1,290 @@ +import { + defaultGrapherConfig, + grapherConfigToQueryParams, +} from "@ourworldindata/grapher" +import { + GrapherInterface, + CHART_VIEW_PROPS_TO_OMIT, + CHART_VIEW_PROPS_TO_PERSIST, + DbPlainChartView, + JsonString, + JsonError, + parseChartConfig, + DbInsertChartView, + ChartViewsTableName, + Base64String, + ChartConfigsTableName, +} from "@ourworldindata/types" +import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils" +import { omit, pick } from "lodash" +import { ApiChartViewOverview } from "../../adminShared/AdminTypes.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { apiRouter } from "../apiRouter.js" +import { + saveNewChartConfigInDbAndR2, + updateChartConfigInDbAndR2, +} from "../chartConfigHelpers.js" +import { deleteGrapherConfigFromR2ByUUID } from "../chartConfigR2Helpers.js" +import { + getRouteWithROTransaction, + postRouteWithRWTransaction, + putRouteWithRWTransaction, + deleteRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" + +import * as db from "../../db/db.js" +import { expectChartById } from "./charts.js" +const createPatchConfigAndQueryParamsForChartView = async ( + knex: db.KnexReadonlyTransaction, + parentChartId: number, + config: GrapherInterface +) => { + const parentChartConfig = await expectChartById(knex, parentChartId) + + config = omit(config, CHART_VIEW_PROPS_TO_OMIT) + + const patchToParentChart = diffGrapherConfigs(config, parentChartConfig) + + const fullConfigIncludingDefaults = mergeGrapherConfigs( + defaultGrapherConfig, + config + ) + const patchConfigToSave = { + ...patchToParentChart, + + // We want to make sure we're explicitly persisting some props like entity selection + // always, so they never change when the parent chart changes. + // For this, we need to ensure we include the default layer, so that we even + // persist these props when they are the same as the default. + ...pick(fullConfigIncludingDefaults, CHART_VIEW_PROPS_TO_PERSIST), + } + + const queryParams = grapherConfigToQueryParams(config) + + const fullConfig = mergeGrapherConfigs(parentChartConfig, patchConfigToSave) + return { patchConfig: patchConfigToSave, fullConfig, queryParams } +} + +getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { + type ChartViewRow = Pick<DbPlainChartView, "id" | "name" | "updatedAt"> & { + lastEditedByUser: string + chartConfigId: string + title: string + parentChartId: number + parentTitle: string + } + + const rows: ChartViewRow[] = await db.knexRaw( + trx, + `-- sql + SELECT + cv.id, + cv.name, + cv.updatedAt, + u.fullName as lastEditedByUser, + cv.chartConfigId, + cc.full ->> "$.title" as title, + cv.parentChartId, + pcc.full ->> "$.title" as parentTitle + FROM chart_views cv + JOIN chart_configs cc ON cv.chartConfigId = cc.id + JOIN charts pc ON cv.parentChartId = pc.id + JOIN chart_configs pcc ON pc.configId = pcc.id + JOIN users u ON cv.lastEditedByUserId = u.id + ORDER BY cv.updatedAt DESC + ` + ) + + const chartViews: ApiChartViewOverview[] = rows.map((row) => ({ + id: row.id, + name: row.name, + updatedAt: row.updatedAt?.toISOString() ?? null, + lastEditedByUser: row.lastEditedByUser, + chartConfigId: row.chartConfigId, + title: row.title, + parent: { + id: row.parentChartId, + title: row.parentTitle, + }, + })) + + return { chartViews } +}) + +getRouteWithROTransaction( + apiRouter, + "/chartViews/:id", + async (req, res, trx) => { + const id = expectInt(req.params.id) + + type ChartViewRow = Pick< + DbPlainChartView, + "id" | "name" | "updatedAt" + > & { + lastEditedByUser: string + chartConfigId: string + configFull: JsonString + configPatch: JsonString + parentChartId: number + parentConfigFull: JsonString + queryParamsForParentChart: JsonString + } + + const row = await db.knexRawFirst<ChartViewRow>( + trx, + `-- sql + SELECT + cv.id, + cv.name, + cv.updatedAt, + u.fullName as lastEditedByUser, + cv.chartConfigId, + cc.full as configFull, + cc.patch as configPatch, + cv.parentChartId, + pcc.full as parentConfigFull, + cv.queryParamsForParentChart + FROM chart_views cv + JOIN chart_configs cc ON cv.chartConfigId = cc.id + JOIN charts pc ON cv.parentChartId = pc.id + JOIN chart_configs pcc ON pc.configId = pcc.id + JOIN users u ON cv.lastEditedByUserId = u.id + WHERE cv.id = ? + `, + [id] + ) + + if (!row) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + const chartView = { + ...row, + configFull: parseChartConfig(row.configFull), + configPatch: parseChartConfig(row.configPatch), + parentConfigFull: parseChartConfig(row.parentConfigFull), + queryParamsForParentChart: JSON.parse( + row.queryParamsForParentChart + ), + } + + return chartView + } +) + +postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { + const { name, parentChartId } = req.body as Pick< + DbPlainChartView, + "name" | "parentChartId" + > + const rawConfig = req.body.config as GrapherInterface + if (!name || !parentChartId || !rawConfig) { + throw new JsonError("Invalid request", 400) + } + + const { patchConfig, fullConfig, queryParams } = + await createPatchConfigAndQueryParamsForChartView( + trx, + parentChartId, + rawConfig + ) + + const { chartConfigId } = await saveNewChartConfigInDbAndR2( + trx, + undefined, + patchConfig, + fullConfig + ) + + // insert into chart_views + const insertRow: DbInsertChartView = { + name, + parentChartId, + lastEditedByUserId: res.locals.user.id, + chartConfigId: chartConfigId, + queryParamsForParentChart: JSON.stringify(queryParams), + } + const result = await trx.table(ChartViewsTableName).insert(insertRow) + const [resultId] = result + + return { chartViewId: resultId, success: true } +}) + +putRouteWithRWTransaction( + apiRouter, + "/chartViews/:id", + async (req, res, trx) => { + const id = expectInt(req.params.id) + const rawConfig = req.body.config as GrapherInterface + if (!rawConfig) { + throw new JsonError("Invalid request", 400) + } + + const existingRow: Pick< + DbPlainChartView, + "chartConfigId" | "parentChartId" + > = await trx(ChartViewsTableName) + .select("parentChartId", "chartConfigId") + .where({ id }) + .first() + + if (!existingRow) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + const { patchConfig, fullConfig, queryParams } = + await createPatchConfigAndQueryParamsForChartView( + trx, + existingRow.parentChartId, + rawConfig + ) + + await updateChartConfigInDbAndR2( + trx, + existingRow.chartConfigId as Base64String, + patchConfig, + fullConfig + ) + + // update chart_views + await trx + .table(ChartViewsTableName) + .where({ id }) + .update({ + updatedAt: new Date(), + lastEditedByUserId: res.locals.user.id, + queryParamsForParentChart: JSON.stringify(queryParams), + }) + + return { success: true } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/chartViews/:id", + async (req, res, trx) => { + const id = expectInt(req.params.id) + + const chartConfigId: string | undefined = await trx(ChartViewsTableName) + .select("chartConfigId") + .where({ id }) + .first() + .then((row) => row?.chartConfigId) + + if (!chartConfigId) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + await trx.table(ChartViewsTableName).where({ id }).delete() + + await deleteGrapherConfigFromR2ByUUID(chartConfigId) + + await trx + .table(ChartConfigsTableName) + .where({ id: chartConfigId }) + .delete() + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/charts.ts b/adminSiteServer/apiRoutes/charts.ts new file mode 100644 index 0000000000..ae295c11fe --- /dev/null +++ b/adminSiteServer/apiRoutes/charts.ts @@ -0,0 +1,801 @@ +import { migrateGrapherConfigToLatestVersion } from "@ourworldindata/grapher" +import { + GrapherInterface, + JsonError, + DbPlainUser, + Base64String, + serializeChartConfig, + DbPlainChart, + DbPlainChartSlugRedirect, + R2GrapherConfigDirectory, + DbInsertChartRevision, + DbRawChartConfig, + ChartConfigsTableName, +} from "@ourworldindata/types" +import { + diffGrapherConfigs, + mergeGrapherConfigs, + parseIntOrUndefined, + omitUndefinedValues, +} from "@ourworldindata/utils" +import Papa from "papaparse" +import { uuidv7 } from "uuidv7" +import { References } from "../../adminSiteClient/AbstractChartEditor.js" +import { ChartViewMinimalInformation } from "../../adminSiteClient/ChartEditor.js" +import { denormalizeLatestCountryData } from "../../baker/countryProfiles.js" +import { + getChartConfigById, + getPatchConfigByChartId, + getParentByChartConfig, + isInheritanceEnabledForChart, + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, + getParentByChartId, + getRedirectsByChartId, + getChartSlugById, + setChartTags, +} from "../../db/model/Chart.js" +import { + getWordpressPostReferencesByChartId, + getGdocsPostReferencesByChartId, +} from "../../db/model/Post.js" +import { expectInt, isValidSlug } from "../../serverUtils/serverUtil.js" +import { + BAKED_BASE_URL, + ADMIN_BASE_URL, +} from "../../settings/clientSettings.js" +import { apiRouter } from "../apiRouter.js" +import { + retrieveChartConfigFromDbAndSaveToR2, + updateChartConfigInDbAndR2, +} from "../chartConfigHelpers.js" +import { + deleteGrapherConfigFromR2, + deleteGrapherConfigFromR2ByUUID, + saveGrapherConfigToR2ByUUID, +} from "../chartConfigR2Helpers.js" +import { + deleteRouteWithRWTransaction, + getRouteWithROTransaction, + postRouteWithRWTransaction, + putRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import { getLogsByChartId } from "../getLogsByChartId.js" +import { getPublishedLinksTo } from "../../db/model/Link.js" + +export const getReferencesByChartId = async ( + chartId: number, + knex: db.KnexReadonlyTransaction +): Promise<References> => { + const postsWordpressPromise = getWordpressPostReferencesByChartId( + chartId, + knex + ) + const postGdocsPromise = getGdocsPostReferencesByChartId(chartId, knex) + const explorerSlugsPromise = db.knexRaw<{ explorerSlug: string }>( + knex, + `SELECT DISTINCT + explorerSlug + FROM + explorer_charts + WHERE + chartId = ?`, + [chartId] + ) + const chartViewsPromise = db.knexRaw<ChartViewMinimalInformation>( + knex, + `-- sql + SELECT cv.id, cv.name, cc.full ->> "$.title" AS title + FROM chart_views cv + JOIN chart_configs cc ON cc.id = cv.chartConfigId + WHERE cv.parentChartId = ?`, + [chartId] + ) + const [postsWordpress, postsGdocs, explorerSlugs, chartViews] = + await Promise.all([ + postsWordpressPromise, + postGdocsPromise, + explorerSlugsPromise, + chartViewsPromise, + ]) + + return { + postsGdocs, + postsWordpress, + explorers: explorerSlugs.map( + (row: { explorerSlug: string }) => row.explorerSlug + ), + chartViews, + } +} + +export const expectChartById = async ( + knex: db.KnexReadonlyTransaction, + chartId: any +): Promise<GrapherInterface> => { + const chart = await getChartConfigById(knex, expectInt(chartId)) + if (chart) return chart.config + + throw new JsonError(`No chart found for id ${chartId}`, 404) +} + +const expectPatchConfigByChartId = async ( + knex: db.KnexReadonlyTransaction, + chartId: any +): Promise<GrapherInterface> => { + const patchConfig = await getPatchConfigByChartId(knex, expectInt(chartId)) + if (!patchConfig) { + throw new JsonError(`No chart found for id ${chartId}`, 404) + } + return patchConfig +} + +const saveNewChart = async ( + knex: db.KnexReadWriteTransaction, + { + config, + user, + // new charts inherit by default + shouldInherit = true, + }: { config: GrapherInterface; user: DbPlainUser; shouldInherit?: boolean } +): Promise<{ + chartConfigId: Base64String + patchConfig: GrapherInterface + fullConfig: GrapherInterface +}> => { + // grab the parent of the chart if inheritance should be enabled + const parent = shouldInherit + ? await getParentByChartConfig(knex, config) + : undefined + + // compute patch and full configs + const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) + const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) + + // insert patch & full configs into the chart_configs table + // We can't quite use `saveNewChartConfigInDbAndR2` here, because + // we need to update the chart id in the config after inserting it. + const chartConfigId = uuidv7() as Base64String + await db.knexRaw( + knex, + `-- sql + INSERT INTO chart_configs (id, patch, full) + VALUES (?, ?, ?) + `, + [ + chartConfigId, + serializeChartConfig(patchConfig), + serializeChartConfig(fullConfig), + ] + ) + + // add a new chart to the charts table + const result = await db.knexRawInsert( + knex, + `-- sql + INSERT INTO charts (configId, isInheritanceEnabled, lastEditedAt, lastEditedByUserId) + VALUES (?, ?, ?, ?) + `, + [chartConfigId, shouldInherit, new Date(), user.id] + ) + + // The chart config itself has an id field that should store the id of the chart - update the chart now so this is true + const chartId = result.insertId + patchConfig.id = chartId + fullConfig.id = chartId + await db.knexRaw( + knex, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch=JSON_SET(cc.patch, '$.id', ?), + cc.full=JSON_SET(cc.full, '$.id', ?) + WHERE c.id = ? + `, + [chartId, chartId, chartId] + ) + + await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId) + + return { chartConfigId, patchConfig, fullConfig } +} + +const updateExistingChart = async ( + knex: db.KnexReadWriteTransaction, + params: { + config: GrapherInterface + user: DbPlainUser + chartId: number + // if undefined, keep inheritance as is. + // if true or false, enable or disable inheritance + shouldInherit?: boolean + } +): Promise<{ + chartConfigId: Base64String + patchConfig: GrapherInterface + fullConfig: GrapherInterface +}> => { + const { config, user, chartId } = params + + // make sure that the id of the incoming config matches the chart id + config.id = chartId + + // if inheritance is enabled, grab the parent from its config + const shouldInherit = + params.shouldInherit ?? + (await isInheritanceEnabledForChart(knex, chartId)) + const parent = shouldInherit + ? await getParentByChartConfig(knex, config) + : undefined + + // compute patch and full configs + const patchConfig = diffGrapherConfigs(config, parent?.config ?? {}) + const fullConfig = mergeGrapherConfigs(parent?.config ?? {}, patchConfig) + + const chartConfigIdRow = await db.knexRawFirst< + Pick<DbPlainChart, "configId"> + >(knex, `SELECT configId FROM charts WHERE id = ?`, [chartId]) + + if (!chartConfigIdRow) + throw new JsonError(`No chart config found for id ${chartId}`, 404) + + const now = new Date() + + const { chartConfigId } = await updateChartConfigInDbAndR2( + knex, + chartConfigIdRow.configId as Base64String, + patchConfig, + fullConfig + ) + + // update charts row + await db.knexRaw( + knex, + `-- sql + UPDATE charts + SET isInheritanceEnabled=?, updatedAt=?, lastEditedAt=?, lastEditedByUserId=? + WHERE id = ? + `, + [shouldInherit, now, now, user.id, chartId] + ) + + return { chartConfigId, patchConfig, fullConfig } +} + +export const saveGrapher = async ( + knex: db.KnexReadWriteTransaction, + { + user, + newConfig, + existingConfig, + shouldInherit, + referencedVariablesMightChange = true, + }: { + user: DbPlainUser + newConfig: GrapherInterface + existingConfig?: GrapherInterface + // if undefined, keep inheritance as is. + // if true or false, enable or disable inheritance + shouldInherit?: boolean + // if the variables a chart uses can change then we need + // to update the latest country data which takes quite a long time (hundreds of ms) + referencedVariablesMightChange?: boolean + } +) => { + // Try to migrate the new config to the latest version + newConfig = migrateGrapherConfigToLatestVersion(newConfig) + + // Slugs need some special logic to ensure public urls remain consistent whenever possible + async function isSlugUsedInRedirect() { + const rows = await db.knexRaw<DbPlainChartSlugRedirect>( + knex, + `SELECT * FROM chart_slug_redirects WHERE chart_id != ? AND slug = ?`, + // -1 is a placeholder ID that will never exist; but we cannot use NULL because + // in that case we would always get back an empty resultset + [existingConfig ? existingConfig.id : -1, newConfig.slug] + ) + return rows.length > 0 + } + + async function isSlugUsedInOtherGrapher() { + const rows = await db.knexRaw<Pick<DbPlainChart, "id">>( + knex, + `-- sql + SELECT c.id + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE + c.id != ? + AND cc.full ->> "$.isPublished" = "true" + AND cc.slug = ? + `, + // -1 is a placeholder ID that will never exist; but we cannot use NULL because + // in that case we would always get back an empty resultset + [existingConfig ? existingConfig.id : -1, newConfig.slug] + ) + return rows.length > 0 + } + + // When a chart is published, check for conflicts + if (newConfig.isPublished) { + if (!isValidSlug(newConfig.slug)) + throw new JsonError(`Invalid chart slug ${newConfig.slug}`) + else if (await isSlugUsedInRedirect()) + throw new JsonError( + `This chart slug was previously used by another chart: ${newConfig.slug}` + ) + else if (await isSlugUsedInOtherGrapher()) + throw new JsonError( + `This chart slug is in use by another published chart: ${newConfig.slug}` + ) + else if ( + existingConfig && + existingConfig.isPublished && + existingConfig.slug !== newConfig.slug + ) { + // Changing slug of an existing chart, delete any old redirect and create new one + await db.knexRaw( + knex, + `DELETE FROM chart_slug_redirects WHERE chart_id = ? AND slug = ?`, + [existingConfig.id, existingConfig.slug] + ) + await db.knexRaw( + knex, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [existingConfig.id, existingConfig.slug] + ) + // When we rename grapher configs, make sure to delete the old one (the new one will be saved below) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${existingConfig.slug}.json` + ) + } + } + + if (existingConfig) + // Bump chart version, very important for cachebusting + newConfig.version = existingConfig.version! + 1 + else if (newConfig.version) + // If a chart is republished, we want to keep incrementing the old version number, + // otherwise it can lead to clients receiving cached versions of the old data. + newConfig.version += 1 + else newConfig.version = 1 + + // add the isPublished field if is missing + if (newConfig.isPublished === undefined) { + newConfig.isPublished = false + } + + // Execute the actual database update or creation + let chartId: number + let chartConfigId: Base64String + let patchConfig: GrapherInterface + let fullConfig: GrapherInterface + if (existingConfig) { + chartId = existingConfig.id! + const configs = await updateExistingChart(knex, { + config: newConfig, + user, + chartId, + shouldInherit, + }) + chartConfigId = configs.chartConfigId + patchConfig = configs.patchConfig + fullConfig = configs.fullConfig + } else { + const configs = await saveNewChart(knex, { + config: newConfig, + user, + shouldInherit, + }) + chartConfigId = configs.chartConfigId + patchConfig = configs.patchConfig + fullConfig = configs.fullConfig + chartId = fullConfig.id! + } + + // Record this change in version history + const chartRevisionLog = { + chartId: chartId as number, + userId: user.id, + config: serializeChartConfig(patchConfig), + createdAt: new Date(), + updatedAt: new Date(), + } satisfies DbInsertChartRevision + await db.knexRaw( + knex, + `INSERT INTO chart_revisions (chartId, userId, config, createdAt, updatedAt) VALUES (?, ?, ?, ?, ?)`, + [ + chartRevisionLog.chartId, + chartRevisionLog.userId, + chartRevisionLog.config, + chartRevisionLog.createdAt, + chartRevisionLog.updatedAt, + ] + ) + + // Remove any old dimensions and store the new ones + // We only note that a relationship exists between the chart and variable in the database; the actual dimension configuration is left to the json + await db.knexRaw(knex, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chartId, + ]) + + const newDimensions = fullConfig.dimensions ?? [] + for (const [i, dim] of newDimensions.entries()) { + await db.knexRaw( + knex, + `INSERT INTO chart_dimensions (chartId, variableId, property, \`order\`) VALUES (?, ?, ?, ?)`, + [chartId, dim.variableId, dim.property, i] + ) + } + + // So we can generate country profiles including this chart data + if (fullConfig.isPublished && referencedVariablesMightChange) + // TODO: remove this ad hoc knex transaction context when we switch the function to knex + await denormalizeLatestCountryData( + knex, + newDimensions.map((d) => d.variableId) + ) + + if (fullConfig.isPublished) { + await retrieveChartConfigFromDbAndSaveToR2(knex, chartConfigId, { + directory: R2GrapherConfigDirectory.publishedGrapherBySlug, + filename: `${fullConfig.slug}.json`, + }) + } + + if ( + fullConfig.isPublished && + (!existingConfig || !existingConfig.isPublished) + ) { + // Newly published, set publication info + await db.knexRaw( + knex, + `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `, + [new Date(), user.id, chartId] + ) + await triggerStaticBuild(user, `Publishing chart ${fullConfig.slug}`) + } else if ( + !fullConfig.isPublished && + existingConfig && + existingConfig.isPublished + ) { + // Unpublishing chart, delete any existing redirects to it + await db.knexRaw( + knex, + `DELETE FROM chart_slug_redirects WHERE chart_id = ?`, + [existingConfig.id] + ) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${existingConfig.slug}.json` + ) + await triggerStaticBuild(user, `Unpublishing chart ${fullConfig.slug}`) + } else if (fullConfig.isPublished) + await triggerStaticBuild(user, `Updating chart ${fullConfig.slug}`) + + return { + chartId, + savedPatch: patchConfig, + } +} + +export async function updateGrapherConfigsInR2( + knex: db.KnexReadonlyTransaction, + updatedCharts: { chartConfigId: string; isPublished: boolean }[], + updatedMultiDimViews: { chartConfigId: string; isPublished: boolean }[] +) { + const idsToUpdate = [ + ...updatedCharts.filter(({ isPublished }) => isPublished), + ...updatedMultiDimViews, + ].map(({ chartConfigId }) => chartConfigId) + const builder = knex<DbRawChartConfig>(ChartConfigsTableName) + .select("id", "full", "fullMd5") + .whereIn("id", idsToUpdate) + for await (const { id, full, fullMd5 } of builder.stream()) { + await saveGrapherConfigToR2ByUUID(id, full, fullMd5) + } +} + +getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 + const charts = await db.knexRaw<OldChartFieldList>( + trx, + `-- sql + SELECT ${oldChartFieldList}, + round(views_365d / 365, 1) as pageviewsPerDay + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN analytics_pageviews on (analytics_pageviews.url = CONCAT("https://ourworldindata.org/grapher/", chart_configs.slug) AND chart_configs.full ->> '$.isPublished' = "true" ) + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + ORDER BY charts.lastEditedAt DESC LIMIT ? + `, + [limit] + ) + + await assignTagsForCharts(trx, charts) + + return { charts } +}) + +getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 + + // note: this query is extended from OldChart.listFields. + const charts = await db.knexRaw( + trx, + `-- sql + SELECT + charts.id, + chart_configs.full->>"$.version" AS version, + CONCAT("${BAKED_BASE_URL}/grapher/", chart_configs.full->>"$.slug") AS url, + CONCAT("${ADMIN_BASE_URL}", "/admin/charts/", charts.id, "/edit") AS editUrl, + chart_configs.full->>"$.slug" AS slug, + chart_configs.full->>"$.title" AS title, + chart_configs.full->>"$.subtitle" AS subtitle, + chart_configs.full->>"$.sourceDesc" AS sourceDesc, + chart_configs.full->>"$.note" AS note, + chart_configs.chartType AS type, + chart_configs.full->>"$.internalNotes" AS internalNotes, + chart_configs.full->>"$.variantName" AS variantName, + chart_configs.full->>"$.isPublished" AS isPublished, + chart_configs.full->>"$.tab" AS tab, + chart_configs.chartType IS NOT NULL AS hasChartTab, + JSON_EXTRACT(chart_configs.full, "$.hasMapTab") = true AS hasMapTab, + chart_configs.full->>"$.originUrl" AS originUrl, + charts.lastEditedAt, + charts.lastEditedByUserId, + lastEditedByUser.fullName AS lastEditedBy, + charts.publishedAt, + charts.publishedByUserId, + publishedByUser.fullName AS publishedBy + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + ORDER BY charts.lastEditedAt DESC + LIMIT ? + `, + [limit] + ) + // note: retrieving references is VERY slow. + // await Promise.all( + // charts.map(async (chart: any) => { + // const references = await getReferencesByChartId(chart.id) + // chart.references = references.length + // ? references.map((ref) => ref.url) + // : "" + // }) + // ) + // await Chart.assignTagsForCharts(charts) + res.setHeader("Content-disposition", "attachment; filename=charts.csv") + res.setHeader("content-type", "text/csv") + const csv = Papa.unparse(charts) + return csv +}) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.config.json", + async (req, res, trx) => expectChartById(trx, req.params.chartId) +) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.parent.json", + async (req, res, trx) => { + const chartId = expectInt(req.params.chartId) + const parent = await getParentByChartId(trx, chartId) + const isInheritanceEnabled = await isInheritanceEnabledForChart( + trx, + chartId + ) + return omitUndefinedValues({ + variableId: parent?.variableId, + config: parent?.config, + isActive: isInheritanceEnabled, + }) + } +) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.patchConfig.json", + async (req, res, trx) => { + const chartId = expectInt(req.params.chartId) + const config = await expectPatchConfigByChartId(trx, chartId) + return config + } +) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.logs.json", + async (req, res, trx) => ({ + logs: await getLogsByChartId( + trx, + parseInt(req.params.chartId as string) + ), + }) +) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.references.json", + async (req, res, trx) => { + const references = { + references: await getReferencesByChartId( + parseInt(req.params.chartId as string), + trx + ), + } + return references + } +) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.redirects.json", + async (req, res, trx) => ({ + redirects: await getRedirectsByChartId( + trx, + parseInt(req.params.chartId as string) + ), + }) +) + +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.pageviews.json", + async (req, res, trx) => { + const slug = await getChartSlugById( + trx, + parseInt(req.params.chartId as string) + ) + if (!slug) return {} + + const pageviewsByUrl = await db.knexRawFirst( + trx, + `-- sql + SELECT * + FROM + analytics_pageviews + WHERE + url = ?`, + [`https://ourworldindata.org/grapher/${slug}`] + ) + + return { + pageviews: pageviewsByUrl ?? undefined, + } + } +) + +postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { + let shouldInherit: boolean | undefined + if (req.query.inheritance) { + shouldInherit = req.query.inheritance === "enable" + } + + try { + const { chartId } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + shouldInherit, + }) + + return { success: true, chartId: chartId } + } catch (err) { + return { success: false, error: String(err) } + } +}) + +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/setTags", + async (req, res, trx) => { + const chartId = expectInt(req.params.chartId) + + await setChartTags(trx, chartId, req.body.tags) + + return { success: true } + } +) + +putRouteWithRWTransaction( + apiRouter, + "/charts/:chartId", + async (req, res, trx) => { + let shouldInherit: boolean | undefined + if (req.query.inheritance) { + shouldInherit = req.query.inheritance === "enable" + } + + const existingConfig = await expectChartById(trx, req.params.chartId) + + try { + const { chartId, savedPatch } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + existingConfig, + shouldInherit, + }) + + const logs = await getLogsByChartId( + trx, + existingConfig.id as number + ) + return { + success: true, + chartId, + savedPatch, + newLog: logs[0], + } + } catch (err) { + return { + success: false, + error: String(err), + } + } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/charts/:chartId", + async (req, res, trx) => { + const chart = await expectChartById(trx, req.params.chartId) + if (chart.slug) { + const links = await getPublishedLinksTo(trx, [chart.slug]) + if (links.length) { + const sources = links.map((link) => link.sourceSlug).join(", ") + throw new Error( + `Cannot delete chart in-use in the following published documents: ${sources}` + ) + } + } + + await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chart.id, + ]) + await db.knexRaw( + trx, + `DELETE FROM chart_slug_redirects WHERE chart_id=?`, + [chart.id] + ) + + const row = await db.knexRawFirst<Pick<DbPlainChart, "configId">>( + trx, + `SELECT configId FROM charts WHERE id = ?`, + [chart.id] + ) + if (!row || !row.configId) + throw new JsonError(`No chart config found for id ${chart.id}`, 404) + if (row) { + await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) + await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ + row.configId, + ]) + } + + if (chart.isPublished) + await triggerStaticBuild( + res.locals.user, + `Deleting chart ${chart.slug}` + ) + + await deleteGrapherConfigFromR2ByUUID(row.configId) + if (chart.isPublished) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${chart.slug}.json` + ) + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/datasets.ts b/adminSiteServer/apiRoutes/datasets.ts new file mode 100644 index 0000000000..365f00be51 --- /dev/null +++ b/adminSiteServer/apiRoutes/datasets.ts @@ -0,0 +1,417 @@ +import { + DbPlainTag, + DbPlainDatasetTag, + JsonError, + DbRawVariable, + DbRawOrigin, + parseOriginsRow, +} from "@ourworldindata/types" +import { + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, +} from "../../db/model/Chart.js" +import { getDatasetById, setTagsForDataset } from "../../db/model/Dataset.js" +import { logErrorAndMaybeSendToBugsnag } from "../../serverUtils/errorLog.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + putRouteWithRWTransaction, + postRouteWithRWTransaction, + deleteRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import { + syncDatasetToGitRepo, + removeDatasetFromGitRepo, +} from "../gitDataExport.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" + +getRouteWithROTransaction( + apiRouter, + "/datasets.json", + async (req, res, trx) => { + const datasets = await db.knexRaw<Record<string, any>>( + trx, + `-- sql + WITH variable_counts AS ( + SELECT + v.datasetId, + COUNT(DISTINCT cd.chartId) as numCharts + FROM chart_dimensions cd + JOIN variables v ON cd.variableId = v.id + GROUP BY v.datasetId + ) + SELECT + ad.id, + ad.namespace, + ad.name, + d.shortName, + ad.description, + ad.dataEditedAt, + du.fullName AS dataEditedByUserName, + ad.metadataEditedAt, + mu.fullName AS metadataEditedByUserName, + ad.isPrivate, + ad.nonRedistributable, + d.version, + vc.numCharts + FROM active_datasets ad + LEFT JOIN variable_counts vc ON ad.id = vc.datasetId + JOIN users du ON du.id=ad.dataEditedByUserId + JOIN users mu ON mu.id=ad.metadataEditedByUserId + JOIN datasets d ON d.id=ad.id + ORDER BY ad.dataEditedAt DESC + ` + ) + + const tags = await db.knexRaw< + Pick<DbPlainTag, "id" | "name"> & + Pick<DbPlainDatasetTag, "datasetId"> + >( + trx, + `-- sql + SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt + JOIN tags t ON dt.tagId = t.id + ` + ) + const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) + for (const dataset of datasets) { + dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => + lodash.omit(t, "datasetId") + ) + } + /*LEFT JOIN variables AS v ON v.datasetId=d.id + GROUP BY d.id*/ + + return { datasets: datasets } + } +) + +getRouteWithROTransaction( + apiRouter, + "/datasets/:datasetId.json", + async (req, res, trx) => { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await db.knexRawFirst<Record<string, any>>( + trx, + `-- sql + SELECT d.id, + d.namespace, + d.name, + d.shortName, + d.version, + d.description, + d.updatedAt, + d.dataEditedAt, + d.dataEditedByUserId, + du.fullName AS dataEditedByUserName, + d.metadataEditedAt, + d.metadataEditedByUserId, + mu.fullName AS metadataEditedByUserName, + d.isPrivate, + d.isArchived, + d.nonRedistributable, + d.updatePeriodDays + FROM datasets AS d + JOIN users du ON du.id=d.dataEditedByUserId + JOIN users mu ON mu.id=d.metadataEditedByUserId + WHERE d.id = ? + `, + [datasetId] + ) + + if (!dataset) + throw new JsonError(`No dataset by id '${datasetId}'`, 404) + + const zipFile = await db.knexRawFirst<{ filename: string }>( + trx, + `SELECT filename FROM dataset_files WHERE datasetId=?`, + [datasetId] + ) + if (zipFile) dataset.zipFile = zipFile + + const variables = await db.knexRaw< + Pick< + DbRawVariable, + "id" | "name" | "description" | "display" | "catalogPath" + > + >( + trx, + `-- sql + SELECT + v.id, + v.name, + v.description, + v.display, + v.catalogPath + FROM + variables AS v + WHERE + v.datasetId = ? + `, + [datasetId] + ) + + for (const v of variables) { + v.display = JSON.parse(v.display) + } + + dataset.variables = variables + + // add all origins + const origins: DbRawOrigin[] = await db.knexRaw<DbRawOrigin>( + trx, + `-- sql + SELECT DISTINCT + o.* + FROM + origins_variables AS ov + JOIN origins AS o ON ov.originId = o.id + JOIN variables AS v ON ov.variableId = v.id + WHERE + v.datasetId = ? + `, + [datasetId] + ) + + const parsedOrigins = origins.map(parseOriginsRow) + + dataset.origins = parsedOrigins + + const sources = await db.knexRaw<{ + id: number + name: string + description: string + }>( + trx, + ` + SELECT s.id, s.name, s.description + FROM sources AS s + WHERE s.datasetId = ? + ORDER BY s.id ASC + `, + [datasetId] + ) + + // expand description of sources and add to dataset as variableSources + dataset.variableSources = sources.map((s: any) => { + return { + id: s.id, + name: s.name, + ...JSON.parse(s.description), + } + }) + + const charts = await db.knexRaw<OldChartFieldList>( + trx, + `-- sql + SELECT ${oldChartFieldList} + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN chart_dimensions AS cd ON cd.chartId = charts.id + JOIN variables AS v ON cd.variableId = v.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE v.datasetId = ? + GROUP BY charts.id + `, + [datasetId] + ) + + dataset.charts = charts + + await assignTagsForCharts(trx, charts) + + const tags = await db.knexRaw<{ id: number; name: string }>( + trx, + ` + SELECT t.id, t.name + FROM tags t + JOIN dataset_tags dt ON dt.tagId = t.id + WHERE dt.datasetId = ? + `, + [datasetId] + ) + dataset.tags = tags + + const availableTags = await db.knexRaw<{ + id: number + name: string + parentName: string + }>( + trx, + ` + SELECT t.id, t.name, p.name AS parentName + FROM tags AS t + JOIN tags AS p ON t.parentId=p.id + ` + ) + dataset.availableTags = availableTags + + return { dataset: dataset } + } +) + +putRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId", + async (req, res, trx) => { + // Only updates `nonRedistributable` and `tags`, other fields come from ETL + // and are not editable + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + const newDataset = (req.body as { dataset: any }).dataset + await db.knexRaw( + trx, + ` + UPDATE datasets + SET + nonRedistributable=?, + metadataEditedAt=?, + metadataEditedByUserId=? + WHERE id=? + `, + [ + newDataset.nonRedistributable, + new Date(), + res.locals.user.id, + datasetId, + ] + ) + + const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) + await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ + datasetId, + ]) + if (tagRows.length) + for (const tagRow of tagRows) { + await db.knexRaw( + trx, + `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, + tagRow + ) + } + + try { + await syncDatasetToGitRepo(trx, datasetId, { + oldDatasetName: dataset.name, + commitName: res.locals.user.fullName, + commitEmail: res.locals.user.email, + }) + } catch (err) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue + } + + return { success: true } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/setArchived", + async (req, res, trx) => { + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ + datasetId, + ]) + return { success: true } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/setTags", + async (req, res, trx) => { + const datasetId = expectInt(req.params.datasetId) + + await setTagsForDataset(trx, datasetId, req.body.tagIds) + + return { success: true } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId", + async (req, res, trx) => { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw( + trx, + `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, + [datasetId] + ) + await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) + + try { + await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { + commitName: res.locals.user.fullName, + commitEmail: res.locals.user.email, + }) + } catch (err: any) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue + } + + return { success: true } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/charts", + async (req, res, trx) => { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + if (req.body.republish) { + await db.knexRaw( + trx, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), + cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) + WHERE c.id IN ( + SELECT DISTINCT chart_dimensions.chartId + FROM chart_dimensions + JOIN variables ON variables.id = chart_dimensions.variableId + WHERE variables.datasetId = ? + )`, + [datasetId] + ) + } + + await triggerStaticBuild( + res.locals.user, + `Republishing all charts in dataset ${dataset.name} (${dataset.id})` + ) + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/explorer.ts b/adminSiteServer/apiRoutes/explorer.ts new file mode 100644 index 0000000000..eb184e2bef --- /dev/null +++ b/adminSiteServer/apiRoutes/explorer.ts @@ -0,0 +1,37 @@ +import { JsonError } from "@ourworldindata/types" +import { apiRouter } from "../apiRouter.js" +import { + postRouteWithRWTransaction, + deleteRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" + +postRouteWithRWTransaction( + apiRouter, + "/explorer/:slug/tags", + async (req, res, trx) => { + const { slug } = req.params + const { tagIds } = req.body + const explorer = await trx.table("explorers").where({ slug }).first() + if (!explorer) + throw new JsonError(`No explorer found for slug ${slug}`, 404) + + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + for (const tagId of tagIds) { + await trx + .table("explorer_tags") + .insert({ explorerSlug: slug, tagId }) + } + + return { success: true } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/explorer/:slug/tags", + async (req, res, trx) => { + const { slug } = req.params + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/gdocs.ts b/adminSiteServer/apiRoutes/gdocs.ts new file mode 100644 index 0000000000..0bd40226c5 --- /dev/null +++ b/adminSiteServer/apiRoutes/gdocs.ts @@ -0,0 +1,283 @@ +import { getCanonicalUrl } from "@ourworldindata/components" +import { + GdocsContentSource, + DbInsertUser, + JsonError, + GDOCS_BASE_URL, + gdocUrlRegex, + PostsGdocsLinksTableName, + PostsGdocsXImagesTableName, + PostsGdocsTableName, + PostsGdocsComponentsTableName, +} from "@ourworldindata/types" +import { checkIsGdocPostExcludingFragments } from "@ourworldindata/utils" +import { isEmpty } from "lodash" +import { match } from "ts-pattern" +import { + checkHasChanges, + getPublishingAction, + GdocPublishingAction, + checkIsLightningUpdate, +} from "../../adminSiteClient/gdocsDeploy.js" +import { + indexIndividualGdocPost, + removeIndividualGdocPostFromIndex, +} from "../../baker/algolia/utils/pages.js" +import { GdocAbout } from "../../db/model/Gdoc/GdocAbout.js" +import { GdocAuthor } from "../../db/model/Gdoc/GdocAuthor.js" +import { getMinimalGdocPostsByIds } from "../../db/model/Gdoc/GdocBase.js" +import { GdocDataInsight } from "../../db/model/Gdoc/GdocDataInsight.js" +import { + getAllGdocIndexItemsOrderedByUpdatedAt, + getAndLoadGdocById, + updateGdocContentOnly, + createOrLoadGdocById, + gdocFromJSON, + addImagesToContentGraph, + setLinksForGdoc, + GdocLinkUpdateMode, + upsertGdoc, + getGdocBaseObjectById, + setTagsForGdoc, +} from "../../db/model/Gdoc/GdocFactory.js" +import { GdocHomepage } from "../../db/model/Gdoc/GdocHomepage.js" +import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + getRouteNonIdempotentWithRWTransaction, + putRouteWithRWTransaction, + deleteRouteWithRWTransaction, + postRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import { triggerStaticBuild, enqueueLightningChange } from "./routeUtils.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" + +getRouteWithROTransaction(apiRouter, "/gdocs", (req, res, trx) => { + return getAllGdocIndexItemsOrderedByUpdatedAt(trx) +}) + +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/gdocs/:id", + async (req, res, trx) => { + const id = req.params.id + const contentSource = req.query.contentSource as + | GdocsContentSource + | undefined + + try { + // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published + const gdoc = await getAndLoadGdocById(trx, id, contentSource) + + if (!gdoc.published) { + await updateGdocContentOnly(trx, id, gdoc) + } + + res.set("Cache-Control", "no-store") + res.send(gdoc) + } catch (error) { + console.error("Error fetching gdoc", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) + } + } +) + +/** + * Handles all four `GdocPublishingAction` cases + * - SavingDraft (no action) + * - Publishing (index and bake) + * - Updating (index and bake (potentially via lightning deploy)) + * - Unpublishing (remove from index and bake) + */ +async function indexAndBakeGdocIfNeccesary( + trx: db.KnexReadWriteTransaction, + user: Required<DbInsertUser>, + prevGdoc: + | GdocPost + | GdocDataInsight + | GdocHomepage + | GdocAbout + | GdocAuthor, + nextGdoc: GdocPost | GdocDataInsight | GdocHomepage | GdocAbout | GdocAuthor +) { + const prevJson = prevGdoc.toJSON() + const nextJson = nextGdoc.toJSON() + const hasChanges = checkHasChanges(prevGdoc, nextGdoc) + const action = getPublishingAction(prevJson, nextJson) + const isGdocPost = checkIsGdocPostExcludingFragments(nextJson) + + await match(action) + .with(GdocPublishingAction.SavingDraft, lodash.noop) + .with(GdocPublishingAction.Publishing, async () => { + if (isGdocPost) { + await indexIndividualGdocPost( + nextJson, + trx, + // If the gdoc is being published for the first time, prevGdoc.slug will be undefined + // In that case, we pass nextJson.slug to see if it has any page views (i.e. from WP) + prevGdoc.slug || nextJson.slug + ) + } + await triggerStaticBuild(user, `${action} ${nextJson.slug}`) + }) + .with(GdocPublishingAction.Updating, async () => { + if (isGdocPost) { + await indexIndividualGdocPost(nextJson, trx, prevGdoc.slug) + } + if (checkIsLightningUpdate(prevJson, nextJson, hasChanges)) { + await enqueueLightningChange( + user, + `Lightning update ${nextJson.slug}`, + nextJson.slug + ) + } else { + await triggerStaticBuild(user, `${action} ${nextJson.slug}`) + } + }) + .with(GdocPublishingAction.Unpublishing, async () => { + if (isGdocPost) { + await removeIndividualGdocPostFromIndex(nextJson) + } + await triggerStaticBuild(user, `${action} ${nextJson.slug}`) + }) + .exhaustive() +} + +/** + * Only supports creating a new empty Gdoc or updating an existing one. Does not + * support creating a new Gdoc from an existing one. Relevant updates will + * trigger a deploy. + */ +putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { + const { id } = req.params + + if (isEmpty(req.body)) { + return createOrLoadGdocById(trx, id) + } + + const prevGdoc = await getAndLoadGdocById(trx, id) + if (!prevGdoc) throw new JsonError(`No Google Doc with id ${id} found`) + + const nextGdoc = gdocFromJSON(req.body) + await nextGdoc.loadState(trx) + + await addImagesToContentGraph(trx, nextGdoc) + + await setLinksForGdoc( + trx, + nextGdoc.id, + nextGdoc.links, + nextGdoc.published + ? GdocLinkUpdateMode.DeleteAndInsert + : GdocLinkUpdateMode.DeleteOnly + ) + + await upsertGdoc(trx, nextGdoc) + + await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc) + + return nextGdoc +}) + +async function validateTombstoneRelatedLinkUrl( + trx: db.KnexReadonlyTransaction, + relatedLink?: string +) { + if (!relatedLink || !relatedLink.startsWith(GDOCS_BASE_URL)) return + const id = relatedLink.match(gdocUrlRegex)?.[1] + if (!id) { + throw new JsonError(`Invalid related link: ${relatedLink}`) + } + const [gdoc] = await getMinimalGdocPostsByIds(trx, [id]) + if (!gdoc) { + throw new JsonError(`Google Doc with ID ${id} not found`) + } + if (!gdoc.published) { + throw new JsonError(`Google Doc with ID ${id} is not published`) + } +} + +deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { + const { id } = req.params + + const gdoc = await getGdocBaseObjectById(trx, id, false) + if (!gdoc) throw new JsonError(`No Google Doc with id ${id} found`) + + const gdocSlug = getCanonicalUrl("", gdoc) + const { tombstone } = req.body + + if (tombstone) { + await validateTombstoneRelatedLinkUrl(trx, tombstone.relatedLinkUrl) + const slug = gdocSlug.replace("/", "") + const { relatedLinkThumbnail } = tombstone + if (relatedLinkThumbnail) { + const thumbnailExists = await db.checkIsImageInDB( + trx, + relatedLinkThumbnail + ) + if (!thumbnailExists) { + throw new JsonError( + `Image with filename "${relatedLinkThumbnail}" not found` + ) + } + } + await trx + .table("posts_gdocs_tombstones") + .insert({ ...tombstone, gdocId: id, slug }) + await trx + .table("redirects") + .insert({ source: gdocSlug, target: `/deleted${gdocSlug}` }) + } + + await trx + .table("posts") + .where({ gdocSuccessorId: gdoc.id }) + .update({ gdocSuccessorId: null }) + + await trx.table(PostsGdocsLinksTableName).where({ sourceId: id }).delete() + await trx.table(PostsGdocsXImagesTableName).where({ gdocId: id }).delete() + await trx.table(PostsGdocsTableName).where({ id }).delete() + await trx + .table(PostsGdocsComponentsTableName) + .where({ gdocId: id }) + .delete() + if (gdoc.published && checkIsGdocPostExcludingFragments(gdoc)) { + await removeIndividualGdocPostFromIndex(gdoc) + } + if (gdoc.published) { + if (!tombstone && gdocSlug && gdocSlug !== "/") { + // Assets have TTL of one week in Cloudflare. Add a redirect to make sure + // the page is no longer accessible. + // https://developers.cloudflare.com/pages/configuration/serving-pages/#asset-retention + console.log(`Creating redirect for "${gdocSlug}" to "/"`) + await db.knexRawInsert( + trx, + `INSERT INTO redirects (source, target, ttl) + VALUES (?, ?, DATE_ADD(NOW(), INTERVAL 8 DAY))`, + [gdocSlug, "/"] + ) + } + await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`) + } + return {} +}) + +postRouteWithRWTransaction( + apiRouter, + "/gdocs/:gdocId/setTags", + async (req, res, trx) => { + const { gdocId } = req.params + const { tagIds } = req.body + const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ + id: id, + })) + + await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/images.ts b/adminSiteServer/apiRoutes/images.ts new file mode 100644 index 0000000000..0c5a611f33 --- /dev/null +++ b/adminSiteServer/apiRoutes/images.ts @@ -0,0 +1,252 @@ +import { DbEnrichedImage, JsonError } from "@ourworldindata/types" +import pMap from "p-map" +import { apiRouter } from "../apiRouter.js" +import { + getRouteNonIdempotentWithRWTransaction, + postRouteWithRWTransaction, + putRouteWithRWTransaction, + patchRouteWithRWTransaction, + deleteRouteWithRWTransaction, + getRouteWithROTransaction, +} from "../functionalRouterHelpers.js" +import { + validateImagePayload, + processImageContent, + uploadToCloudflare, + deleteFromCloudflare, +} from "../imagesHelpers.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" + +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/images.json", + async (_, res, trx) => { + try { + const images = await db.getCloudflareImages(trx) + res.set("Cache-Control", "no-store") + res.send({ images }) + } catch (error) { + console.error("Error fetching images", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) + } + } +) + +postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { + const { filename, type, content } = validateImagePayload(req.body) + + const { asBlob, dimensions, hash } = await processImageContent( + content, + type + ) + + const collision = await trx<DbEnrichedImage>("images") + .where({ + hash, + replacedBy: null, + }) + .first() + + if (collision) { + return { + success: false, + error: `An image with this content already exists (filename: ${collision.filename})`, + } + } + + const preexisting = await trx<DbEnrichedImage>("images") + .where("filename", "=", filename) + .first() + + if (preexisting) { + return { + success: false, + error: "An image with this filename already exists", + } + } + + const cloudflareId = await uploadToCloudflare(filename, asBlob) + + if (!cloudflareId) { + return { + success: false, + error: "Failed to upload image", + } + } + + await trx<DbEnrichedImage>("images").insert({ + filename, + originalWidth: dimensions.width, + originalHeight: dimensions.height, + cloudflareId, + updatedAt: new Date().getTime(), + userId: res.locals.user.id, + hash, + }) + + const image = await db.getCloudflareImage(trx, filename) + + return { + success: true, + image, + } +}) + +/** + * Similar to the POST route, but for updating an existing image. + * Creates a new image entry in the database and uploads the new image to Cloudflare. + * The old image is marked as replaced by the new image. + */ +putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { + const { type, content } = validateImagePayload(req.body) + const { asBlob, dimensions, hash } = await processImageContent( + content, + type + ) + const collision = await trx<DbEnrichedImage>("images") + .where({ + hash, + replacedBy: null, + }) + .first() + + if (collision) { + return { + success: false, + error: `An exact copy of this image already exists (filename: ${collision.filename})`, + } + } + + const { id } = req.params + + const image = await trx<DbEnrichedImage>("images") + .where("id", "=", id) + .first() + + if (!image) { + throw new JsonError(`No image found for id ${id}`, 404) + } + + const originalCloudflareId = image.cloudflareId + const originalFilename = image.filename + const originalAltText = image.defaultAlt + + if (!originalCloudflareId) { + throw new JsonError( + `Image with id ${id} has no associated Cloudflare image`, + 400 + ) + } + + const newCloudflareId = await uploadToCloudflare(originalFilename, asBlob) + + if (!newCloudflareId) { + throw new JsonError("Failed to upload image", 500) + } + + const [newImageId] = await trx<DbEnrichedImage>("images").insert({ + filename: originalFilename, + originalWidth: dimensions.width, + originalHeight: dimensions.height, + cloudflareId: newCloudflareId, + updatedAt: new Date().getTime(), + userId: res.locals.user.id, + defaultAlt: originalAltText, + hash, + version: image.version + 1, + }) + + await trx<DbEnrichedImage>("images").where("id", "=", id).update({ + replacedBy: newImageId, + }) + + const updated = await db.getCloudflareImage(trx, originalFilename) + + await triggerStaticBuild( + res.locals.user, + `Updating image "${originalFilename}"` + ) + + return { + success: true, + image: updated, + } +}) + +// Update alt text via patch +patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { + const { id } = req.params + + const image = await trx<DbEnrichedImage>("images") + .where("id", "=", id) + .first() + + if (!image) { + throw new JsonError(`No image found for id ${id}`, 404) + } + + const patchableImageProperties = ["defaultAlt"] as const + const patch = lodash.pick(req.body, patchableImageProperties) + + if (Object.keys(patch).length === 0) { + throw new JsonError("No patchable properties provided", 400) + } + + await trx("images").where({ id }).update(patch) + + const updated = await trx<DbEnrichedImage>("images") + .where("id", "=", id) + .first() + + return { + success: true, + image: updated, + } +}) + +deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { + const { id } = req.params + + const image = await trx<DbEnrichedImage>("images") + .where("id", "=", id) + .first() + + if (!image) { + throw new JsonError(`No image found for id ${id}`, 404) + } + if (!image.cloudflareId) { + throw new JsonError(`Image does not have a cloudflare ID`, 400) + } + + const replacementChain = await db.selectReplacementChainForImage(trx, id) + + await pMap( + replacementChain, + async (image) => { + if (image.cloudflareId) { + await deleteFromCloudflare(image.cloudflareId) + } + }, + { concurrency: 5 } + ) + + // There's an ON DELETE CASCADE which will delete the replacements + await trx("images").where({ id }).delete() + + return { + success: true, + } +}) + +getRouteWithROTransaction(apiRouter, "/images/usage", async (_, __, trx) => { + const usage = await db.getImageUsage(trx) + + return { + success: true, + usage, + } +}) diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts new file mode 100644 index 0000000000..a26116472e --- /dev/null +++ b/adminSiteServer/apiRoutes/mdims.ts @@ -0,0 +1,34 @@ +import { JsonError, MultiDimDataPageConfigRaw } from "@ourworldindata/types" +import { isMultiDimDataPagePublished } from "../../db/model/MultiDimDataPage.js" +import { isValidSlug } from "../../serverUtils/serverUtil.js" +import { + FEATURE_FLAGS, + FeatureFlagFeature, +} from "../../settings/clientSettings.js" +import { apiRouter } from "../apiRouter.js" +import { putRouteWithRWTransaction } from "../functionalRouterHelpers.js" +import { createMultiDimConfig } from "../multiDim.js" +import { triggerStaticBuild } from "./routeUtils.js" + +putRouteWithRWTransaction( + apiRouter, + "/multi-dim/:slug", + async (req, res, trx) => { + const { slug } = req.params + if (!isValidSlug(slug)) { + throw new JsonError(`Invalid multi-dim slug ${slug}`) + } + const rawConfig = req.body as MultiDimDataPageConfigRaw + const id = await createMultiDimConfig(trx, slug, rawConfig) + if ( + FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && + (await isMultiDimDataPagePublished(trx, slug)) + ) { + await triggerStaticBuild( + res.locals.user, + `Publishing multidimensional chart ${slug}` + ) + } + return { success: true, id } + } +) diff --git a/adminSiteServer/apiRoutes/misc.ts b/adminSiteServer/apiRoutes/misc.ts new file mode 100644 index 0000000000..eb4efaef31 --- /dev/null +++ b/adminSiteServer/apiRoutes/misc.ts @@ -0,0 +1,183 @@ +// Get an ArchieML output of all the work produced by an author. This includes +// gdoc articles, gdoc modular/linear topic pages and wordpress modular topic +// pages. Data insights are excluded. This is used to manually populate the +// [.secondary] section of the {.research-and-writing} block of author pages + +import { DbRawPostGdoc, JsonError } from "@ourworldindata/types" +import { apiRouter } from "../apiRouter.js" +import { getRouteWithROTransaction } from "../functionalRouterHelpers.js" + +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import path from "path" +import { DeployQueueServer } from "../../baker/DeployQueueServer.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { triggerStaticBuild } from "./routeUtils.js" +// using the alternate template, which highlights topics rather than articles. +getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { + type WordpressPageRecord = { + isWordpressPage: number + } & Record< + "slug" | "title" | "subtitle" | "thumbnail" | "authors" | "publishedAt", + string + > + type GdocRecord = Pick<DbRawPostGdoc, "id" | "publishedAt"> + + const author = req.query.author + const gdocs = await db.knexRaw<GdocRecord>( + trx, + `-- sql + SELECT id, publishedAt + FROM posts_gdocs + WHERE JSON_CONTAINS(content->'$.authors', '"${author}"') + AND type NOT IN ("data-insight", "fragment") + AND published = 1 + ` + ) + + // type: page + const wpModularTopicPages = await db.knexRaw<WordpressPageRecord>( + trx, + `-- sql + SELECT + wpApiSnapshot->>"$.slug" as slug, + wpApiSnapshot->>"$.title.rendered" as title, + wpApiSnapshot->>"$.excerpt.rendered" as subtitle, + TRUE as isWordpressPage, + wpApiSnapshot->>"$.authors_name" as authors, + wpApiSnapshot->>"$.featured_media_paths.medium_large" as thumbnail, + wpApiSnapshot->>"$.date" as publishedAt + FROM posts p + WHERE wpApiSnapshot->>"$.content" LIKE '%topic-page%' + AND JSON_CONTAINS(wpApiSnapshot->'$.authors_name', '"${author}"') + AND wpApiSnapshot->>"$.status" = 'publish' + AND NOT EXISTS ( + SELECT 1 FROM posts_gdocs pg + WHERE pg.slug = p.slug + AND pg.content->>'$.type' LIKE '%topic-page' + ) + ` + ) + + const isWordpressPage = ( + post: WordpressPageRecord | GdocRecord + ): post is WordpressPageRecord => + (post as WordpressPageRecord).isWordpressPage === 1 + + function* generateProperty(key: string, value: string) { + yield `${key}: ${value}\n` + } + + const sortByDateDesc = ( + a: GdocRecord | WordpressPageRecord, + b: GdocRecord | WordpressPageRecord + ): number => { + if (!a.publishedAt || !b.publishedAt) return 0 + return ( + new Date(b.publishedAt).getTime() - + new Date(a.publishedAt).getTime() + ) + } + + function* generateAllWorkArchieMl() { + for (const post of [...gdocs, ...wpModularTopicPages].sort( + sortByDateDesc + )) { + if (isWordpressPage(post)) { + yield* generateProperty( + "url", + `https://ourworldindata.org/${post.slug}` + ) + yield* generateProperty("title", post.title) + yield* generateProperty("subtitle", post.subtitle) + yield* generateProperty( + "authors", + JSON.parse(post.authors).join(", ") + ) + const parsedPath = path.parse(post.thumbnail) + yield* generateProperty( + "filename", + // /app/uploads/2021/09/reducing-fertilizer-768x301.png -> reducing-fertilizer.png + path.format({ + name: parsedPath.name.replace(/-\d+x\d+$/, ""), + ext: parsedPath.ext, + }) + ) + yield "\n" + } else { + // this is a gdoc + yield* generateProperty( + "url", + `https://docs.google.com/document/d/${post.id}/edit` + ) + yield "\n" + } + } + } + + res.type("text/plain") + return [...generateAllWorkArchieMl()].join("") +}) + +getRouteWithROTransaction( + apiRouter, + "/editorData/namespaces.json", + async (req, res, trx) => { + const rows = await db.knexRaw<{ + name: string + description?: string + isArchived: boolean + }>( + trx, + `SELECT DISTINCT + namespace AS name, + namespaces.description AS description, + namespaces.isArchived AS isArchived + FROM active_datasets + JOIN namespaces ON namespaces.name = active_datasets.namespace` + ) + + return { + namespaces: lodash + .sortBy(rows, (row) => row.description) + .map((namespace) => ({ + ...namespace, + isArchived: !!namespace.isArchived, + })), + } + } +) + +getRouteWithROTransaction( + apiRouter, + "/sources/:sourceId.json", + async (req, res, trx) => { + const sourceId = expectInt(req.params.sourceId) + + const source = await db.knexRawFirst<Record<string, any>>( + trx, + ` + SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace + FROM sources AS s + JOIN active_datasets AS d ON d.id=s.datasetId + WHERE s.id=?`, + [sourceId] + ) + if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) + source.variables = await db.knexRaw( + trx, + `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, + [sourceId] + ) + + return { source: source } + } +) + +apiRouter.get("/deploys.json", async () => ({ + deploys: await new DeployQueueServer().getDeploys(), +})) + +apiRouter.put("/deploy", async (req, res) => { + return triggerStaticBuild(res.locals.user, "Manually triggered deploy") +}) diff --git a/adminSiteServer/apiRoutes/posts.ts b/adminSiteServer/apiRoutes/posts.ts new file mode 100644 index 0000000000..4f36fd0a12 --- /dev/null +++ b/adminSiteServer/apiRoutes/posts.ts @@ -0,0 +1,220 @@ +import { + PostsTableName, + DbRawPost, + DbRawPostWithGdocPublishStatus, + JsonError, + OwidGdocPostInterface, + OwidGdocType, + PostsGdocsTableName, +} from "@ourworldindata/types" +import { camelCaseProperties } from "@ourworldindata/utils" +import { createGdocAndInsertOwidGdocPostContent } from "../../db/model/Gdoc/archieToGdoc.js" +import { upsertGdoc, setTagsForGdoc } from "../../db/model/Gdoc/GdocFactory.js" +import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" +import { setTagsForPost, getTagsByPostId } from "../../db/model/Post.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + postRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import * as db from "../../db/db.js" + +getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { + const raw_rows = await db.knexRaw( + trx, + `-- sql + WITH + posts_tags_aggregated AS ( + SELECT + post_id, + IF( + COUNT(tags.id) = 0, + JSON_ARRAY(), + JSON_ARRAYAGG(JSON_OBJECT("id", tags.id, "name", tags.name)) + ) AS tags + FROM + post_tags + LEFT JOIN tags ON tags.id = post_tags.tag_id + GROUP BY + post_id + ), + post_gdoc_slug_successors AS ( + SELECT + posts.id, + IF( + COUNT(gdocSlugSuccessor.id) = 0, + JSON_ARRAY(), + JSON_ARRAYAGG( + JSON_OBJECT("id", gdocSlugSuccessor.id, "published", gdocSlugSuccessor.published) + ) + ) AS gdocSlugSuccessors + FROM + posts + LEFT JOIN posts_gdocs gdocSlugSuccessor ON gdocSlugSuccessor.slug = posts.slug + GROUP BY + posts.id + ) + SELECT + posts.id AS id, + posts.title AS title, + posts.type AS TYPE, + posts.slug AS slug, + STATUS, + updated_at_in_wordpress, + posts.authors, + posts_tags_aggregated.tags AS tags, + gdocSuccessorId, + gdocSuccessor.published AS isGdocSuccessorPublished, + -- posts can either have explict successors via the gdocSuccessorId column + -- or implicit successors if a gdoc has been created that uses the same slug + -- as a Wp post (the gdoc one wins once it is published) + post_gdoc_slug_successors.gdocSlugSuccessors AS gdocSlugSuccessors + FROM + posts + LEFT JOIN post_gdoc_slug_successors ON post_gdoc_slug_successors.id = posts.id + LEFT JOIN posts_gdocs gdocSuccessor ON gdocSuccessor.id = posts.gdocSuccessorId + LEFT JOIN posts_tags_aggregated ON posts_tags_aggregated.post_id = posts.id + ORDER BY + updated_at_in_wordpress DESC`, + [] + ) + const rows = raw_rows.map((row: any) => ({ + ...row, + tags: JSON.parse(row.tags), + isGdocSuccessorPublished: !!row.isGdocSuccessorPublished, + gdocSlugSuccessors: JSON.parse(row.gdocSlugSuccessors), + authors: JSON.parse(row.authors), + })) + + return { posts: rows } +}) + +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/setTags", + async (req, res, trx) => { + const postId = expectInt(req.params.postId) + + await setTagsForPost(trx, postId, req.body.tagIds) + + return { success: true } + } +) + +getRouteWithROTransaction( + apiRouter, + "/posts/:postId.json", + async (req, res, trx) => { + const postId = expectInt(req.params.postId) + const post = (await trx + .table(PostsTableName) + .where({ id: postId }) + .select("*") + .first()) as DbRawPost | undefined + return camelCaseProperties({ ...post }) + } +) + +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/createGdoc", + async (req, res, trx) => { + const postId = expectInt(req.params.postId) + const allowRecreate = !!req.body.allowRecreate + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined + + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!allowRecreate && existingGdocId) + throw new JsonError("A gdoc already exists for this post", 400) + if (allowRecreate && existingGdocId && post.isGdocPublished) { + throw new JsonError( + "A gdoc already exists for this post and it is already published", + 400 + ) + } + if (post.archieml === null) + throw new JsonError( + `ArchieML was not present for post with id ${postId}`, + 500 + ) + const tagsByPostId = await getTagsByPostId(trx) + const tags = tagsByPostId.get(postId) || [] + const archieMl = JSON.parse( + // Google Docs interprets ®ion in grapher URLS as ®ion + // So we escape them here + post.archieml.replaceAll("&", "&") + ) as OwidGdocPostInterface + const gdocId = await createGdocAndInsertOwidGdocPostContent( + archieMl.content, + post.gdocSuccessorId + ) + // If we did not yet have a gdoc associated with this post, we need to register + // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise + // we don't need to make changes to the DB (only the gdoc regeneration was required) + if (!existingGdocId) { + post.gdocSuccessorId = gdocId + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx + .table(PostsTableName) + .where({ id: postId }) + .update("gdocSuccessorId", gdocId) + + const gdoc = new GdocPost(gdocId) + gdoc.slug = post.slug + gdoc.content.title = post.title + gdoc.content.type = archieMl.content.type || OwidGdocType.Article + gdoc.published = false + gdoc.createdAt = new Date() + gdoc.publishedAt = post.published_at + await upsertGdoc(trx, gdoc) + await setTagsForGdoc(trx, gdocId, tags) + } + return { googleDocsId: gdocId } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/unlinkGdoc", + async (req, res, trx) => { + const postId = expectInt(req.params.postId) + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined + + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!existingGdocId) + throw new JsonError("No gdoc exists for this post", 400) + if (existingGdocId && post.isGdocPublished) { + throw new JsonError( + "The GDoc is already published - you can't unlink it", + 400 + ) + } + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx + .table(PostsTableName) + .where({ id: postId }) + .update("gdocSuccessorId", null) + + await trx + .table(PostsGdocsTableName) + .where({ id: existingGdocId }) + .delete() + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/redirects.ts b/adminSiteServer/apiRoutes/redirects.ts new file mode 100644 index 0000000000..0752c4ece1 --- /dev/null +++ b/adminSiteServer/apiRoutes/redirects.ts @@ -0,0 +1,152 @@ +import { DbPlainChartSlugRedirect, JsonError } from "@ourworldindata/types" +import { getRedirects } from "../../baker/redirects.js" +import { + redirectWithSourceExists, + getChainedRedirect, + getRedirectById, +} from "../../db/model/Redirect.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + postRouteWithRWTransaction, + deleteRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as db from "../../db/db.js" + +getRouteWithROTransaction( + apiRouter, + "/site-redirects.json", + async (req, res, trx) => ({ redirects: await getRedirects(trx) }) +) + +postRouteWithRWTransaction( + apiRouter, + "/site-redirects/new", + async (req, res, trx) => { + const { source, target } = req.body + const sourceAsUrl = new URL(source, "https://ourworldindata.org") + if (sourceAsUrl.pathname === "/") + throw new JsonError("Cannot redirect from /", 400) + if (await redirectWithSourceExists(trx, source)) { + throw new JsonError( + `Redirect with source ${source} already exists`, + 400 + ) + } + const chainedRedirect = await getChainedRedirect(trx, source, target) + if (chainedRedirect) { + throw new JsonError( + "Creating this redirect would create a chain, redirect from " + + `${chainedRedirect.source} to ${chainedRedirect.target} ` + + "already exists. " + + (target === chainedRedirect.source + ? `Please create the redirect from ${source} to ` + + `${chainedRedirect.target} directly instead.` + : `Please delete the existing redirect and create a ` + + `new redirect from ${chainedRedirect.source} to ` + + `${target} instead.`), + 400 + ) + } + const { insertId: id } = await db.knexRawInsert( + trx, + `INSERT INTO redirects (source, target) VALUES (?, ?)`, + [source, target] + ) + await triggerStaticBuild( + res.locals.user, + `Creating redirect id=${id} source=${source} target=${target}` + ) + return { success: true, redirect: { id, source, target } } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/site-redirects/:id", + async (req, res, trx) => { + const id = expectInt(req.params.id) + const redirect = await getRedirectById(trx, id) + if (!redirect) { + throw new JsonError(`No redirect found for id ${id}`, 404) + } + await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` + ) + return { success: true } + } +) + +// Get a list of redirects that map old slugs to charts +getRouteWithROTransaction( + apiRouter, + "/redirects.json", + async (req, res, trx) => ({ + redirects: await db.knexRaw( + trx, + `-- sql + SELECT + r.id, + r.slug, + r.chart_id as chartId, + chart_configs.slug AS chartSlug + FROM chart_slug_redirects AS r + JOIN charts ON charts.id = r.chart_id + JOIN chart_configs ON chart_configs.id = charts.configId + ORDER BY r.id DESC + ` + ), + }) +) + +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/redirects/new", + async (req, res, trx) => { + const chartId = expectInt(req.params.chartId) + const fields = req.body as { slug: string } + const result = await db.knexRawInsert( + trx, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [chartId, fields.slug] + ) + const redirectId = result.insertId + const redirect = await db.knexRaw<DbPlainChartSlugRedirect>( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [redirectId] + ) + return { success: true, redirect: redirect } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/redirects/:id", + async (req, res, trx) => { + const id = expectInt(req.params.id) + + const redirect = await db.knexRawFirst<DbPlainChartSlugRedirect>( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [id] + ) + + if (!redirect) + throw new JsonError(`No redirect found for id ${id}`, 404) + + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [ + id, + ]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect from ${redirect.slug}` + ) + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/routeUtils.ts b/adminSiteServer/apiRoutes/routeUtils.ts new file mode 100644 index 0000000000..c9a8bbc908 --- /dev/null +++ b/adminSiteServer/apiRoutes/routeUtils.ts @@ -0,0 +1,51 @@ +import { DbPlainUser } from "@ourworldindata/types" +import { DeployQueueServer } from "../../baker/DeployQueueServer.js" +import { BAKE_ON_CHANGE } from "../../settings/serverSettings.js" +import { References } from "../../adminSiteClient/AbstractChartEditor.js" +import { ChartViewMinimalInformation } from "../../adminSiteClient/ChartEditor.js" +import * as db from "../../db/db.js" +import { + getWordpressPostReferencesByChartId, + getGdocsPostReferencesByChartId, +} from "../../db/model/Post.js" + +// Call this to trigger build and deployment of static charts on change +export const triggerStaticBuild = async ( + user: DbPlainUser, + commitMessage: string +) => { + if (!BAKE_ON_CHANGE) { + console.log( + "Not triggering static build because BAKE_ON_CHANGE is false" + ) + return + } + + return new DeployQueueServer().enqueueChange({ + timeISOString: new Date().toISOString(), + authorName: user.fullName, + authorEmail: user.email, + message: commitMessage, + }) +} + +export const enqueueLightningChange = async ( + user: DbPlainUser, + commitMessage: string, + slug: string +) => { + if (!BAKE_ON_CHANGE) { + console.log( + "Not triggering static build because BAKE_ON_CHANGE is false" + ) + return + } + + return new DeployQueueServer().enqueueChange({ + timeISOString: new Date().toISOString(), + authorName: user.fullName, + authorEmail: user.email, + message: commitMessage, + slug, + }) +} diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts new file mode 100644 index 0000000000..2b9e3303fa --- /dev/null +++ b/adminSiteServer/apiRoutes/suggest.ts @@ -0,0 +1,71 @@ +import { + TaggableType, + DbChartTagJoin, + JsonError, + DbEnrichedImage, +} from "@ourworldindata/types" +import { parseIntOrUndefined } from "@ourworldindata/utils" +import { getGptTopicSuggestions } from "../../db/model/Chart.js" +import { CLOUDFLARE_IMAGES_URL } from "../../settings/clientSettings.js" +import { apiRouter } from "../apiRouter.js" +import { getRouteWithROTransaction } from "../functionalRouterHelpers.js" +import { fetchGptGeneratedAltText } from "../imagesHelpers.js" + +getRouteWithROTransaction( + apiRouter, + `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, + async (req, res, trx): Promise<Record<"topics", DbChartTagJoin[]>> => { + const chartId = parseIntOrUndefined(req.params.chartId) + if (!chartId) throw new JsonError(`Invalid chart ID`, 400) + + const topics = await getGptTopicSuggestions(trx, chartId) + + if (!topics.length) + throw new JsonError( + `No GPT topic suggestions found for chart ${chartId}`, + 404 + ) + + return { + topics, + } + } +) + +getRouteWithROTransaction( + apiRouter, + `/gpt/suggest-alt-text/:imageId`, + async ( + req, + res, + trx + ): Promise<{ + success: boolean + altText: string | null + }> => { + const imageId = parseIntOrUndefined(req.params.imageId) + if (!imageId) throw new JsonError(`Invalid image ID`, 400) + const image = await trx<DbEnrichedImage>("images") + .where("id", imageId) + .first() + if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) + + const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` + let altText: string | null = "" + try { + altText = await fetchGptGeneratedAltText(src) + } catch (error) { + console.error( + `Error fetching GPT alt text for image ${imageId}`, + error + ) + throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) + } + + if (!altText) { + throw new JsonError(`Unable to generate alt text for image`, 404) + } + + return { success: true, altText } + } +) diff --git a/adminSiteServer/apiRoutes/tagGraph.ts b/adminSiteServer/apiRoutes/tagGraph.ts new file mode 100644 index 0000000000..3690e2e541 --- /dev/null +++ b/adminSiteServer/apiRoutes/tagGraph.ts @@ -0,0 +1,60 @@ +import { JsonError, FlatTagGraph } from "@ourworldindata/types" +import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + postRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" + +getRouteWithROTransaction( + apiRouter, + "/flatTagGraph.json", + async (req, res, trx) => { + const flatTagGraph = await db.getFlatTagGraph(trx) + return flatTagGraph + } +) + +postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { + const tagGraph = req.body?.tagGraph as unknown + if (!tagGraph) { + throw new JsonError("No tagGraph provided", 400) + } + + function validateFlatTagGraph( + tagGraph: Record<any, any> + ): tagGraph is FlatTagGraph { + if (lodash.isObject(tagGraph)) { + for (const [key, value] of Object.entries(tagGraph)) { + if (!lodash.isString(key) && isNaN(Number(key))) { + return false + } + if (!lodash.isArray(value)) { + return false + } + for (const tag of value) { + if ( + !( + checkIsPlainObjectWithGuard(tag) && + lodash.isNumber(tag.weight) && + lodash.isNumber(tag.parentId) && + lodash.isNumber(tag.childId) + ) + ) { + return false + } + } + } + } + + return true + } + const isValid = validateFlatTagGraph(tagGraph) + if (!isValid) { + throw new JsonError("Invalid tag graph provided", 400) + } + await db.updateTagGraph(trx, tagGraph) + res.send({ success: true }) +}) diff --git a/adminSiteServer/apiRoutes/tags.ts b/adminSiteServer/apiRoutes/tags.ts new file mode 100644 index 0000000000..0e698df454 --- /dev/null +++ b/adminSiteServer/apiRoutes/tags.ts @@ -0,0 +1,269 @@ +import { + DbPlainTag, + DbPlainDataset, + DbRawPostGdoc, + JsonError, +} from "@ourworldindata/types" +import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils" +import { + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, +} from "../../db/model/Chart.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { UNCATEGORIZED_TAG_ID } from "../../settings/serverSettings.js" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + putRouteWithRWTransaction, + postRouteWithRWTransaction, + deleteRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import * as db from "../../db/db.js" +import * as lodash from "lodash" +import { Request } from "../authentication.js" + +getRouteWithROTransaction( + apiRouter, + "/tags/:tagId.json", + async (req, res, trx) => { + const tagId = expectInt(req.params.tagId) as number | null + + // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff + // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag + // every time we create a new chart etcs + const uncategorized = tagId === UNCATEGORIZED_TAG_ID + + // TODO: when we have types for our endpoints, make tag of that type instead of any + const tag: any = await db.knexRawFirst< + Pick< + DbPlainTag, + | "id" + | "name" + | "specialType" + | "updatedAt" + | "parentId" + | "slug" + > + >( + trx, + `-- sql + SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug + FROM tags t LEFT JOIN tags p ON t.parentId=p.id + WHERE t.id = ? + `, + [tagId] + ) + + // Datasets tagged with this tag + const datasets = await db.knexRaw< + Pick< + DbPlainDataset, + | "id" + | "namespace" + | "name" + | "description" + | "createdAt" + | "updatedAt" + | "dataEditedAt" + | "isPrivate" + | "nonRedistributable" + > & { dataEditedByUserName: string } + >( + trx, + `-- sql + SELECT + d.id, + d.namespace, + d.name, + d.description, + d.createdAt, + d.updatedAt, + d.dataEditedAt, + du.fullName AS dataEditedByUserName, + d.isPrivate, + d.nonRedistributable + FROM active_datasets d + JOIN users du ON du.id=d.dataEditedByUserId + LEFT JOIN dataset_tags dt ON dt.datasetId = d.id + WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} + ORDER BY d.dataEditedAt DESC + `, + uncategorized ? [] : [tagId] + ) + tag.datasets = datasets + + // The other tags for those datasets + if (tag.datasets.length) { + if (uncategorized) { + for (const dataset of tag.datasets) dataset.tags = [] + } else { + const datasetTags = await db.knexRaw<{ + datasetId: number + id: number + name: string + }>( + trx, + `-- sql + SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt + JOIN tags t ON dt.tagId = t.id + WHERE dt.datasetId IN (?) + `, + [tag.datasets.map((d: any) => d.id)] + ) + const tagsByDatasetId = lodash.groupBy( + datasetTags, + (t) => t.datasetId + ) + for (const dataset of tag.datasets) { + dataset.tags = tagsByDatasetId[dataset.id].map((t) => + lodash.omit(t, "datasetId") + ) + } + } + } + + // Charts using datasets under this tag + const charts = await db.knexRaw<OldChartFieldList>( + trx, + `-- sql + SELECT ${oldChartFieldList} FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + LEFT JOIN chart_tags ct ON ct.chartId=charts.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE ct.tagId ${tagId === UNCATEGORIZED_TAG_ID ? "IS NULL" : "= ?"} + GROUP BY charts.id + ORDER BY charts.updatedAt DESC + `, + uncategorized ? [] : [tagId] + ) + tag.charts = charts + + await assignTagsForCharts(trx, charts) + + // Subcategories + const children = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql + SELECT t.id, t.name FROM tags t + WHERE t.parentId = ? + `, + [tag.id] + ) + tag.children = children + + // Possible parents to choose from + const possibleParents = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql + SELECT t.id, t.name FROM tags t + WHERE t.parentId IS NULL + ` + ) + tag.possibleParents = possibleParents + + return { + tag, + } + } +) + +putRouteWithRWTransaction( + apiRouter, + "/tags/:tagId", + async (req: Request, res, trx) => { + const tagId = expectInt(req.params.tagId) + const tag = (req.body as { tag: any }).tag + await db.knexRaw( + trx, + `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, + [tag.name, new Date(), tag.slug, tagId] + ) + if (tag.slug) { + // See if there's a published gdoc with a matching slug. + // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, + // where the page for the topic is just an article. + const gdoc = await db.knexRaw<Pick<DbRawPostGdoc, "slug">>( + trx, + `-- sql + SELECT slug FROM posts_gdocs pg + WHERE EXISTS ( + SELECT 1 + FROM posts_gdocs_x_tags gt + WHERE pg.id = gt.gdocId AND gt.tagId = ? + ) AND pg.published = TRUE AND pg.slug = ?`, + [tagId, tag.slug] + ) + if (!gdoc.length) { + return { + success: true, + tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. + +Are you sure you haven't made a typo?`, + } + } + } + return { success: true } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/tags/new", + async (req: Request, res, trx) => { + const tag = req.body + function validateTag( + tag: unknown + ): tag is { name: string; slug: string | null } { + return ( + checkIsPlainObjectWithGuard(tag) && + typeof tag.name === "string" && + (tag.slug === null || + (typeof tag.slug === "string" && tag.slug !== "")) + ) + } + if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) + + const conflictingTag = await db.knexRawFirst<{ + name: string + slug: string | null + }>( + trx, + `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, + [tag.name, tag.slug] + ) + if (conflictingTag) + throw new JsonError( + conflictingTag.name === tag.name + ? `Tag with name ${tag.name} already exists` + : `Tag with slug ${tag.slug} already exists`, + 400 + ) + + const now = new Date() + const result = await db.knexRawInsert( + trx, + `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, + // parentId will be deprecated soon once we migrate fully to the tag graph + [tag.name, tag.slug, now, now] + ) + return { success: true, tagId: result.insertId } + } +) + +getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { + return { tags: await db.getMinimalTagsWithIsTopic(trx) } +}) + +deleteRouteWithRWTransaction( + apiRouter, + "/tags/:tagId/delete", + async (req, res, trx) => { + const tagId = expectInt(req.params.tagId) + + await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) + + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/users.ts b/adminSiteServer/apiRoutes/users.ts new file mode 100644 index 0000000000..256ad22995 --- /dev/null +++ b/adminSiteServer/apiRoutes/users.ts @@ -0,0 +1,118 @@ +import { DbPlainUser, UsersTableName, JsonError } from "@ourworldindata/types" +import { parseIntOrUndefined } from "@ourworldindata/utils" +import { pick } from "lodash" +import { getUserById, updateUser, insertUser } from "../../db/model/User.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { apiRouter } from "../apiRouter.js" +import { + getRouteWithROTransaction, + deleteRouteWithRWTransaction, + putRouteWithRWTransaction, + postRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import * as db from "../../db/db.js" + +getRouteWithROTransaction(apiRouter, "/users.json", async (req, res, trx) => ({ + users: await trx + .select( + "id" satisfies keyof DbPlainUser, + "email" satisfies keyof DbPlainUser, + "fullName" satisfies keyof DbPlainUser, + "isActive" satisfies keyof DbPlainUser, + "isSuperuser" satisfies keyof DbPlainUser, + "createdAt" satisfies keyof DbPlainUser, + "updatedAt" satisfies keyof DbPlainUser, + "lastLogin" satisfies keyof DbPlainUser, + "lastSeen" satisfies keyof DbPlainUser + ) + .from<DbPlainUser>(UsersTableName) + .orderBy("lastSeen", "desc"), +})) + +getRouteWithROTransaction( + apiRouter, + "/users/:userId.json", + async (req, res, trx) => { + const id = parseIntOrUndefined(req.params.userId) + if (!id) throw new JsonError("No user id given") + const user = await getUserById(trx, id) + return { user } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/users/:userId", + async (req, res, trx) => { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = expectInt(req.params.userId) + await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) + + return { success: true } + } +) + +putRouteWithRWTransaction( + apiRouter, + "/users/:userId", + async (req, res, trx: db.KnexReadWriteTransaction) => { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = parseIntOrUndefined(req.params.userId) + const user = + userId !== undefined ? await getUserById(trx, userId) : null + if (!user) throw new JsonError("No such user", 404) + + user.fullName = req.body.fullName + user.isActive = req.body.isActive + + await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) + + return { success: true } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/users/add", + async (req, res, trx: db.KnexReadWriteTransaction) => { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const { email, fullName } = req.body + + await insertUser(trx, { + email, + fullName, + }) + + return { success: true } + } +) + +postRouteWithRWTransaction( + apiRouter, + "/users/:userId/images/:imageId", + async (req, res, trx) => { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId }).update({ userId }) + return { success: true } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/users/:userId/images/:imageId", + async (req, res, trx) => { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images") + .where({ id: imageId, userId }) + .update({ userId: null }) + return { success: true } + } +) diff --git a/adminSiteServer/apiRoutes/variables.ts b/adminSiteServer/apiRoutes/variables.ts new file mode 100644 index 0000000000..0853e92934 --- /dev/null +++ b/adminSiteServer/apiRoutes/variables.ts @@ -0,0 +1,547 @@ +import { + getVariableDataRoute, + getVariableMetadataRoute, + migrateGrapherConfigToLatestVersion, +} from "@ourworldindata/grapher" +import { + DbRawVariable, + DbPlainDataset, + JsonError, + DbPlainChart, + DbRawChartConfig, + GrapherInterface, + OwidVariableWithSource, + parseChartConfig, +} from "@ourworldindata/types" +import { + fetchS3DataValuesByPath, + fetchS3MetadataByPath, + getAllChartsForIndicator, + getGrapherConfigsForVariable, + getMergedGrapherConfigForVariable, + searchVariables, + updateAllChartsThatInheritFromIndicator, + updateAllMultiDimViewsThatInheritFromIndicator, + updateGrapherConfigAdminOfVariable, + updateGrapherConfigETLOfVariable, +} from "../../db/model/Variable.js" +import { DATA_API_URL } from "../../settings/clientSettings.js" +import { apiRouter } from "../apiRouter.js" +import { + deleteRouteWithRWTransaction, + getRouteWithROTransaction, + putRouteWithRWTransaction, +} from "../functionalRouterHelpers.js" +import * as db from "../../db/db.js" +import { + getParentVariableIdFromChartConfig, + omit, + parseIntOrUndefined, +} from "@ourworldindata/utils" +import { + OldChartFieldList, + oldChartFieldList, + assignTagsForCharts, +} from "../../db/model/Chart.js" +import { updateExistingFullConfig } from "../../db/model/ChartConfigs.js" +import { expectInt } from "../../serverUtils/serverUtil.js" +import { triggerStaticBuild } from "./routeUtils.js" +import * as lodash from "lodash" +import { updateGrapherConfigsInR2 } from "./charts.js" + +getRouteWithROTransaction( + apiRouter, + "/editorData/variables.json", + async (req, res, trx) => { + const datasets = [] + const rows = await db.knexRaw< + Pick<DbRawVariable, "name" | "id"> & { + datasetId: number + datasetName: string + datasetVersion: string + } & Pick< + DbPlainDataset, + "namespace" | "isPrivate" | "nonRedistributable" + > + >( + trx, + `-- sql + SELECT + v.name, + v.id, + d.id as datasetId, + d.name as datasetName, + d.version as datasetVersion, + d.namespace, + d.isPrivate, + d.nonRedistributable + FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id + ORDER BY d.updatedAt DESC + ` + ) + + let dataset: + | { + id: number + name: string + version: string + namespace: string + isPrivate: boolean + nonRedistributable: boolean + variables: { id: number; name: string }[] + } + | undefined + for (const row of rows) { + if (!dataset || row.datasetName !== dataset.name) { + if (dataset) datasets.push(dataset) + + dataset = { + id: row.datasetId, + name: row.datasetName, + version: row.datasetVersion, + namespace: row.namespace, + isPrivate: !!row.isPrivate, + nonRedistributable: !!row.nonRedistributable, + variables: [], + } + } + + dataset.variables.push({ + id: row.id, + name: row.name ?? "", + }) + } + + if (dataset) datasets.push(dataset) + + return { datasets: datasets } + } +) + +apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { + const variableStr = req.params.variableStr as string + if (!variableStr) throw new JsonError("No variable id given") + if (variableStr.includes("+")) + throw new JsonError( + "Requesting multiple variables at the same time is no longer supported" + ) + const variableId = parseInt(variableStr) + if (isNaN(variableId)) throw new JsonError("Invalid variable id") + return await fetchS3DataValuesByPath( + getVariableDataRoute(DATA_API_URL, variableId) + "?nocache" + ) +}) + +apiRouter.get( + "/data/variables/metadata/:variableStr.json", + async (req, res) => { + const variableStr = req.params.variableStr as string + if (!variableStr) throw new JsonError("No variable id given") + if (variableStr.includes("+")) + throw new JsonError( + "Requesting multiple variables at the same time is no longer supported" + ) + const variableId = parseInt(variableStr) + if (isNaN(variableId)) throw new JsonError("Invalid variable id") + return await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) + } +) + +getRouteWithROTransaction( + apiRouter, + "/variables.json", + async (req, res, trx) => { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 + const query = req.query.search as string + return await searchVariables(query, limit, trx) + } +) + +getRouteWithROTransaction( + apiRouter, + "/variables.usages.json", + async (req, res, trx) => { + const query = `-- sql + SELECT + variableId, + COUNT(DISTINCT chartId) AS usageCount + FROM + chart_dimensions + GROUP BY + variableId + ORDER BY + usageCount DESC` + + const rows = await db.knexRaw(trx, query) + + return rows + } +) + +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigETL/:variableId.patchConfig.json", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + return variable.etl?.patchConfig ?? {} + } +) + +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigAdmin/:variableId.patchConfig.json", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + return variable.admin?.patchConfig ?? {} + } +) + +getRouteWithROTransaction( + apiRouter, + "/variables/mergedGrapherConfig/:variableId.json", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + const config = await getMergedGrapherConfigForVariable(trx, variableId) + return config ?? {} + } +) + +// Used in VariableEditPage +getRouteWithROTransaction( + apiRouter, + "/variables/:variableId.json", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + + const variable = await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) + + // XXX: Patch shortName onto the end of catalogPath when it's missing, + // a temporary hack since our S3 metadata is out of date with our DB. + // See: https://github.com/owid/etl/issues/2135 + if (variable.catalogPath && !variable.catalogPath.includes("#")) { + variable.catalogPath += `#${variable.shortName}` + } + + const rawCharts = await db.knexRaw< + OldChartFieldList & { + isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] + config: DbRawChartConfig["full"] + } + >( + trx, + `-- sql + SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + JOIN chart_dimensions cd ON cd.chartId = charts.id + WHERE cd.variableId = ? + GROUP BY charts.id + `, + [variableId] + ) + + // check for parent indicators + const charts = rawCharts.map((chart) => { + const parentIndicatorId = getParentVariableIdFromChartConfig( + parseChartConfig(chart.config) + ) + const hasParentIndicator = parentIndicatorId !== undefined + return omit({ ...chart, hasParentIndicator }, "config") + }) + + await assignTagsForCharts(trx, charts) + + const variableWithConfigs = await getGrapherConfigsForVariable( + trx, + variableId + ) + const grapherConfigETL = variableWithConfigs?.etl?.patchConfig + const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig + const mergedGrapherConfig = + variableWithConfigs?.admin?.fullConfig ?? + variableWithConfigs?.etl?.fullConfig + + // add the variable's display field to the merged grapher config + if (mergedGrapherConfig) { + const [varDims, otherDims] = lodash.partition( + mergedGrapherConfig.dimensions ?? [], + (dim) => dim.variableId === variableId + ) + const varDimsWithDisplay = varDims.map((dim) => ({ + display: variable.display, + ...dim, + })) + mergedGrapherConfig.dimensions = [ + ...varDimsWithDisplay, + ...otherDims, + ] + } + + const variableWithCharts: OwidVariableWithSource & { + charts: Record<string, any> + grapherConfig: GrapherInterface | undefined + grapherConfigETL: GrapherInterface | undefined + grapherConfigAdmin: GrapherInterface | undefined + } = { + ...variable, + charts, + grapherConfig: mergedGrapherConfig, + grapherConfigETL, + grapherConfigAdmin, + } + + return { + variable: variableWithCharts, + } /*, vardata: await getVariableData([variableId]) }*/ + } +) + +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), + } + } + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigETLOfVariable(trx, variable, validConfig) + + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) + } + + return { success: true, savedPatch } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + // no-op if the variable doesn't have an ETL config + if (!variable.etl) return { success: true } + + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql + UPDATE variables + SET grapherConfigIdETL = NULL + WHERE id = ? + `, + [variableId] + ) + + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql + DELETE FROM chart_configs + WHERE id = ? + `, + [variable.etl.configId] + ) + + // update admin config if there is one + if (variable.admin) { + await updateExistingFullConfig(trx, { + configId: variable.admin.configId, + config: variable.admin.patchConfig, + updatedAt: now, + }) + } + + const updates = { + patchConfigAdmin: variable.admin?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( + trx, + variableId, + updates + ) + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) + } + + return { success: true } + } +) + +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), + } + } + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) + + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) + } + + return { success: true, savedPatch } + } +) + +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } + + // no-op if the variable doesn't have an admin-authored config + if (!variable.admin) return { success: true } + + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql + UPDATE variables + SET grapherConfigIdAdmin = NULL + WHERE id = ? + `, + [variableId] + ) + + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql + DELETE FROM chart_configs + WHERE id = ? + `, + [variable.admin.configId] + ) + + const updates = { + patchConfigETL: variable.etl?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( + trx, + variableId, + updates + ) + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) + } + + return { success: true } + } +) + +getRouteWithROTransaction( + apiRouter, + "/variables/:variableId/charts.json", + async (req, res, trx) => { + const variableId = expectInt(req.params.variableId) + const charts = await getAllChartsForIndicator(trx, variableId) + return charts.map((chart) => ({ + id: chart.chartId, + title: chart.config.title, + variantName: chart.config.variantName, + isChild: chart.isChild, + isInheritanceEnabled: chart.isInheritanceEnabled, + isPublished: chart.isPublished, + })) + } +) diff --git a/adminSiteServer/getLogsByChartId.ts b/adminSiteServer/getLogsByChartId.ts new file mode 100644 index 0000000000..bbffc94380 --- /dev/null +++ b/adminSiteServer/getLogsByChartId.ts @@ -0,0 +1,34 @@ +import { Json } from "@ourworldindata/utils" +import * as db from "../db/db.js" + +export async function getLogsByChartId( + knex: db.KnexReadonlyTransaction, + chartId: number +): Promise< + { + userId: number + config: Json + userName: string + createdAt: Date + }[] +> { + const logs = await db.knexRaw<{ + userId: number + config: string + userName: string + createdAt: Date + }>( + knex, + `SELECT userId, config, fullName as userName, l.createdAt + FROM chart_revisions l + LEFT JOIN users u on u.id = userId + WHERE chartId = ? + ORDER BY l.id DESC + LIMIT 50`, + [chartId] + ) + return logs.map((log) => ({ + ...log, + config: JSON.parse(log.config), + })) +} From 51f6dfa5a18108afb05bc592fb9c46f1b31f2ff4 Mon Sep 17 00:00:00 2001 From: Daniel Bachler <daniel@danielbachler.de> Date: Fri, 20 Dec 2024 15:33:00 +0100 Subject: [PATCH 37/39] =?UTF-8?q?=F0=9F=94=A8=20refactor=20request=20handl?= =?UTF-8?q?er=20lambdas=20to=20named=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRoutes/bulkUpdates.ts | 370 +++++----- adminSiteServer/apiRoutes/chartViews.ts | 238 +++---- adminSiteServer/apiRoutes/charts.ts | 399 ++++++----- adminSiteServer/apiRoutes/datasets.ts | 720 ++++++++++---------- adminSiteServer/apiRoutes/explorer.ts | 55 +- adminSiteServer/apiRoutes/gdocs.ts | 110 +-- adminSiteServer/apiRoutes/images.ts | 91 ++- adminSiteServer/apiRoutes/mdims.ts | 45 +- adminSiteServer/apiRoutes/misc.ts | 110 +-- adminSiteServer/apiRoutes/posts.ts | 250 +++---- adminSiteServer/apiRoutes/redirects.ts | 239 ++++--- adminSiteServer/apiRoutes/suggest.ts | 104 +-- adminSiteServer/apiRoutes/tagGraph.ts | 35 +- adminSiteServer/apiRoutes/tags.ts | 343 +++++----- adminSiteServer/apiRoutes/users.ts | 214 +++--- adminSiteServer/apiRoutes/variables.ts | 826 +++++++++++++---------- 16 files changed, 2267 insertions(+), 1882 deletions(-) diff --git a/adminSiteServer/apiRoutes/bulkUpdates.ts b/adminSiteServer/apiRoutes/bulkUpdates.ts index 4dbb3cc902..364146238c 100644 --- a/adminSiteServer/apiRoutes/bulkUpdates.ts +++ b/adminSiteServer/apiRoutes/bulkUpdates.ts @@ -30,35 +30,34 @@ import { saveGrapher } from "./charts.js" import * as db from "../../db/db.js" import * as lodash from "lodash" import { apiRouter } from "../apiRouter.js" +import { Request } from "../authentication.js" +import e from "express" -getRouteWithROTransaction( - apiRouter, - "/chart-bulk-update", - async ( - req, - res, - trx - ): Promise<BulkGrapherConfigResponse<BulkChartEditResponseRow>> => { - const context: OperationContext = { - grapherConfigFieldName: "chart_configs.full", - whitelistedColumnNamesAndTypes: - chartBulkUpdateAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined +export async function getChartBulkUpdate( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +): Promise<BulkGrapherConfigResponse<BulkChartEditResponseRow>> { + const context: OperationContext = { + grapherConfigFieldName: "chart_configs.full", + whitelistedColumnNamesAndTypes: + chartBulkUpdateAllowedColumnNamesAndTypes, + } + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql SELECT charts.id as id, chart_configs.full as config, @@ -77,180 +76,191 @@ getRouteWithROTransaction( LIMIT 50 OFFSET ${offset.toString()} ` - ) + ) - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql SELECT count(*) as count FROM charts JOIN chart_configs ON chart_configs.id = charts.configId WHERE ${whereClause} ` - ) - return { rows: results, numTotalRows: resultCount[0].count } - } -) + ) + return { rows: results, numTotalRows: resultCount[0].count } +} -patchRouteWithRWTransaction( - apiRouter, - "/chart-bulk-update", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const chartIds = new Set(patchesList.map((patch) => patch.id)) +export async function updateBulkChartConfigs( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const patchesList = req.body as GrapherConfigPatch[] + const chartIds = new Set(patchesList.map((patch) => patch.id)) - const configsAndIds = await db.knexRaw< - Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] } - >( - trx, - `-- sql - SELECT c.id, cc.full as config - FROM charts c - JOIN chart_configs cc ON cc.id = c.configId - WHERE c.id IN (?) - `, - [[...chartIds.values()]] - ) - const configMap = new Map<number, GrapherInterface>( - configsAndIds.map((item: any) => [ - item.id, - // make sure that the id is set, otherwise the update behaviour is weird - // TODO: discuss if this has unintended side effects - item.config ? { ...JSON.parse(item.config), id: item.id } : {}, - ]) - ) - const oldValuesConfigMap = new Map(configMap) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } + const configsAndIds = await db.knexRaw< + Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] } + >( + trx, + `-- sql + SELECT c.id, cc.full as config + FROM charts c + JOIN chart_configs cc ON cc.id = c.configId + WHERE c.id IN (?) + `, + [[...chartIds.values()]] + ) + const configMap = new Map<number, GrapherInterface>( + configsAndIds.map((item: any) => [ + item.id, + // make sure that the id is set, otherwise the update behaviour is weird + // TODO: discuss if this has unintended side effects + item.config ? { ...JSON.parse(item.config), id: item.id } : {}, + ]) + ) + const oldValuesConfigMap = new Map(configMap) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) + } - for (const [id, newConfig] of configMap.entries()) { - await saveGrapher(trx, { - user: res.locals.user, - newConfig, - existingConfig: oldValuesConfigMap.get(id), - referencedVariablesMightChange: false, - }) - } + for (const [id, newConfig] of configMap.entries()) { + await saveGrapher(trx, { + user: res.locals.user, + newConfig, + existingConfig: oldValuesConfigMap.get(id), + referencedVariablesMightChange: false, + }) + } + + return { success: true } +} - return { success: true } +export async function getVariableAnnotations( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +): Promise<BulkGrapherConfigResponse<VariableAnnotationsResponseRow>> { + const context: OperationContext = { + grapherConfigFieldName: "grapherConfigAdmin", + whitelistedColumnNamesAndTypes: + variableAnnotationAllowedColumnNamesAndTypes, } -) + const filterSExpr = + req.query.filter !== undefined + ? parseToOperation(req.query.filter as string, context) + : undefined -getRouteWithROTransaction( - apiRouter, - "/variable-annotations", - async ( - req, - res, - trx - ): Promise<BulkGrapherConfigResponse<VariableAnnotationsResponseRow>> => { - const context: OperationContext = { - grapherConfigFieldName: "grapherConfigAdmin", - whitelistedColumnNamesAndTypes: - variableAnnotationAllowedColumnNamesAndTypes, - } - const filterSExpr = - req.query.filter !== undefined - ? parseToOperation(req.query.filter as string, context) - : undefined + const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 - const offset = parseIntOrUndefined(req.query.offset as string) ?? 0 + // Note that our DSL generates sql here that we splice directly into the SQL as text + // This is a potential for a SQL injection attack but we control the DSL and are + // careful there to only allow carefully guarded vocabularies from being used, not + // arbitrary user input + const whereClause = filterSExpr?.toSql() ?? "true" + const resultsWithStringGrapherConfigs = await db.knexRaw( + trx, + `-- sql + SELECT + variables.id as id, + variables.name as name, + chart_configs.patch as config, + d.name as datasetname, + namespaces.name as namespacename, + variables.createdAt as createdAt, + variables.updatedAt as updatedAt, + variables.description as description + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ORDER BY variables.id DESC + LIMIT 50 + OFFSET ${offset.toString()} + ` + ) - // Note that our DSL generates sql here that we splice directly into the SQL as text - // This is a potential for a SQL injection attack but we control the DSL and are - // careful there to only allow carefully guarded vocabularies from being used, not - // arbitrary user input - const whereClause = filterSExpr?.toSql() ?? "true" - const resultsWithStringGrapherConfigs = await db.knexRaw( - trx, - `-- sql - SELECT - variables.id as id, - variables.name as name, - chart_configs.patch as config, - d.name as datasetname, - namespaces.name as namespacename, - variables.createdAt as createdAt, - variables.updatedAt as updatedAt, - variables.description as description - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ORDER BY variables.id DESC - LIMIT 50 - OFFSET ${offset.toString()} - ` - ) + const results = resultsWithStringGrapherConfigs.map((row: any) => ({ + ...row, + config: lodash.isNil(row.config) ? null : JSON.parse(row.config), + })) + const resultCount = await db.knexRaw<{ count: number }>( + trx, + `-- sql + SELECT count(*) as count + FROM variables + LEFT JOIN active_datasets as d on variables.datasetId = d.id + LEFT JOIN namespaces on d.namespace = namespaces.name + LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id + WHERE ${whereClause} + ` + ) + return { rows: results, numTotalRows: resultCount[0].count } +} - const results = resultsWithStringGrapherConfigs.map((row: any) => ({ - ...row, - config: lodash.isNil(row.config) ? null : JSON.parse(row.config), - })) - const resultCount = await db.knexRaw<{ count: number }>( - trx, - `-- sql - SELECT count(*) as count - FROM variables - LEFT JOIN active_datasets as d on variables.datasetId = d.id - LEFT JOIN namespaces on d.namespace = namespaces.name - LEFT JOIN chart_configs on variables.grapherConfigIdAdmin = chart_configs.id - WHERE ${whereClause} - ` - ) - return { rows: results, numTotalRows: resultCount[0].count } +export async function updateVariableAnnotations( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const patchesList = req.body as GrapherConfigPatch[] + const variableIds = new Set(patchesList.map((patch) => patch.id)) + + const configsAndIds = await db.knexRaw< + Pick<DbRawVariable, "id"> & { + grapherConfigAdmin: DbRawChartConfig["patch"] + } + >( + trx, + `-- sql + SELECT v.id, cc.patch AS grapherConfigAdmin + FROM variables v + LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id + WHERE v.id IN (?)`, + [[...variableIds.values()]] + ) + const configMap = new Map( + configsAndIds.map((item: any) => [ + item.id, + item.grapherConfigAdmin ? JSON.parse(item.grapherConfigAdmin) : {}, + ]) + ) + // console.log("ids", configsAndIds.map((item : any) => item.id)) + for (const patchSet of patchesList) { + const config = configMap.get(patchSet.id) + configMap.set(patchSet.id, applyPatch(patchSet, config)) } -) + + for (const [variableId, newConfig] of configMap.entries()) { + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) continue + await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) + } + + return { success: true } +} patchRouteWithRWTransaction( apiRouter, "/variable-annotations", - async (req, res, trx) => { - const patchesList = req.body as GrapherConfigPatch[] - const variableIds = new Set(patchesList.map((patch) => patch.id)) - - const configsAndIds = await db.knexRaw< - Pick<DbRawVariable, "id"> & { - grapherConfigAdmin: DbRawChartConfig["patch"] - } - >( - trx, - `-- sql - SELECT v.id, cc.patch AS grapherConfigAdmin - FROM variables v - LEFT JOIN chart_configs cc ON v.grapherConfigIdAdmin = cc.id - WHERE v.id IN (?) - `, - [[...variableIds.values()]] - ) - const configMap = new Map( - configsAndIds.map((item: any) => [ - item.id, - item.grapherConfigAdmin - ? JSON.parse(item.grapherConfigAdmin) - : {}, - ]) - ) - // console.log("ids", configsAndIds.map((item : any) => item.id)) - for (const patchSet of patchesList) { - const config = configMap.get(patchSet.id) - configMap.set(patchSet.id, applyPatch(patchSet, config)) - } + updateVariableAnnotations +) - for (const [variableId, newConfig] of configMap.entries()) { - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) continue - await updateGrapherConfigAdminOfVariable(trx, variable, newConfig) - } +getRouteWithROTransaction(apiRouter, "/chart-bulk-update", getChartBulkUpdate) - return { success: true } - } +patchRouteWithRWTransaction( + apiRouter, + "/chart-bulk-update", + updateBulkChartConfigs +) +getRouteWithROTransaction( + apiRouter, + "/variable-annotations", + getVariableAnnotations ) diff --git a/adminSiteServer/apiRoutes/chartViews.ts b/adminSiteServer/apiRoutes/chartViews.ts index c0013b57ef..4eda8ff3aa 100644 --- a/adminSiteServer/apiRoutes/chartViews.ts +++ b/adminSiteServer/apiRoutes/chartViews.ts @@ -34,6 +34,8 @@ import { import * as db from "../../db/db.js" import { expectChartById } from "./charts.js" +import { Request } from "../authentication.js" +import e from "express" const createPatchConfigAndQueryParamsForChartView = async ( knex: db.KnexReadonlyTransaction, parentChartId: number, @@ -65,7 +67,11 @@ const createPatchConfigAndQueryParamsForChartView = async ( return { patchConfig: patchConfigToSave, fullConfig, queryParams } } -getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { +export async function getChartViews( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { type ChartViewRow = Pick<DbPlainChartView, "id" | "name" | "updatedAt"> & { lastEditedByUser: string chartConfigId: string @@ -109,30 +115,28 @@ getRouteWithROTransaction(apiRouter, "/chartViews", async (req, res, trx) => { })) return { chartViews } -}) - -getRouteWithROTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - type ChartViewRow = Pick< - DbPlainChartView, - "id" | "name" | "updatedAt" - > & { - lastEditedByUser: string - chartConfigId: string - configFull: JsonString - configPatch: JsonString - parentChartId: number - parentConfigFull: JsonString - queryParamsForParentChart: JsonString - } - - const row = await db.knexRawFirst<ChartViewRow>( - trx, - `-- sql +} + +export async function getChartViewById( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const id = expectInt(req.params.id) + + type ChartViewRow = Pick<DbPlainChartView, "id" | "name" | "updatedAt"> & { + lastEditedByUser: string + chartConfigId: string + configFull: JsonString + configPatch: JsonString + parentChartId: number + parentConfigFull: JsonString + queryParamsForParentChart: JsonString + } + + const row = await db.knexRawFirst<ChartViewRow>( + trx, + `-- sql SELECT cv.id, cv.name, @@ -151,28 +155,29 @@ getRouteWithROTransaction( JOIN users u ON cv.lastEditedByUserId = u.id WHERE cv.id = ? `, - [id] - ) + [id] + ) - if (!row) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const chartView = { - ...row, - configFull: parseChartConfig(row.configFull), - configPatch: parseChartConfig(row.configPatch), - parentConfigFull: parseChartConfig(row.parentConfigFull), - queryParamsForParentChart: JSON.parse( - row.queryParamsForParentChart - ), - } - - return chartView + if (!row) { + throw new JsonError(`No chart view found for id ${id}`, 404) } -) -postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { + const chartView = { + ...row, + configFull: parseChartConfig(row.configFull), + configPatch: parseChartConfig(row.configPatch), + parentConfigFull: parseChartConfig(row.parentConfigFull), + queryParamsForParentChart: JSON.parse(row.queryParamsForParentChart), + } + + return chartView +} + +export async function createChartView( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { const { name, parentChartId } = req.body as Pick< DbPlainChartView, "name" | "parentChartId" @@ -195,7 +200,6 @@ postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { patchConfig, fullConfig ) - // insert into chart_views const insertRow: DbInsertChartView = { name, @@ -208,83 +212,89 @@ postRouteWithRWTransaction(apiRouter, "/chartViews", async (req, res, trx) => { const [resultId] = result return { chartViewId: resultId, success: true } -}) - -putRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const rawConfig = req.body.config as GrapherInterface - if (!rawConfig) { - throw new JsonError("Invalid request", 400) - } - - const existingRow: Pick< - DbPlainChartView, - "chartConfigId" | "parentChartId" - > = await trx(ChartViewsTableName) - .select("parentChartId", "chartConfigId") - .where({ id }) - .first() - - if (!existingRow) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } - - const { patchConfig, fullConfig, queryParams } = - await createPatchConfigAndQueryParamsForChartView( - trx, - existingRow.parentChartId, - rawConfig - ) - - await updateChartConfigInDbAndR2( +} + +export async function updateChartView( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + const rawConfig = req.body.config as GrapherInterface + if (!rawConfig) { + throw new JsonError("Invalid request", 400) + } + + const existingRow: Pick< + DbPlainChartView, + "chartConfigId" | "parentChartId" + > = await trx(ChartViewsTableName) + .select("parentChartId", "chartConfigId") + .where({ id }) + .first() + + if (!existingRow) { + throw new JsonError(`No chart view found for id ${id}`, 404) + } + + const { patchConfig, fullConfig, queryParams } = + await createPatchConfigAndQueryParamsForChartView( trx, - existingRow.chartConfigId as Base64String, - patchConfig, - fullConfig + existingRow.parentChartId, + rawConfig ) - // update chart_views - await trx - .table(ChartViewsTableName) - .where({ id }) - .update({ - updatedAt: new Date(), - lastEditedByUserId: res.locals.user.id, - queryParamsForParentChart: JSON.stringify(queryParams), - }) - - return { success: true } + await updateChartConfigInDbAndR2( + trx, + existingRow.chartConfigId as Base64String, + patchConfig, + fullConfig + ) + + await trx + .table(ChartViewsTableName) + .where({ id }) + .update({ + updatedAt: new Date(), + lastEditedByUserId: res.locals.user.id, + queryParamsForParentChart: JSON.stringify(queryParams), + }) + + return { success: true } +} + +export async function deleteChartView( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + + const chartConfigId: string | undefined = await trx(ChartViewsTableName) + .select("chartConfigId") + .where({ id }) + .first() + .then((row) => row?.chartConfigId) + + if (!chartConfigId) { + throw new JsonError(`No chart view found for id ${id}`, 404) } -) -deleteRouteWithRWTransaction( - apiRouter, - "/chartViews/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) + await trx.table(ChartViewsTableName).where({ id }).delete() - const chartConfigId: string | undefined = await trx(ChartViewsTableName) - .select("chartConfigId") - .where({ id }) - .first() - .then((row) => row?.chartConfigId) + await deleteGrapherConfigFromR2ByUUID(chartConfigId) - if (!chartConfigId) { - throw new JsonError(`No chart view found for id ${id}`, 404) - } + await trx.table(ChartConfigsTableName).where({ id: chartConfigId }).delete() - await trx.table(ChartViewsTableName).where({ id }).delete() + return { success: true } +} - await deleteGrapherConfigFromR2ByUUID(chartConfigId) +getRouteWithROTransaction(apiRouter, "/chartViews", getChartViews) - await trx - .table(ChartConfigsTableName) - .where({ id: chartConfigId }) - .delete() +getRouteWithROTransaction(apiRouter, "/chartViews/:id", getChartViewById) - return { success: true } - } -) +postRouteWithRWTransaction(apiRouter, "/chartViews", createChartView) + +putRouteWithRWTransaction(apiRouter, "/chartViews/:id", updateChartView) + +deleteRouteWithRWTransaction(apiRouter, "/chartViews/:id", deleteChartView) diff --git a/adminSiteServer/apiRoutes/charts.ts b/adminSiteServer/apiRoutes/charts.ts index ae295c11fe..0ab2670cdd 100644 --- a/adminSiteServer/apiRoutes/charts.ts +++ b/adminSiteServer/apiRoutes/charts.ts @@ -66,6 +66,8 @@ import * as db from "../../db/db.js" import { getLogsByChartId } from "../getLogsByChartId.js" import { getPublishedLinksTo } from "../../db/model/Link.js" +import { Request } from "../authentication.js" +import e from "express" export const getReferencesByChartId = async ( chartId: number, knex: db.KnexReadonlyTransaction @@ -501,7 +503,11 @@ export async function updateGrapherConfigsInR2( } } -getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { +async function getChartsJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 const charts = await db.knexRaw<OldChartFieldList>( trx, @@ -521,9 +527,13 @@ getRouteWithROTransaction(apiRouter, "/charts.json", async (req, res, trx) => { await assignTagsForCharts(trx, charts) return { charts } -}) +} -getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { +async function getChartsCsv( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const limit = parseIntOrUndefined(req.query.limit as string) ?? 10000 // note: this query is extended from OldChart.listFields. @@ -577,106 +587,116 @@ getRouteWithROTransaction(apiRouter, "/charts.csv", async (req, res, trx) => { res.setHeader("content-type", "text/csv") const csv = Papa.unparse(charts) return csv -}) +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.config.json", - async (req, res, trx) => expectChartById(trx, req.params.chartId) -) +async function getChartConfigJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + return expectChartById(trx, req.params.chartId) +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.parent.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const parent = await getParentByChartId(trx, chartId) - const isInheritanceEnabled = await isInheritanceEnabledForChart( - trx, - chartId - ) - return omitUndefinedValues({ - variableId: parent?.variableId, - config: parent?.config, - isActive: isInheritanceEnabled, - }) - } -) +async function getChartParentJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const chartId = expectInt(req.params.chartId) + const parent = await getParentByChartId(trx, chartId) + const isInheritanceEnabled = await isInheritanceEnabledForChart( + trx, + chartId + ) + return omitUndefinedValues({ + variableId: parent?.variableId, + config: parent?.config, + isActive: isInheritanceEnabled, + }) +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.patchConfig.json", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const config = await expectPatchConfigByChartId(trx, chartId) - return config - } -) +async function getChartPatchConfigJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const chartId = expectInt(req.params.chartId) + const config = await expectPatchConfigByChartId(trx, chartId) + return config +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.logs.json", - async (req, res, trx) => ({ +async function getChartLogsJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + return { logs: await getLogsByChartId( trx, parseInt(req.params.chartId as string) ), - }) -) + } +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.references.json", - async (req, res, trx) => { - const references = { - references: await getReferencesByChartId( - parseInt(req.params.chartId as string), - trx - ), - } - return references +async function getChartReferencesJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const references = { + references: await getReferencesByChartId( + parseInt(req.params.chartId as string), + trx + ), } -) + return references +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.redirects.json", - async (req, res, trx) => ({ +async function getChartRedirectsJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await getRedirectsByChartId( trx, parseInt(req.params.chartId as string) ), - }) -) + } +} -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.pageviews.json", - async (req, res, trx) => { - const slug = await getChartSlugById( - trx, - parseInt(req.params.chartId as string) - ) - if (!slug) return {} +async function getChartPageviewsJson( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const slug = await getChartSlugById( + trx, + parseInt(req.params.chartId as string) + ) + if (!slug) return {} - const pageviewsByUrl = await db.knexRawFirst( - trx, - `-- sql - SELECT * - FROM - analytics_pageviews - WHERE - url = ?`, - [`https://ourworldindata.org/grapher/${slug}`] - ) + const pageviewsByUrl = await db.knexRawFirst( + trx, + `-- sql + SELECT * + FROM + analytics_pageviews + WHERE + url = ?`, + [`https://ourworldindata.org/grapher/${slug}`] + ) - return { - pageviews: pageviewsByUrl ?? undefined, - } + return { + pageviews: pageviewsByUrl ?? undefined, } -) +} -postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { +async function createChart( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { let shouldInherit: boolean | undefined if (req.query.inheritance) { shouldInherit = req.query.inheritance === "enable" @@ -693,109 +713,150 @@ postRouteWithRWTransaction(apiRouter, "/charts", async (req, res, trx) => { } catch (err) { return { success: false, error: String(err) } } -}) +} -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/setTags", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) +async function setChartTagsHandler( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const chartId = expectInt(req.params.chartId) + + await setChartTags(trx, chartId, req.body.tags) - await setChartTags(trx, chartId, req.body.tags) + return { success: true } +} - return { success: true } +async function updateChart( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + let shouldInherit: boolean | undefined + if (req.query.inheritance) { + shouldInherit = req.query.inheritance === "enable" } -) -putRouteWithRWTransaction( - apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - let shouldInherit: boolean | undefined - if (req.query.inheritance) { - shouldInherit = req.query.inheritance === "enable" - } + const existingConfig = await expectChartById(trx, req.params.chartId) - const existingConfig = await expectChartById(trx, req.params.chartId) + try { + const { chartId, savedPatch } = await saveGrapher(trx, { + user: res.locals.user, + newConfig: req.body, + existingConfig, + shouldInherit, + }) - try { - const { chartId, savedPatch } = await saveGrapher(trx, { - user: res.locals.user, - newConfig: req.body, - existingConfig, - shouldInherit, - }) + const logs = await getLogsByChartId(trx, existingConfig.id as number) + return { + success: true, + chartId, + savedPatch, + newLog: logs[0], + } + } catch (err) { + return { + success: false, + error: String(err), + } + } +} - const logs = await getLogsByChartId( - trx, - existingConfig.id as number +async function deleteChart( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const chart = await expectChartById(trx, req.params.chartId) + if (chart.slug) { + const links = await getPublishedLinksTo(trx, [chart.slug]) + if (links.length) { + const sources = links.map((link) => link.sourceSlug).join(", ") + throw new Error( + `Cannot delete chart in-use in the following published documents: ${sources}` ) - return { - success: true, - chartId, - savedPatch, - newLog: logs[0], - } - } catch (err) { - return { - success: false, - error: String(err), - } } } -) -deleteRouteWithRWTransaction( - apiRouter, - "/charts/:chartId", - async (req, res, trx) => { - const chart = await expectChartById(trx, req.params.chartId) - if (chart.slug) { - const links = await getPublishedLinksTo(trx, [chart.slug]) - if (links.length) { - const sources = links.map((link) => link.sourceSlug).join(", ") - throw new Error( - `Cannot delete chart in-use in the following published documents: ${sources}` - ) - } - } + await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ + chart.id, + ]) + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE chart_id=?`, [ + chart.id, + ]) - await db.knexRaw(trx, `DELETE FROM chart_dimensions WHERE chartId=?`, [ - chart.id, + const row = await db.knexRawFirst<Pick<DbPlainChart, "configId">>( + trx, + `SELECT configId FROM charts WHERE id = ?`, + [chart.id] + ) + if (!row || !row.configId) + throw new JsonError(`No chart config found for id ${chart.id}`, 404) + if (row) { + await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) + await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ + row.configId, ]) - await db.knexRaw( - trx, - `DELETE FROM chart_slug_redirects WHERE chart_id=?`, - [chart.id] - ) + } - const row = await db.knexRawFirst<Pick<DbPlainChart, "configId">>( - trx, - `SELECT configId FROM charts WHERE id = ?`, - [chart.id] + if (chart.isPublished) + await triggerStaticBuild( + res.locals.user, + `Deleting chart ${chart.slug}` ) - if (!row || !row.configId) - throw new JsonError(`No chart config found for id ${chart.id}`, 404) - if (row) { - await db.knexRaw(trx, `DELETE FROM charts WHERE id=?`, [chart.id]) - await db.knexRaw(trx, `DELETE FROM chart_configs WHERE id=?`, [ - row.configId, - ]) - } - if (chart.isPublished) - await triggerStaticBuild( - res.locals.user, - `Deleting chart ${chart.slug}` - ) + await deleteGrapherConfigFromR2ByUUID(row.configId) + if (chart.isPublished) + await deleteGrapherConfigFromR2( + R2GrapherConfigDirectory.publishedGrapherBySlug, + `${chart.slug}.json` + ) - await deleteGrapherConfigFromR2ByUUID(row.configId) - if (chart.isPublished) - await deleteGrapherConfigFromR2( - R2GrapherConfigDirectory.publishedGrapherBySlug, - `${chart.slug}.json` - ) + return { success: true } +} - return { success: true } - } +getRouteWithROTransaction(apiRouter, "/charts.json", getChartsJson) +getRouteWithROTransaction(apiRouter, "/charts.csv", getChartsCsv) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.config.json", + getChartConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.parent.json", + getChartParentJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.patchConfig.json", + getChartPatchConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.logs.json", + getChartLogsJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.references.json", + getChartReferencesJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.redirects.json", + getChartRedirectsJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.pageviews.json", + getChartPageviewsJson +) +postRouteWithRWTransaction(apiRouter, "/charts", createChart) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/setTags", + setChartTagsHandler ) +putRouteWithRWTransaction(apiRouter, "/charts/:chartId", updateChart) +deleteRouteWithRWTransaction(apiRouter, "/charts/:chartId", deleteChart) diff --git a/adminSiteServer/apiRoutes/datasets.ts b/adminSiteServer/apiRoutes/datasets.ts index 365f00be51..d6bac477a2 100644 --- a/adminSiteServer/apiRoutes/datasets.ts +++ b/adminSiteServer/apiRoutes/datasets.ts @@ -28,390 +28,404 @@ import { import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" - -getRouteWithROTransaction( - apiRouter, - "/datasets.json", - async (req, res, trx) => { - const datasets = await db.knexRaw<Record<string, any>>( - trx, - `-- sql - WITH variable_counts AS ( - SELECT - v.datasetId, - COUNT(DISTINCT cd.chartId) as numCharts - FROM chart_dimensions cd - JOIN variables v ON cd.variableId = v.id - GROUP BY v.datasetId - ) +import { Request } from "express" +import * as e from "express" + +export async function getDatasets( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const datasets = await db.knexRaw<Record<string, any>>( + trx, + `-- sql + WITH variable_counts AS ( SELECT - ad.id, - ad.namespace, - ad.name, - d.shortName, - ad.description, - ad.dataEditedAt, - du.fullName AS dataEditedByUserName, - ad.metadataEditedAt, - mu.fullName AS metadataEditedByUserName, - ad.isPrivate, - ad.nonRedistributable, - d.version, - vc.numCharts - FROM active_datasets ad - LEFT JOIN variable_counts vc ON ad.id = vc.datasetId - JOIN users du ON du.id=ad.dataEditedByUserId - JOIN users mu ON mu.id=ad.metadataEditedByUserId - JOIN datasets d ON d.id=ad.id - ORDER BY ad.dataEditedAt DESC + v.datasetId, + COUNT(DISTINCT cd.chartId) as numCharts + FROM chart_dimensions cd + JOIN variables v ON cd.variableId = v.id + GROUP BY v.datasetId + ) + SELECT + ad.id, + ad.namespace, + ad.name, + d.shortName, + ad.description, + ad.dataEditedAt, + du.fullName AS dataEditedByUserName, + ad.metadataEditedAt, + mu.fullName AS metadataEditedByUserName, + ad.isPrivate, + ad.nonRedistributable, + d.version, + vc.numCharts + FROM active_datasets ad + LEFT JOIN variable_counts vc ON ad.id = vc.datasetId + JOIN users du ON du.id=ad.dataEditedByUserId + JOIN users mu ON mu.id=ad.metadataEditedByUserId + JOIN datasets d ON d.id=ad.id + ORDER BY ad.dataEditedAt DESC ` - ) - - const tags = await db.knexRaw< - Pick<DbPlainTag, "id" | "name"> & - Pick<DbPlainDatasetTag, "datasetId"> - >( - trx, - `-- sql - SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt - JOIN tags t ON dt.tagId = t.id + ) + + const tags = await db.knexRaw< + Pick<DbPlainTag, "id" | "name"> & Pick<DbPlainDatasetTag, "datasetId"> + >( + trx, + `-- sql + SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt + JOIN tags t ON dt.tagId = t.id ` + ) + const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) + for (const dataset of datasets) { + dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => + lodash.omit(t, "datasetId") ) - const tagsByDatasetId = lodash.groupBy(tags, (t) => t.datasetId) - for (const dataset of datasets) { - dataset.tags = (tagsByDatasetId[dataset.id] || []).map((t) => - lodash.omit(t, "datasetId") - ) - } - /*LEFT JOIN variables AS v ON v.datasetId=d.id - GROUP BY d.id*/ - - return { datasets: datasets } } -) - -getRouteWithROTransaction( - apiRouter, - "/datasets/:datasetId.json", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) + /*LEFT JOIN variables AS v ON v.datasetId=d.id + GROUP BY d.id*/ - const dataset = await db.knexRawFirst<Record<string, any>>( - trx, - `-- sql - SELECT d.id, - d.namespace, - d.name, - d.shortName, - d.version, - d.description, - d.updatedAt, - d.dataEditedAt, - d.dataEditedByUserId, - du.fullName AS dataEditedByUserName, - d.metadataEditedAt, - d.metadataEditedByUserId, - mu.fullName AS metadataEditedByUserName, - d.isPrivate, - d.isArchived, - d.nonRedistributable, - d.updatePeriodDays - FROM datasets AS d - JOIN users du ON du.id=d.dataEditedByUserId - JOIN users mu ON mu.id=d.metadataEditedByUserId - WHERE d.id = ? + return { datasets: datasets } +} + +export async function getDataset( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await db.knexRawFirst<Record<string, any>>( + trx, + `-- sql + SELECT d.id, + d.namespace, + d.name, + d.shortName, + d.version, + d.description, + d.updatedAt, + d.dataEditedAt, + d.dataEditedByUserId, + du.fullName AS dataEditedByUserName, + d.metadataEditedAt, + d.metadataEditedByUserId, + mu.fullName AS metadataEditedByUserName, + d.isPrivate, + d.isArchived, + d.nonRedistributable, + d.updatePeriodDays + FROM datasets AS d + JOIN users du ON du.id=d.dataEditedByUserId + JOIN users mu ON mu.id=d.metadataEditedByUserId + WHERE d.id = ? `, - [datasetId] - ) - - if (!dataset) - throw new JsonError(`No dataset by id '${datasetId}'`, 404) - - const zipFile = await db.knexRawFirst<{ filename: string }>( - trx, - `SELECT filename FROM dataset_files WHERE datasetId=?`, - [datasetId] - ) - if (zipFile) dataset.zipFile = zipFile - - const variables = await db.knexRaw< - Pick< - DbRawVariable, - "id" | "name" | "description" | "display" | "catalogPath" - > - >( - trx, - `-- sql - SELECT - v.id, - v.name, - v.description, - v.display, - v.catalogPath - FROM - variables AS v - WHERE - v.datasetId = ? + [datasetId] + ) + + if (!dataset) throw new JsonError(`No dataset by id '${datasetId}'`, 404) + + const zipFile = await db.knexRawFirst<{ filename: string }>( + trx, + `SELECT filename FROM dataset_files WHERE datasetId=?`, + [datasetId] + ) + if (zipFile) dataset.zipFile = zipFile + + const variables = await db.knexRaw< + Pick< + DbRawVariable, + "id" | "name" | "description" | "display" | "catalogPath" + > + >( + trx, + `-- sql + SELECT + v.id, + v.name, + v.description, + v.display, + v.catalogPath + FROM + variables AS v + WHERE + v.datasetId = ? `, - [datasetId] - ) + [datasetId] + ) - for (const v of variables) { - v.display = JSON.parse(v.display) - } - - dataset.variables = variables + for (const v of variables) { + v.display = JSON.parse(v.display) + } - // add all origins - const origins: DbRawOrigin[] = await db.knexRaw<DbRawOrigin>( - trx, - `-- sql - SELECT DISTINCT - o.* - FROM - origins_variables AS ov - JOIN origins AS o ON ov.originId = o.id - JOIN variables AS v ON ov.variableId = v.id - WHERE - v.datasetId = ? + dataset.variables = variables + + // add all origins + const origins: DbRawOrigin[] = await db.knexRaw<DbRawOrigin>( + trx, + `-- sql + SELECT DISTINCT + o.* + FROM + origins_variables AS ov + JOIN origins AS o ON ov.originId = o.id + JOIN variables AS v ON ov.variableId = v.id + WHERE + v.datasetId = ? `, - [datasetId] - ) - - const parsedOrigins = origins.map(parseOriginsRow) - - dataset.origins = parsedOrigins - - const sources = await db.knexRaw<{ - id: number - name: string - description: string - }>( - trx, - ` - SELECT s.id, s.name, s.description - FROM sources AS s - WHERE s.datasetId = ? - ORDER BY s.id ASC + [datasetId] + ) + + const parsedOrigins = origins.map(parseOriginsRow) + + dataset.origins = parsedOrigins + + const sources = await db.knexRaw<{ + id: number + name: string + description: string + }>( + trx, + ` + SELECT s.id, s.name, s.description + FROM sources AS s + WHERE s.datasetId = ? + ORDER BY s.id ASC `, - [datasetId] - ) - - // expand description of sources and add to dataset as variableSources - dataset.variableSources = sources.map((s: any) => { - return { - id: s.id, - name: s.name, - ...JSON.parse(s.description), - } - }) - - const charts = await db.knexRaw<OldChartFieldList>( - trx, - `-- sql - SELECT ${oldChartFieldList} - FROM charts - JOIN chart_configs ON chart_configs.id = charts.configId - JOIN chart_dimensions AS cd ON cd.chartId = charts.id - JOIN variables AS v ON cd.variableId = v.id - JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId - LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId - WHERE v.datasetId = ? - GROUP BY charts.id - `, - [datasetId] - ) - - dataset.charts = charts - - await assignTagsForCharts(trx, charts) - - const tags = await db.knexRaw<{ id: number; name: string }>( - trx, - ` - SELECT t.id, t.name - FROM tags t - JOIN dataset_tags dt ON dt.tagId = t.id - WHERE dt.datasetId = ? + [datasetId] + ) + + // expand description of sources and add to dataset as variableSources + dataset.variableSources = sources.map((s: any) => { + return { + id: s.id, + name: s.name, + ...JSON.parse(s.description), + } + }) + + const charts = await db.knexRaw<OldChartFieldList>( + trx, + `-- sql + SELECT ${oldChartFieldList} + FROM charts + JOIN chart_configs ON chart_configs.id = charts.configId + JOIN chart_dimensions AS cd ON cd.chartId = charts.id + JOIN variables AS v ON cd.variableId = v.id + JOIN users lastEditedByUser ON lastEditedByUser.id = charts.lastEditedByUserId + LEFT JOIN users publishedByUser ON publishedByUser.id = charts.publishedByUserId + WHERE v.datasetId = ? + GROUP BY charts.id + `, + [datasetId] + ) + + dataset.charts = charts + + await assignTagsForCharts(trx, charts) + + const tags = await db.knexRaw<{ id: number; name: string }>( + trx, + ` + SELECT t.id, t.name + FROM tags t + JOIN dataset_tags dt ON dt.tagId = t.id + WHERE dt.datasetId = ? `, - [datasetId] - ) - dataset.tags = tags - - const availableTags = await db.knexRaw<{ - id: number - name: string - parentName: string - }>( - trx, - ` - SELECT t.id, t.name, p.name AS parentName - FROM tags AS t - JOIN tags AS p ON t.parentId=p.id + [datasetId] + ) + dataset.tags = tags + + const availableTags = await db.knexRaw<{ + id: number + name: string + parentName: string + }>( + trx, + ` + SELECT t.id, t.name, p.name AS parentName + FROM tags AS t + JOIN tags AS p ON t.parentId=p.id ` - ) - dataset.availableTags = availableTags - - return { dataset: dataset } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - // Only updates `nonRedistributable` and `tags`, other fields come from ETL - // and are not editable - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - const newDataset = (req.body as { dataset: any }).dataset - await db.knexRaw( - trx, - ` - UPDATE datasets - SET - nonRedistributable=?, - metadataEditedAt=?, - metadataEditedByUserId=? - WHERE id=? - `, - [ - newDataset.nonRedistributable, - new Date(), - res.locals.user.id, - datasetId, - ] - ) - - const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) - await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ + ) + dataset.availableTags = availableTags + + return { dataset: dataset } +} + +export async function updateDataset( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + // Only updates `nonRedistributable` and `tags`, other fields come from ETL + // and are not editable + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + const newDataset = (req.body as { dataset: any }).dataset + await db.knexRaw( + trx, + ` + UPDATE datasets + SET + nonRedistributable=?, + metadataEditedAt=?, + metadataEditedByUserId=? + WHERE id=? + `, + [ + newDataset.nonRedistributable, + new Date(), + _res.locals.user.id, datasetId, - ]) - if (tagRows.length) - for (const tagRow of tagRows) { - await db.knexRaw( - trx, - `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, - tagRow - ) - } - - try { - await syncDatasetToGitRepo(trx, datasetId, { - oldDatasetName: dataset.name, - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue + ] + ) + + const tagRows = newDataset.tags.map((tag: any) => [tag.id, datasetId]) + await db.knexRaw(trx, `DELETE FROM dataset_tags WHERE datasetId=?`, [ + datasetId, + ]) + if (tagRows.length) + for (const tagRow of tagRows) { + await db.knexRaw( + trx, + `INSERT INTO dataset_tags (tagId, datasetId) VALUES (?, ?)`, + tagRow + ) } - return { success: true } + try { + await syncDatasetToGitRepo(trx, datasetId, { + oldDatasetName: dataset.name, + commitName: _res.locals.user.fullName, + commitEmail: _res.locals.user.email, + }) + } catch (err) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setArchived", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ - datasetId, - ]) - return { success: true } + return { success: true } +} + +export async function setArchived( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw(trx, `UPDATE datasets SET isArchived = 1 WHERE id=?`, [ + datasetId, + ]) + return { success: true } +} + +export async function setTags( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + await setTagsForDataset(trx, datasetId, req.body.tagIds) + + return { success: true } +} + +export async function deleteDataset( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) + + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + + await db.knexRaw( + trx, + `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, + [datasetId] + ) + await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ + datasetId, + ]) + await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [datasetId]) + await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) + + try { + await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { + commitName: _res.locals.user.fullName, + commitEmail: _res.locals.user.email, + }) + } catch (err: any) { + await logErrorAndMaybeSendToBugsnag(err, req) + // Continue } -) - -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setTags", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - await setTagsForDataset(trx, datasetId, req.body.tagIds) + return { success: true } +} - return { success: true } - } -) +export async function republishCharts( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const datasetId = expectInt(req.params.datasetId) -deleteRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + const dataset = await getDatasetById(trx, datasetId) + if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) + if (req.body.republish) { await db.knexRaw( trx, - `DELETE d FROM country_latest_data AS d JOIN variables AS v ON d.variable_id=v.id WHERE v.datasetId=?`, + `-- sql + UPDATE chart_configs cc + JOIN charts c ON c.configId = cc.id + SET + cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), + cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) + WHERE c.id IN ( + SELECT DISTINCT chart_dimensions.chartId + FROM chart_dimensions + JOIN variables ON variables.id = chart_dimensions.variableId + WHERE variables.datasetId = ? + )`, [datasetId] ) - await db.knexRaw(trx, `DELETE FROM dataset_files WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM variables WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM sources WHERE datasetId=?`, [ - datasetId, - ]) - await db.knexRaw(trx, `DELETE FROM datasets WHERE id=?`, [datasetId]) - - try { - await removeDatasetFromGitRepo(dataset.name, dataset.namespace, { - commitName: res.locals.user.fullName, - commitEmail: res.locals.user.email, - }) - } catch (err: any) { - await logErrorAndMaybeSendToBugsnag(err, req) - // Continue - } - - return { success: true } } -) + await triggerStaticBuild( + _res.locals.user, + `Republishing all charts in dataset ${dataset.name} (${dataset.id})` + ) + + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/datasets.json", getDatasets) +getRouteWithROTransaction(apiRouter, "/datasets/:datasetId.json", getDataset) +putRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", updateDataset) +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/setArchived", + setArchived +) +postRouteWithRWTransaction(apiRouter, "/datasets/:datasetId/setTags", setTags) +deleteRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", deleteDataset) postRouteWithRWTransaction( apiRouter, "/datasets/:datasetId/charts", - async (req, res, trx) => { - const datasetId = expectInt(req.params.datasetId) - - const dataset = await getDatasetById(trx, datasetId) - if (!dataset) throw new JsonError(`No dataset by id ${datasetId}`, 404) - - if (req.body.republish) { - await db.knexRaw( - trx, - `-- sql - UPDATE chart_configs cc - JOIN charts c ON c.configId = cc.id - SET - cc.patch = JSON_SET(cc.patch, "$.version", cc.patch->"$.version" + 1), - cc.full = JSON_SET(cc.full, "$.version", cc.full->"$.version" + 1) - WHERE c.id IN ( - SELECT DISTINCT chart_dimensions.chartId - FROM chart_dimensions - JOIN variables ON variables.id = chart_dimensions.variableId - WHERE variables.datasetId = ? - )`, - [datasetId] - ) - } - - await triggerStaticBuild( - res.locals.user, - `Republishing all charts in dataset ${dataset.name} (${dataset.id})` - ) - - return { success: true } - } + republishCharts ) diff --git a/adminSiteServer/apiRoutes/explorer.ts b/adminSiteServer/apiRoutes/explorer.ts index eb184e2bef..f0228fafff 100644 --- a/adminSiteServer/apiRoutes/explorer.ts +++ b/adminSiteServer/apiRoutes/explorer.ts @@ -4,34 +4,43 @@ import { postRouteWithRWTransaction, deleteRouteWithRWTransaction, } from "../functionalRouterHelpers.js" +import { Request } from "express" +import * as e from "express" -postRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - async (req, res, trx) => { - const { slug } = req.params - const { tagIds } = req.body - const explorer = await trx.table("explorers").where({ slug }).first() - if (!explorer) - throw new JsonError(`No explorer found for slug ${slug}`, 404) +import * as db from "../../db/db.js" +export async function addExplorerTags( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const { slug } = req.params + const { tagIds } = req.body + const explorer = await trx.table("explorers").where({ slug }).first() + if (!explorer) + throw new JsonError(`No explorer found for slug ${slug}`, 404) - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - for (const tagId of tagIds) { - await trx - .table("explorer_tags") - .insert({ explorerSlug: slug, tagId }) - } - - return { success: true } + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + for (const tagId of tagIds) { + await trx.table("explorer_tags").insert({ explorerSlug: slug, tagId }) } -) + + return { success: true } +} + +export async function deleteExplorerTags( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const { slug } = req.params + await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() + return { success: true } +} + +postRouteWithRWTransaction(apiRouter, "/explorer/:slug/tags", addExplorerTags) deleteRouteWithRWTransaction( apiRouter, "/explorer/:slug/tags", - async (req, res, trx) => { - const { slug } = req.params - await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() - return { success: true } - } + deleteExplorerTags ) diff --git a/adminSiteServer/apiRoutes/gdocs.ts b/adminSiteServer/apiRoutes/gdocs.ts index 0bd40226c5..ed96cb2417 100644 --- a/adminSiteServer/apiRoutes/gdocs.ts +++ b/adminSiteServer/apiRoutes/gdocs.ts @@ -53,38 +53,44 @@ import { import { triggerStaticBuild, enqueueLightningChange } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" +import { Request } from "../authentication.js" +import e from "express" -getRouteWithROTransaction(apiRouter, "/gdocs", (req, res, trx) => { +export async function getAllGdocIndexItems( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { return getAllGdocIndexItemsOrderedByUpdatedAt(trx) -}) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/gdocs/:id", - async (req, res, trx) => { - const id = req.params.id - const contentSource = req.query.contentSource as - | GdocsContentSource - | undefined +} - try { - // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published - const gdoc = await getAndLoadGdocById(trx, id, contentSource) +export async function getIndividualGdoc( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const id = req.params.id + const contentSource = req.query.contentSource as + | GdocsContentSource + | undefined - if (!gdoc.published) { - await updateGdocContentOnly(trx, id, gdoc) - } + try { + // Beware: if contentSource=gdocs this will update images in the DB+S3 even if the gdoc is published + const gdoc = await getAndLoadGdocById(trx, id, contentSource) - res.set("Cache-Control", "no-store") - res.send(gdoc) - } catch (error) { - console.error("Error fetching gdoc", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) + if (!gdoc.published) { + await updateGdocContentOnly(trx, id, gdoc) } + + res.set("Cache-Control", "no-store") + res.send(gdoc) + } catch (error) { + console.error("Error fetching gdoc", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) } -) +} /** * Handles all four `GdocPublishingAction` cases @@ -152,7 +158,11 @@ async function indexAndBakeGdocIfNeccesary( * support creating a new Gdoc from an existing one. Relevant updates will * trigger a deploy. */ -putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { +export async function createOrUpdateGdoc( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { const { id } = req.params if (isEmpty(req.body)) { @@ -181,7 +191,7 @@ putRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { await indexAndBakeGdocIfNeccesary(trx, res.locals.user, prevGdoc, nextGdoc) return nextGdoc -}) +} async function validateTombstoneRelatedLinkUrl( trx: db.KnexReadonlyTransaction, @@ -201,7 +211,11 @@ async function validateTombstoneRelatedLinkUrl( } } -deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { +export async function deleteGdoc( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { const { id } = req.params const gdoc = await getGdocBaseObjectById(trx, id, false) @@ -264,20 +278,34 @@ deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", async (req, res, trx) => { await triggerStaticBuild(res.locals.user, `Deleting ${gdocSlug}`) } return {} -}) +} -postRouteWithRWTransaction( - apiRouter, - "/gdocs/:gdocId/setTags", - async (req, res, trx) => { - const { gdocId } = req.params - const { tagIds } = req.body - const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ - id: id, - })) +export async function setGdocTags( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const { gdocId } = req.params + const { tagIds } = req.body + const tagIdsAsObjects: { id: number }[] = tagIds.map((id: number) => ({ + id: id, + })) - await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) + await setTagsForGdoc(trx, gdocId, tagIdsAsObjects) - return { success: true } - } + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/gdocs", getAllGdocIndexItems) + +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/gdocs/:id", + getIndividualGdoc ) + +putRouteWithRWTransaction(apiRouter, "/gdocs/:id", createOrUpdateGdoc) + +deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", deleteGdoc) + +postRouteWithRWTransaction(apiRouter, "/gdocs/:gdocId/setTags", setGdocTags) diff --git a/adminSiteServer/apiRoutes/images.ts b/adminSiteServer/apiRoutes/images.ts index 0c5a611f33..b8b3b3db07 100644 --- a/adminSiteServer/apiRoutes/images.ts +++ b/adminSiteServer/apiRoutes/images.ts @@ -19,24 +19,30 @@ import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/images.json", - async (_, res, trx) => { - try { - const images = await db.getCloudflareImages(trx) - res.set("Cache-Control", "no-store") - res.send({ images }) - } catch (error) { - console.error("Error fetching images", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } +import { Request } from "../authentication.js" +import e from "express" +export async function getImagesHandler( + _: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + try { + const images = await db.getCloudflareImages(trx) + res.set("Cache-Control", "no-store") + res.send({ images }) + } catch (error) { + console.error("Error fetching images", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) } -) +} -postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { +export async function postImageHandler( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const { filename, type, content } = validateImagePayload(req.body) const { asBlob, dimensions, hash } = await processImageContent( @@ -94,14 +100,17 @@ postRouteWithRWTransaction(apiRouter, "/images", async (req, res, trx) => { success: true, image, } -}) - +} /** * Similar to the POST route, but for updating an existing image. * Creates a new image entry in the database and uploads the new image to Cloudflare. * The old image is marked as replaced by the new image. */ -putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { +export async function putImageHandler( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const { type, content } = validateImagePayload(req.body) const { asBlob, dimensions, hash } = await processImageContent( content, @@ -175,10 +184,13 @@ putRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { success: true, image: updated, } -}) - +} // Update alt text via patch -patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { +export async function patchImageHandler( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const { id } = req.params const image = await trx<DbEnrichedImage>("images") @@ -206,9 +218,13 @@ patchRouteWithRWTransaction(apiRouter, "/images/:id", async (req, res, trx) => { success: true, image: updated, } -}) +} -deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { +export async function deleteImageHandler( + req: Request, + _: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const { id } = req.params const image = await trx<DbEnrichedImage>("images") @@ -240,13 +256,34 @@ deleteRouteWithRWTransaction(apiRouter, "/images/:id", async (req, _, trx) => { return { success: true, } -}) +} -getRouteWithROTransaction(apiRouter, "/images/usage", async (_, __, trx) => { +export async function getImageUsageHandler( + _: Request, + __: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const usage = await db.getImageUsage(trx) return { success: true, usage, } -}) +} + +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/images.json", + getImagesHandler +) + +postRouteWithRWTransaction(apiRouter, "/images", postImageHandler) + +putRouteWithRWTransaction(apiRouter, "/images/:id", putImageHandler) + +// Update alt text via patch +patchRouteWithRWTransaction(apiRouter, "/images/:id", patchImageHandler) + +deleteRouteWithRWTransaction(apiRouter, "/images/:id", deleteImageHandler) + +getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler) diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts index a26116472e..34a05595d2 100644 --- a/adminSiteServer/apiRoutes/mdims.ts +++ b/adminSiteServer/apiRoutes/mdims.ts @@ -9,26 +9,35 @@ import { apiRouter } from "../apiRouter.js" import { putRouteWithRWTransaction } from "../functionalRouterHelpers.js" import { createMultiDimConfig } from "../multiDim.js" import { triggerStaticBuild } from "./routeUtils.js" +import { Request } from "../authentication.js" +import * as db from "../../db/db.js" +import e from "express" + +export async function handleMultiDimDataPageRequest( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const { slug } = req.params + if (!isValidSlug(slug)) { + throw new JsonError(`Invalid multi-dim slug ${slug}`) + } + const rawConfig = req.body as MultiDimDataPageConfigRaw + const id = await createMultiDimConfig(trx, slug, rawConfig) + if ( + FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && + (await isMultiDimDataPagePublished(trx, slug)) + ) { + await triggerStaticBuild( + res.locals.user, + `Publishing multidimensional chart ${slug}` + ) + } + return { success: true, id } +} putRouteWithRWTransaction( apiRouter, "/multi-dim/:slug", - async (req, res, trx) => { - const { slug } = req.params - if (!isValidSlug(slug)) { - throw new JsonError(`Invalid multi-dim slug ${slug}`) - } - const rawConfig = req.body as MultiDimDataPageConfigRaw - const id = await createMultiDimConfig(trx, slug, rawConfig) - if ( - FEATURE_FLAGS.has(FeatureFlagFeature.MultiDimDataPage) && - (await isMultiDimDataPagePublished(trx, slug)) - ) { - await triggerStaticBuild( - res.locals.user, - `Publishing multidimensional chart ${slug}` - ) - } - return { success: true, id } - } + handleMultiDimDataPageRequest ) diff --git a/adminSiteServer/apiRoutes/misc.ts b/adminSiteServer/apiRoutes/misc.ts index eb4efaef31..6d6d448dd9 100644 --- a/adminSiteServer/apiRoutes/misc.ts +++ b/adminSiteServer/apiRoutes/misc.ts @@ -13,8 +13,14 @@ import path from "path" import { DeployQueueServer } from "../../baker/DeployQueueServer.js" import { expectInt } from "../../serverUtils/serverUtil.js" import { triggerStaticBuild } from "./routeUtils.js" +import { Request } from "../authentication.js" +import e from "express" // using the alternate template, which highlights topics rather than articles. -getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { +export async function fetchAllWork( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { type WordpressPageRecord = { isWordpressPage: number } & Record< @@ -117,62 +123,62 @@ getRouteWithROTransaction(apiRouter, "/all-work", async (req, res, trx) => { res.type("text/plain") return [...generateAllWorkArchieMl()].join("") -}) - -getRouteWithROTransaction( - apiRouter, - "/editorData/namespaces.json", - async (req, res, trx) => { - const rows = await db.knexRaw<{ - name: string - description?: string - isArchived: boolean - }>( - trx, - `SELECT DISTINCT - namespace AS name, - namespaces.description AS description, - namespaces.isArchived AS isArchived - FROM active_datasets - JOIN namespaces ON namespaces.name = active_datasets.namespace` - ) +} + +export async function fetchNamespaces( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const rows = await db.knexRaw<{ + name: string + description?: string + isArchived: boolean + }>( + trx, + `SELECT DISTINCT + namespace AS name, + namespaces.description AS description, + namespaces.isArchived AS isArchived + FROM active_datasets + JOIN namespaces ON namespaces.name = active_datasets.namespace` + ) - return { - namespaces: lodash - .sortBy(rows, (row) => row.description) - .map((namespace) => ({ - ...namespace, - isArchived: !!namespace.isArchived, - })), - } + return { + namespaces: lodash + .sortBy(rows, (row) => row.description) + .map((namespace) => ({ + ...namespace, + isArchived: !!namespace.isArchived, + })), } -) +} -getRouteWithROTransaction( - apiRouter, - "/sources/:sourceId.json", - async (req, res, trx) => { - const sourceId = expectInt(req.params.sourceId) +export async function fetchSourceById( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const sourceId = expectInt(req.params.sourceId) - const source = await db.knexRawFirst<Record<string, any>>( - trx, - ` + const source = await db.knexRawFirst<Record<string, any>>( + trx, + ` SELECT s.id, s.name, s.description, s.createdAt, s.updatedAt, d.namespace FROM sources AS s JOIN active_datasets AS d ON d.id=s.datasetId WHERE s.id=?`, - [sourceId] - ) - if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) - source.variables = await db.knexRaw( - trx, - `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, - [sourceId] - ) + [sourceId] + ) + if (!source) throw new JsonError(`No source by id '${sourceId}'`, 404) + source.variables = await db.knexRaw( + trx, + `SELECT id, name, updatedAt FROM variables WHERE variables.sourceId=?`, + [sourceId] + ) - return { source: source } - } -) + return { source: source } +} apiRouter.get("/deploys.json", async () => ({ deploys: await new DeployQueueServer().getDeploys(), @@ -181,3 +187,11 @@ apiRouter.get("/deploys.json", async () => ({ apiRouter.put("/deploy", async (req, res) => { return triggerStaticBuild(res.locals.user, "Manually triggered deploy") }) + +getRouteWithROTransaction(apiRouter, "/all-work", fetchAllWork) +getRouteWithROTransaction( + apiRouter, + "/editorData/namespaces.json", + fetchNamespaces +) +getRouteWithROTransaction(apiRouter, "/sources/:sourceId.json", fetchSourceById) diff --git a/adminSiteServer/apiRoutes/posts.ts b/adminSiteServer/apiRoutes/posts.ts index 4f36fd0a12..efd31d99db 100644 --- a/adminSiteServer/apiRoutes/posts.ts +++ b/adminSiteServer/apiRoutes/posts.ts @@ -19,8 +19,13 @@ import { postRouteWithRWTransaction, } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" - -getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { +import { Request } from "../authentication.js" +import e from "express" +export async function handleGetPostsJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { const raw_rows = await db.knexRaw( trx, `-- sql @@ -88,133 +93,150 @@ getRouteWithROTransaction(apiRouter, "/posts.json", async (req, res, trx) => { })) return { posts: rows } -}) +} -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/setTags", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) +export async function handleSetTagsForPost( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + await setTagsForPost(trx, postId, req.body.tagIds) + return { success: true } +} - await setTagsForPost(trx, postId, req.body.tagIds) +export async function handleGetPostById( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const postId = expectInt(req.params.postId) + const post = (await trx + .table(PostsTableName) + .where({ id: postId }) + .select("*") + .first()) as DbRawPost | undefined + return camelCaseProperties({ ...post }) +} - return { success: true } - } -) +export async function handleCreateGdoc( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + const allowRecreate = !!req.body.allowRecreate + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined -getRouteWithROTransaction( - apiRouter, - "/posts/:postId.json", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!allowRecreate && existingGdocId) + throw new JsonError("A gdoc already exists for this post", 400) + if (allowRecreate && existingGdocId && post.isGdocPublished) { + throw new JsonError( + "A gdoc already exists for this post and it is already published", + 400 + ) + } + if (post.archieml === null) + throw new JsonError( + `ArchieML was not present for post with id ${postId}`, + 500 + ) + const tagsByPostId = await getTagsByPostId(trx) + const tags = tagsByPostId.get(postId) || [] + const archieMl = JSON.parse( + // Google Docs interprets ®ion in grapher URLS as ®ion + // So we escape them here + post.archieml.replaceAll("&", "&") + ) as OwidGdocPostInterface + const gdocId = await createGdocAndInsertOwidGdocPostContent( + archieMl.content, + post.gdocSuccessorId + ) + // If we did not yet have a gdoc associated with this post, we need to register + // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise + // we don't need to make changes to the DB (only the gdoc regeneration was required) + if (!existingGdocId) { + post.gdocSuccessorId = gdocId + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx .table(PostsTableName) .where({ id: postId }) - .select("*") - .first()) as DbRawPost | undefined - return camelCaseProperties({ ...post }) + .update("gdocSuccessorId", gdocId) + + const gdoc = new GdocPost(gdocId) + gdoc.slug = post.slug + gdoc.content.title = post.title + gdoc.content.type = archieMl.content.type || OwidGdocType.Article + gdoc.published = false + gdoc.createdAt = new Date() + gdoc.publishedAt = post.published_at + await upsertGdoc(trx, gdoc) + await setTagsForGdoc(trx, gdocId, tags) } -) + return { googleDocsId: gdocId } +} -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/createGdoc", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const allowRecreate = !!req.body.allowRecreate - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined +export async function handleUnlinkGdoc( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const postId = expectInt(req.params.postId) + const post = (await trx + .table("posts_with_gdoc_publish_status") + .where({ id: postId }) + .select("*") + .first()) as DbRawPostWithGdocPublishStatus | undefined - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!allowRecreate && existingGdocId) - throw new JsonError("A gdoc already exists for this post", 400) - if (allowRecreate && existingGdocId && post.isGdocPublished) { - throw new JsonError( - "A gdoc already exists for this post and it is already published", - 400 - ) - } - if (post.archieml === null) - throw new JsonError( - `ArchieML was not present for post with id ${postId}`, - 500 - ) - const tagsByPostId = await getTagsByPostId(trx) - const tags = tagsByPostId.get(postId) || [] - const archieMl = JSON.parse( - // Google Docs interprets ®ion in grapher URLS as ®ion - // So we escape them here - post.archieml.replaceAll("&", "&") - ) as OwidGdocPostInterface - const gdocId = await createGdocAndInsertOwidGdocPostContent( - archieMl.content, - post.gdocSuccessorId + if (!post) throw new JsonError(`No post found for id ${postId}`, 404) + const existingGdocId = post.gdocSuccessorId + if (!existingGdocId) + throw new JsonError("No gdoc exists for this post", 400) + if (existingGdocId && post.isGdocPublished) { + throw new JsonError( + "The GDoc is already published - you can't unlink it", + 400 ) - // If we did not yet have a gdoc associated with this post, we need to register - // the gdocSuccessorId and create an entry in the posts_gdocs table. Otherwise - // we don't need to make changes to the DB (only the gdoc regeneration was required) - if (!existingGdocId) { - post.gdocSuccessorId = gdocId - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", gdocId) - - const gdoc = new GdocPost(gdocId) - gdoc.slug = post.slug - gdoc.content.title = post.title - gdoc.content.type = archieMl.content.type || OwidGdocType.Article - gdoc.published = false - gdoc.createdAt = new Date() - gdoc.publishedAt = post.published_at - await upsertGdoc(trx, gdoc) - await setTagsForGdoc(trx, gdocId, tags) - } - return { googleDocsId: gdocId } } -) + // This is not ideal - we are using knex for on thing and typeorm for another + // which means that we can't wrap this in a transaction. We should probably + // move posts to use typeorm as well or at least have a typeorm alternative for it + await trx + .table(PostsTableName) + .where({ id: postId }) + .update("gdocSuccessorId", null) + + await trx.table(PostsGdocsTableName).where({ id: existingGdocId }).delete() + + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/posts.json", handleGetPostsJson) postRouteWithRWTransaction( apiRouter, - "/posts/:postId/unlinkGdoc", - async (req, res, trx) => { - const postId = expectInt(req.params.postId) - const post = (await trx - .table("posts_with_gdoc_publish_status") - .where({ id: postId }) - .select("*") - .first()) as DbRawPostWithGdocPublishStatus | undefined + "/posts/:postId/setTags", + handleSetTagsForPost +) - if (!post) throw new JsonError(`No post found for id ${postId}`, 404) - const existingGdocId = post.gdocSuccessorId - if (!existingGdocId) - throw new JsonError("No gdoc exists for this post", 400) - if (existingGdocId && post.isGdocPublished) { - throw new JsonError( - "The GDoc is already published - you can't unlink it", - 400 - ) - } - // This is not ideal - we are using knex for on thing and typeorm for another - // which means that we can't wrap this in a transaction. We should probably - // move posts to use typeorm as well or at least have a typeorm alternative for it - await trx - .table(PostsTableName) - .where({ id: postId }) - .update("gdocSuccessorId", null) +getRouteWithROTransaction(apiRouter, "/posts/:postId.json", handleGetPostById) - await trx - .table(PostsGdocsTableName) - .where({ id: existingGdocId }) - .delete() +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/createGdoc", + handleCreateGdoc +) - return { success: true } - } +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/unlinkGdoc", + handleUnlinkGdoc ) diff --git a/adminSiteServer/apiRoutes/redirects.ts b/adminSiteServer/apiRoutes/redirects.ts index 0752c4ece1..00f8971b07 100644 --- a/adminSiteServer/apiRoutes/redirects.ts +++ b/adminSiteServer/apiRoutes/redirects.ts @@ -14,78 +14,82 @@ import { } from "../functionalRouterHelpers.js" import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" +import { Request } from "../authentication.js" +import e from "express" +export async function handleGetSiteRedirects( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await getRedirects(trx) } +} -getRouteWithROTransaction( - apiRouter, - "/site-redirects.json", - async (req, res, trx) => ({ redirects: await getRedirects(trx) }) -) - -postRouteWithRWTransaction( - apiRouter, - "/site-redirects/new", - async (req, res, trx) => { - const { source, target } = req.body - const sourceAsUrl = new URL(source, "https://ourworldindata.org") - if (sourceAsUrl.pathname === "/") - throw new JsonError("Cannot redirect from /", 400) - if (await redirectWithSourceExists(trx, source)) { - throw new JsonError( - `Redirect with source ${source} already exists`, - 400 - ) - } - const chainedRedirect = await getChainedRedirect(trx, source, target) - if (chainedRedirect) { - throw new JsonError( - "Creating this redirect would create a chain, redirect from " + - `${chainedRedirect.source} to ${chainedRedirect.target} ` + - "already exists. " + - (target === chainedRedirect.source - ? `Please create the redirect from ${source} to ` + - `${chainedRedirect.target} directly instead.` - : `Please delete the existing redirect and create a ` + - `new redirect from ${chainedRedirect.source} to ` + - `${target} instead.`), - 400 - ) - } - const { insertId: id } = await db.knexRawInsert( - trx, - `INSERT INTO redirects (source, target) VALUES (?, ?)`, - [source, target] +export async function handlePostNewSiteRedirect( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const { source, target } = req.body + const sourceAsUrl = new URL(source, "https://ourworldindata.org") + if (sourceAsUrl.pathname === "/") + throw new JsonError("Cannot redirect from /", 400) + if (await redirectWithSourceExists(trx, source)) { + throw new JsonError( + `Redirect with source ${source} already exists`, + 400 ) - await triggerStaticBuild( - res.locals.user, - `Creating redirect id=${id} source=${source} target=${target}` + } + const chainedRedirect = await getChainedRedirect(trx, source, target) + if (chainedRedirect) { + throw new JsonError( + "Creating this redirect would create a chain, redirect from " + + `${chainedRedirect.source} to ${chainedRedirect.target} ` + + "already exists. " + + (target === chainedRedirect.source + ? `Please create the redirect from ${source} to ` + + `${chainedRedirect.target} directly instead.` + : `Please delete the existing redirect and create a ` + + `new redirect from ${chainedRedirect.source} to ` + + `${target} instead.`), + 400 ) - return { success: true, redirect: { id, source, target } } } -) + const { insertId: id } = await db.knexRawInsert( + trx, + `INSERT INTO redirects (source, target) VALUES (?, ?)`, + [source, target] + ) + await triggerStaticBuild( + res.locals.user, + `Creating redirect id=${id} source=${source} target=${target}` + ) + return { success: true, redirect: { id, source, target } } +} -deleteRouteWithRWTransaction( - apiRouter, - "/site-redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - const redirect = await getRedirectById(trx, id) - if (!redirect) { - throw new JsonError(`No redirect found for id ${id}`, 404) - } - await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` - ) - return { success: true } +export async function handleDeleteSiteRedirect( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + const redirect = await getRedirectById(trx, id) + if (!redirect) { + throw new JsonError(`No redirect found for id ${id}`, 404) } -) + await db.knexRaw(trx, `DELETE FROM redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect id=${id} source=${redirect.source} target=${redirect.target}` + ) + return { success: true } +} -// Get a list of redirects that map old slugs to charts -getRouteWithROTransaction( - apiRouter, - "/redirects.json", - async (req, res, trx) => ({ +export async function handleGetRedirects( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + return { redirects: await db.knexRaw( trx, `-- sql @@ -100,53 +104,82 @@ getRouteWithROTransaction( ORDER BY r.id DESC ` ), - }) + } +} + +export async function handlePostNewChartRedirect( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const chartId = expectInt(req.params.chartId) + const fields = req.body as { slug: string } + const result = await db.knexRawInsert( + trx, + `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, + [chartId, fields.slug] + ) + const redirectId = result.insertId + const redirect = await db.knexRaw<DbPlainChartSlugRedirect>( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [redirectId] + ) + return { success: true, redirect: redirect } +} + +export async function handleDeleteChartRedirect( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const id = expectInt(req.params.id) + + const redirect = await db.knexRawFirst<DbPlainChartSlugRedirect>( + trx, + `SELECT * FROM chart_slug_redirects WHERE id = ?`, + [id] + ) + + if (!redirect) throw new JsonError(`No redirect found for id ${id}`, 404) + + await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [id]) + await triggerStaticBuild( + res.locals.user, + `Deleting redirect from ${redirect.slug}` + ) + + return { success: true } +} + +getRouteWithROTransaction( + apiRouter, + "/site-redirects.json", + handleGetSiteRedirects ) postRouteWithRWTransaction( apiRouter, - "/charts/:chartId/redirects/new", - async (req, res, trx) => { - const chartId = expectInt(req.params.chartId) - const fields = req.body as { slug: string } - const result = await db.knexRawInsert( - trx, - `INSERT INTO chart_slug_redirects (chart_id, slug) VALUES (?, ?)`, - [chartId, fields.slug] - ) - const redirectId = result.insertId - const redirect = await db.knexRaw<DbPlainChartSlugRedirect>( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [redirectId] - ) - return { success: true, redirect: redirect } - } + "/site-redirects/new", + handlePostNewSiteRedirect ) deleteRouteWithRWTransaction( apiRouter, - "/redirects/:id", - async (req, res, trx) => { - const id = expectInt(req.params.id) - - const redirect = await db.knexRawFirst<DbPlainChartSlugRedirect>( - trx, - `SELECT * FROM chart_slug_redirects WHERE id = ?`, - [id] - ) + "/site-redirects/:id", + handleDeleteSiteRedirect +) - if (!redirect) - throw new JsonError(`No redirect found for id ${id}`, 404) +getRouteWithROTransaction(apiRouter, "/redirects.json", handleGetRedirects) - await db.knexRaw(trx, `DELETE FROM chart_slug_redirects WHERE id=?`, [ - id, - ]) - await triggerStaticBuild( - res.locals.user, - `Deleting redirect from ${redirect.slug}` - ) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/redirects/new", + handlePostNewChartRedirect +) - return { success: true } - } +deleteRouteWithRWTransaction( + apiRouter, + "/redirects/:id", + handleDeleteChartRedirect ) diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts index 2b9e3303fa..657d0b6b1f 100644 --- a/adminSiteServer/apiRoutes/suggest.ts +++ b/adminSiteServer/apiRoutes/suggest.ts @@ -10,62 +10,70 @@ import { CLOUDFLARE_IMAGES_URL } from "../../settings/clientSettings.js" import { apiRouter } from "../apiRouter.js" import { getRouteWithROTransaction } from "../functionalRouterHelpers.js" import { fetchGptGeneratedAltText } from "../imagesHelpers.js" +import * as db from "../../db/db.js" +import e from "express" +import { Request } from "../authentication.js" -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, - async (req, res, trx): Promise<Record<"topics", DbChartTagJoin[]>> => { - const chartId = parseIntOrUndefined(req.params.chartId) - if (!chartId) throw new JsonError(`Invalid chart ID`, 400) +export async function suggestGptTopics( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +): Promise<Record<"topics", DbChartTagJoin[]>> { + const chartId = parseIntOrUndefined(req.params.chartId) + if (!chartId) throw new JsonError(`Invalid chart ID`, 400) - const topics = await getGptTopicSuggestions(trx, chartId) + const topics = await getGptTopicSuggestions(trx, chartId) - if (!topics.length) - throw new JsonError( - `No GPT topic suggestions found for chart ${chartId}`, - 404 - ) + if (!topics.length) + throw new JsonError( + `No GPT topic suggestions found for chart ${chartId}`, + 404 + ) - return { - topics, - } + return { + topics, } +} + +export async function suggestGptAltText( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +): Promise<{ + success: boolean + altText: string | null +}> { + const imageId = parseIntOrUndefined(req.params.imageId) + if (!imageId) throw new JsonError(`Invalid image ID`, 400) + const image = await trx<DbEnrichedImage>("images") + .where("id", imageId) + .first() + if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) + + const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` + let altText: string | null = "" + try { + altText = await fetchGptGeneratedAltText(src) + } catch (error) { + console.error(`Error fetching GPT alt text for image ${imageId}`, error) + throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) + } + + if (!altText) { + throw new JsonError(`Unable to generate alt text for image`, 404) + } + + return { success: true, altText } +} + +getRouteWithROTransaction( + apiRouter, + `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, + suggestGptTopics ) getRouteWithROTransaction( apiRouter, `/gpt/suggest-alt-text/:imageId`, - async ( - req, - res, - trx - ): Promise<{ - success: boolean - altText: string | null - }> => { - const imageId = parseIntOrUndefined(req.params.imageId) - if (!imageId) throw new JsonError(`Invalid image ID`, 400) - const image = await trx<DbEnrichedImage>("images") - .where("id", imageId) - .first() - if (!image) throw new JsonError(`No image found for ID ${imageId}`, 404) - - const src = `${CLOUDFLARE_IMAGES_URL}/${image.cloudflareId}/public` - let altText: string | null = "" - try { - altText = await fetchGptGeneratedAltText(src) - } catch (error) { - console.error( - `Error fetching GPT alt text for image ${imageId}`, - error - ) - throw new JsonError(`Error fetching GPT alt text: ${error}`, 500) - } - - if (!altText) { - throw new JsonError(`Unable to generate alt text for image`, 404) - } - - return { success: true, altText } - } + suggestGptAltText ) diff --git a/adminSiteServer/apiRoutes/tagGraph.ts b/adminSiteServer/apiRoutes/tagGraph.ts index 3690e2e541..f4dfc8b7b2 100644 --- a/adminSiteServer/apiRoutes/tagGraph.ts +++ b/adminSiteServer/apiRoutes/tagGraph.ts @@ -7,17 +7,23 @@ import { } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" +import { Request } from "../authentication.js" +import e from "express" -getRouteWithROTransaction( - apiRouter, - "/flatTagGraph.json", - async (req, res, trx) => { - const flatTagGraph = await db.getFlatTagGraph(trx) - return flatTagGraph - } -) +export async function handleGetFlatTagGraph( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const flatTagGraph = await db.getFlatTagGraph(trx) + return flatTagGraph +} -postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { +export async function handlePostTagGraph( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { const tagGraph = req.body?.tagGraph as unknown if (!tagGraph) { throw new JsonError("No tagGraph provided", 400) @@ -51,10 +57,19 @@ postRouteWithRWTransaction(apiRouter, "/tagGraph", async (req, res, trx) => { return true } + const isValid = validateFlatTagGraph(tagGraph) if (!isValid) { throw new JsonError("Invalid tag graph provided", 400) } await db.updateTagGraph(trx, tagGraph) res.send({ success: true }) -}) +} + +getRouteWithROTransaction( + apiRouter, + "/flatTagGraph.json", + handleGetFlatTagGraph +) + +postRouteWithRWTransaction(apiRouter, "/tagGraph", handlePostTagGraph) diff --git a/adminSiteServer/apiRoutes/tags.ts b/adminSiteServer/apiRoutes/tags.ts index 0e698df454..578209cfe2 100644 --- a/adminSiteServer/apiRoutes/tags.ts +++ b/adminSiteServer/apiRoutes/tags.ts @@ -21,57 +21,54 @@ import { } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" +import e from "express" import { Request } from "../authentication.js" -getRouteWithROTransaction( - apiRouter, - "/tags/:tagId.json", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) as number | null +export async function getTagById( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const tagId = expectInt(req.params.tagId) as number | null - // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff - // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag - // every time we create a new chart etcs - const uncategorized = tagId === UNCATEGORIZED_TAG_ID + // NOTE (Mispy): The "uncategorized" tag is special -- it represents all untagged stuff + // Bit fiddly to handle here but more true to normalized schema than having to remember to add the special tag + // every time we create a new chart etcs + const uncategorized = tagId === UNCATEGORIZED_TAG_ID - // TODO: when we have types for our endpoints, make tag of that type instead of any - const tag: any = await db.knexRawFirst< - Pick< - DbPlainTag, - | "id" - | "name" - | "specialType" - | "updatedAt" - | "parentId" - | "slug" - > - >( - trx, - `-- sql + // TODO: when we have types for our endpoints, make tag of that type instead of any + const tag: any = await db.knexRawFirst< + Pick< + DbPlainTag, + "id" | "name" | "specialType" | "updatedAt" | "parentId" | "slug" + > + >( + trx, + `-- sql SELECT t.id, t.name, t.specialType, t.updatedAt, t.parentId, t.slug FROM tags t LEFT JOIN tags p ON t.parentId=p.id WHERE t.id = ? `, - [tagId] - ) + [tagId] + ) - // Datasets tagged with this tag - const datasets = await db.knexRaw< - Pick< - DbPlainDataset, - | "id" - | "namespace" - | "name" - | "description" - | "createdAt" - | "updatedAt" - | "dataEditedAt" - | "isPrivate" - | "nonRedistributable" - > & { dataEditedByUserName: string } - >( - trx, - `-- sql + // Datasets tagged with this tag + const datasets = await db.knexRaw< + Pick< + DbPlainDataset, + | "id" + | "namespace" + | "name" + | "description" + | "createdAt" + | "updatedAt" + | "dataEditedAt" + | "isPrivate" + | "nonRedistributable" + > & { dataEditedByUserName: string } + >( + trx, + `-- sql SELECT d.id, d.namespace, @@ -89,44 +86,44 @@ getRouteWithROTransaction( WHERE dt.tagId ${uncategorized ? "IS NULL" : "= ?"} ORDER BY d.dataEditedAt DESC `, - uncategorized ? [] : [tagId] - ) - tag.datasets = datasets + uncategorized ? [] : [tagId] + ) + tag.datasets = datasets - // The other tags for those datasets - if (tag.datasets.length) { - if (uncategorized) { - for (const dataset of tag.datasets) dataset.tags = [] - } else { - const datasetTags = await db.knexRaw<{ - datasetId: number - id: number - name: string - }>( - trx, - `-- sql + // The other tags for those datasets + if (tag.datasets.length) { + if (uncategorized) { + for (const dataset of tag.datasets) dataset.tags = [] + } else { + const datasetTags = await db.knexRaw<{ + datasetId: number + id: number + name: string + }>( + trx, + `-- sql SELECT dt.datasetId, t.id, t.name FROM dataset_tags dt JOIN tags t ON dt.tagId = t.id WHERE dt.datasetId IN (?) `, - [tag.datasets.map((d: any) => d.id)] - ) - const tagsByDatasetId = lodash.groupBy( - datasetTags, - (t) => t.datasetId + [tag.datasets.map((d: any) => d.id)] + ) + const tagsByDatasetId = lodash.groupBy( + datasetTags, + (t) => t.datasetId + ) + for (const dataset of tag.datasets) { + dataset.tags = tagsByDatasetId[dataset.id].map((t) => + lodash.omit(t, "datasetId") ) - for (const dataset of tag.datasets) { - dataset.tags = tagsByDatasetId[dataset.id].map((t) => - lodash.omit(t, "datasetId") - ) - } } } + } - // Charts using datasets under this tag - const charts = await db.knexRaw<OldChartFieldList>( - trx, - `-- sql + // Charts using datasets under this tag + const charts = await db.knexRaw<OldChartFieldList>( + trx, + `-- sql SELECT ${oldChartFieldList} FROM charts JOIN chart_configs ON chart_configs.id = charts.configId LEFT JOIN chart_tags ct ON ct.chartId=charts.id @@ -136,134 +133,142 @@ getRouteWithROTransaction( GROUP BY charts.id ORDER BY charts.updatedAt DESC `, - uncategorized ? [] : [tagId] - ) - tag.charts = charts + uncategorized ? [] : [tagId] + ) + tag.charts = charts - await assignTagsForCharts(trx, charts) + await assignTagsForCharts(trx, charts) - // Subcategories - const children = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql + // Subcategories + const children = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.parentId = ? `, - [tag.id] - ) - tag.children = children + [tag.id] + ) + tag.children = children - // Possible parents to choose from - const possibleParents = await db.knexRaw<{ id: number; name: string }>( - trx, - `-- sql + const possibleParents = await db.knexRaw<{ id: number; name: string }>( + trx, + `-- sql SELECT t.id, t.name FROM tags t WHERE t.parentId IS NULL ` - ) - tag.possibleParents = possibleParents + ) + tag.possibleParents = possibleParents - return { - tag, - } + return { + tag, } -) +} -putRouteWithRWTransaction( - apiRouter, - "/tags/:tagId", - async (req: Request, res, trx) => { - const tagId = expectInt(req.params.tagId) - const tag = (req.body as { tag: any }).tag - await db.knexRaw( +export async function updateTag( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const tagId = expectInt(req.params.tagId) + const tag = (req.body as { tag: any }).tag + await db.knexRaw( + trx, + `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, + [tag.name, new Date(), tag.slug, tagId] + ) + if (tag.slug) { + // See if there's a published gdoc with a matching slug. + // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, + // where the page for the topic is just an article. + const gdoc = await db.knexRaw<Pick<DbRawPostGdoc, "slug">>( trx, - `UPDATE tags SET name=?, updatedAt=?, slug=? WHERE id=?`, - [tag.name, new Date(), tag.slug, tagId] - ) - if (tag.slug) { - // See if there's a published gdoc with a matching slug. - // We're not enforcing that the gdoc be a topic page, as there are cases like /human-development-index, - // where the page for the topic is just an article. - const gdoc = await db.knexRaw<Pick<DbRawPostGdoc, "slug">>( - trx, - `-- sql + `-- sql SELECT slug FROM posts_gdocs pg WHERE EXISTS ( SELECT 1 FROM posts_gdocs_x_tags gt WHERE pg.id = gt.gdocId AND gt.tagId = ? ) AND pg.published = TRUE AND pg.slug = ?`, - [tagId, tag.slug] - ) - if (!gdoc.length) { - return { - success: true, - tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. + [tagId, tag.slug] + ) + if (!gdoc.length) { + return { + success: true, + tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. Are you sure you haven't made a typo?`, - } } } - return { success: true } } -) + return { success: true } +} -postRouteWithRWTransaction( - apiRouter, - "/tags/new", - async (req: Request, res, trx) => { - const tag = req.body - function validateTag( - tag: unknown - ): tag is { name: string; slug: string | null } { - return ( - checkIsPlainObjectWithGuard(tag) && - typeof tag.name === "string" && - (tag.slug === null || - (typeof tag.slug === "string" && tag.slug !== "")) - ) - } - if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) - - const conflictingTag = await db.knexRawFirst<{ - name: string - slug: string | null - }>( - trx, - `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, - [tag.name, tag.slug] +export async function createTag( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const tag = req.body + function validateTag( + tag: unknown + ): tag is { name: string; slug: string | null } { + return ( + checkIsPlainObjectWithGuard(tag) && + typeof tag.name === "string" && + (tag.slug === null || + (typeof tag.slug === "string" && tag.slug !== "")) ) - if (conflictingTag) - throw new JsonError( - conflictingTag.name === tag.name - ? `Tag with name ${tag.name} already exists` - : `Tag with slug ${tag.slug} already exists`, - 400 - ) + } + if (!validateTag(tag)) throw new JsonError("Invalid tag", 400) - const now = new Date() - const result = await db.knexRawInsert( - trx, - `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, - // parentId will be deprecated soon once we migrate fully to the tag graph - [tag.name, tag.slug, now, now] + const conflictingTag = await db.knexRawFirst<{ + name: string + slug: string | null + }>( + trx, + `SELECT name, slug FROM tags WHERE name = ? OR (slug IS NOT NULL AND slug = ?)`, + [tag.name, tag.slug] + ) + if (conflictingTag) + throw new JsonError( + conflictingTag.name === tag.name + ? `Tag with name ${tag.name} already exists` + : `Tag with slug ${tag.slug} already exists`, + 400 ) - return { success: true, tagId: result.insertId } - } -) -getRouteWithROTransaction(apiRouter, "/tags.json", async (req, res, trx) => { + const now = new Date() + const result = await db.knexRawInsert( + trx, + `INSERT INTO tags (name, slug, createdAt, updatedAt) VALUES (?, ?, ?, ?)`, + // parentId will be deprecated soon once we migrate fully to the tag graph + [tag.name, tag.slug, now, now] + ) + return { success: true, tagId: result.insertId } +} + +export async function getAllTags( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { return { tags: await db.getMinimalTagsWithIsTopic(trx) } -}) +} -deleteRouteWithRWTransaction( - apiRouter, - "/tags/:tagId/delete", - async (req, res, trx) => { - const tagId = expectInt(req.params.tagId) +export async function deleteTag( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const tagId = expectInt(req.params.tagId) - await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) + await db.knexRaw(trx, `DELETE FROM tags WHERE id=?`, [tagId]) - return { success: true } - } -) + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/tags/:tagId.json", getTagById) +putRouteWithRWTransaction(apiRouter, "/tags/:tagId", updateTag) +postRouteWithRWTransaction(apiRouter, "/tags/new", createTag) +getRouteWithROTransaction(apiRouter, "/tags.json", getAllTags) +deleteRouteWithRWTransaction(apiRouter, "/tags/:tagId/delete", deleteTag) diff --git a/adminSiteServer/apiRoutes/users.ts b/adminSiteServer/apiRoutes/users.ts index 256ad22995..ea2016608e 100644 --- a/adminSiteServer/apiRoutes/users.ts +++ b/adminSiteServer/apiRoutes/users.ts @@ -11,108 +11,134 @@ import { postRouteWithRWTransaction, } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" - -getRouteWithROTransaction(apiRouter, "/users.json", async (req, res, trx) => ({ - users: await trx - .select( - "id" satisfies keyof DbPlainUser, - "email" satisfies keyof DbPlainUser, - "fullName" satisfies keyof DbPlainUser, - "isActive" satisfies keyof DbPlainUser, - "isSuperuser" satisfies keyof DbPlainUser, - "createdAt" satisfies keyof DbPlainUser, - "updatedAt" satisfies keyof DbPlainUser, - "lastLogin" satisfies keyof DbPlainUser, - "lastSeen" satisfies keyof DbPlainUser - ) - .from<DbPlainUser>(UsersTableName) - .orderBy("lastSeen", "desc"), -})) - -getRouteWithROTransaction( - apiRouter, - "/users/:userId.json", - async (req, res, trx) => { - const id = parseIntOrUndefined(req.params.userId) - if (!id) throw new JsonError("No user id given") - const user = await getUserById(trx, id) - return { user } - } -) - -deleteRouteWithRWTransaction( - apiRouter, - "/users/:userId", - async (req, res, trx) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = expectInt(req.params.userId) - await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) - - return { success: true } - } -) - -putRouteWithRWTransaction( - apiRouter, - "/users/:userId", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const userId = parseIntOrUndefined(req.params.userId) - const user = - userId !== undefined ? await getUserById(trx, userId) : null - if (!user) throw new JsonError("No such user", 404) - - user.fullName = req.body.fullName - user.isActive = req.body.isActive - - await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) - - return { success: true } - } -) - -postRouteWithRWTransaction( - apiRouter, - "/users/add", - async (req, res, trx: db.KnexReadWriteTransaction) => { - if (!res.locals.user.isSuperuser) - throw new JsonError("Permission denied", 403) - - const { email, fullName } = req.body - - await insertUser(trx, { - email, - fullName, - }) - - return { success: true } +import { Request } from "../authentication.js" +import e from "express" +export async function getUsers( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + return { + users: await trx + .select( + "id" satisfies keyof DbPlainUser, + "email" satisfies keyof DbPlainUser, + "fullName" satisfies keyof DbPlainUser, + "isActive" satisfies keyof DbPlainUser, + "isSuperuser" satisfies keyof DbPlainUser, + "createdAt" satisfies keyof DbPlainUser, + "updatedAt" satisfies keyof DbPlainUser, + "lastLogin" satisfies keyof DbPlainUser, + "lastSeen" satisfies keyof DbPlainUser + ) + .from<DbPlainUser>(UsersTableName) + .orderBy("lastSeen", "desc"), } -) +} + +export async function getUserByIdHandler( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const id = parseIntOrUndefined(req.params.userId) + if (!id) throw new JsonError("No user id given") + const user = await getUserById(trx, id) + return { user } +} + +export async function deleteUser( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = expectInt(req.params.userId) + await db.knexRaw(trx, `DELETE FROM users WHERE id=?`, [userId]) + + return { success: true } +} + +export async function updateUserHandler( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const userId = parseIntOrUndefined(req.params.userId) + const user = userId !== undefined ? await getUserById(trx, userId) : null + if (!user) throw new JsonError("No such user", 404) + + user.fullName = req.body.fullName + user.isActive = req.body.isActive + + await updateUser(trx, userId!, pick(user, ["fullName", "isActive"])) + + return { success: true } +} + +export async function addUser( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + if (!res.locals.user.isSuperuser) + throw new JsonError("Permission denied", 403) + + const { email, fullName } = req.body + + await insertUser(trx, { + email, + fullName, + }) + + return { success: true } +} + +export async function addImageToUser( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId }).update({ userId }) + return { success: true } +} + +export async function removeUserImage( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const userId = expectInt(req.params.userId) + const imageId = expectInt(req.params.imageId) + await trx("images").where({ id: imageId, userId }).update({ userId: null }) + return { success: true } +} + +getRouteWithROTransaction(apiRouter, "/users.json", getUsers) + +getRouteWithROTransaction(apiRouter, "/users/:userId.json", getUserByIdHandler) + +deleteRouteWithRWTransaction(apiRouter, "/users/:userId", deleteUser) + +putRouteWithRWTransaction(apiRouter, "/users/:userId", updateUserHandler) + +postRouteWithRWTransaction(apiRouter, "/users/add", addUser) postRouteWithRWTransaction( apiRouter, "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images").where({ id: imageId }).update({ userId }) - return { success: true } - } + addImageToUser ) deleteRouteWithRWTransaction( apiRouter, "/users/:userId/images/:imageId", - async (req, res, trx) => { - const userId = expectInt(req.params.userId) - const imageId = expectInt(req.params.imageId) - await trx("images") - .where({ id: imageId, userId }) - .update({ userId: null }) - return { success: true } - } + removeUserImage ) diff --git a/adminSiteServer/apiRoutes/variables.ts b/adminSiteServer/apiRoutes/variables.ts index 0853e92934..f8f21a65ab 100644 --- a/adminSiteServer/apiRoutes/variables.ts +++ b/adminSiteServer/apiRoutes/variables.ts @@ -48,24 +48,27 @@ import { expectInt } from "../../serverUtils/serverUtil.js" import { triggerStaticBuild } from "./routeUtils.js" import * as lodash from "lodash" import { updateGrapherConfigsInR2 } from "./charts.js" - -getRouteWithROTransaction( - apiRouter, - "/editorData/variables.json", - async (req, res, trx) => { - const datasets = [] - const rows = await db.knexRaw< - Pick<DbRawVariable, "name" | "id"> & { - datasetId: number - datasetName: string - datasetVersion: string - } & Pick< - DbPlainDataset, - "namespace" | "isPrivate" | "nonRedistributable" - > - >( - trx, - `-- sql +import { Request } from "../authentication.js" +import e from "express" + +export async function getEditorVariablesJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const datasets = [] + const rows = await db.knexRaw< + Pick<DbRawVariable, "name" | "id"> & { + datasetId: number + datasetName: string + datasetVersion: string + } & Pick< + DbPlainDataset, + "namespace" | "isPrivate" | "nonRedistributable" + > + >( + trx, + `-- sql SELECT v.name, v.id, @@ -78,47 +81,50 @@ getRouteWithROTransaction( FROM variables as v JOIN active_datasets as d ON v.datasetId = d.id ORDER BY d.updatedAt DESC ` - ) + ) - let dataset: - | { - id: number - name: string - version: string - namespace: string - isPrivate: boolean - nonRedistributable: boolean - variables: { id: number; name: string }[] - } - | undefined - for (const row of rows) { - if (!dataset || row.datasetName !== dataset.name) { - if (dataset) datasets.push(dataset) - - dataset = { - id: row.datasetId, - name: row.datasetName, - version: row.datasetVersion, - namespace: row.namespace, - isPrivate: !!row.isPrivate, - nonRedistributable: !!row.nonRedistributable, - variables: [], - } + let dataset: + | { + id: number + name: string + version: string + namespace: string + isPrivate: boolean + nonRedistributable: boolean + variables: { id: number; name: string }[] + } + | undefined + for (const row of rows) { + if (!dataset || row.datasetName !== dataset.name) { + if (dataset) datasets.push(dataset) + + dataset = { + id: row.datasetId, + name: row.datasetName, + version: row.datasetVersion, + namespace: row.namespace, + isPrivate: !!row.isPrivate, + nonRedistributable: !!row.nonRedistributable, + variables: [], } - - dataset.variables.push({ - id: row.id, - name: row.name ?? "", - }) } - if (dataset) datasets.push(dataset) - - return { datasets: datasets } + dataset.variables.push({ + id: row.id, + name: row.name ?? "", + }) } -) -apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { + if (dataset) datasets.push(dataset) + + return { datasets: datasets } +} + +export async function getVariableDataJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + _trx: db.KnexReadonlyTransaction +) { const variableStr = req.params.variableStr as string if (!variableStr) throw new JsonError("No variable id given") if (variableStr.includes("+")) @@ -130,40 +136,42 @@ apiRouter.get("/data/variables/data/:variableStr.json", async (req, res) => { return await fetchS3DataValuesByPath( getVariableDataRoute(DATA_API_URL, variableId) + "?nocache" ) -}) +} -apiRouter.get( - "/data/variables/metadata/:variableStr.json", - async (req, res) => { - const variableStr = req.params.variableStr as string - if (!variableStr) throw new JsonError("No variable id given") - if (variableStr.includes("+")) - throw new JsonError( - "Requesting multiple variables at the same time is no longer supported" - ) - const variableId = parseInt(variableStr) - if (isNaN(variableId)) throw new JsonError("Invalid variable id") - return await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" +export async function getVariableMetadataJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + _trx: db.KnexReadonlyTransaction +) { + const variableStr = req.params.variableStr as string + if (!variableStr) throw new JsonError("No variable id given") + if (variableStr.includes("+")) + throw new JsonError( + "Requesting multiple variables at the same time is no longer supported" ) - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables.json", - async (req, res, trx) => { - const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 - const query = req.query.search as string - return await searchVariables(query, limit, trx) - } -) - -getRouteWithROTransaction( - apiRouter, - "/variables.usages.json", - async (req, res, trx) => { - const query = `-- sql + const variableId = parseInt(variableStr) + if (isNaN(variableId)) throw new JsonError("Invalid variable id") + return await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) +} + +export async function getVariablesJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const limit = parseIntOrUndefined(req.query.limit as string) ?? 50 + const query = req.query.search as string + return await searchVariables(query, limit, trx) +} + +export async function getVariablesUsagesJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const query = `-- sql SELECT variableId, COUNT(DISTINCT chartId) AS usageCount @@ -174,74 +182,73 @@ getRouteWithROTransaction( ORDER BY usageCount DESC` - const rows = await db.knexRaw(trx, query) + const rows = await db.knexRaw(trx, query) - return rows - } -) + return rows +} -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigETL/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.etl?.patchConfig ?? {} +export async function getVariablesGrapherConfigETLPatchConfigJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) } -) - -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigAdmin/:variableId.patchConfig.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } - return variable.admin?.patchConfig ?? {} + return variable.etl?.patchConfig ?? {} +} + +export async function getVariablesGrapherConfigAdminPatchConfigJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) } -) + return variable.admin?.patchConfig ?? {} +} + +export async function getVariablesMergedGrapherConfigJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const config = await getMergedGrapherConfigForVariable(trx, variableId) + return config ?? {} +} + +export async function getVariablesVariableIdJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + + const variable = await fetchS3MetadataByPath( + getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" + ) -getRouteWithROTransaction( - apiRouter, - "/variables/mergedGrapherConfig/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const config = await getMergedGrapherConfigForVariable(trx, variableId) - return config ?? {} + // XXX: Patch shortName onto the end of catalogPath when it's missing, + // a temporary hack since our S3 metadata is out of date with our DB. + // See: https://github.com/owid/etl/issues/2135 + if (variable.catalogPath && !variable.catalogPath.includes("#")) { + variable.catalogPath += `#${variable.shortName}` } -) -// Used in VariableEditPage -getRouteWithROTransaction( - apiRouter, - "/variables/:variableId.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - const variable = await fetchS3MetadataByPath( - getVariableMetadataRoute(DATA_API_URL, variableId) + "?nocache" - ) - - // XXX: Patch shortName onto the end of catalogPath when it's missing, - // a temporary hack since our S3 metadata is out of date with our DB. - // See: https://github.com/owid/etl/issues/2135 - if (variable.catalogPath && !variable.catalogPath.includes("#")) { - variable.catalogPath += `#${variable.shortName}` + const rawCharts = await db.knexRaw< + OldChartFieldList & { + isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] + config: DbRawChartConfig["full"] } - - const rawCharts = await db.knexRaw< - OldChartFieldList & { - isInheritanceEnabled: DbPlainChart["isInheritanceEnabled"] - config: DbRawChartConfig["full"] - } - >( - trx, - `-- sql + >( + trx, + `-- sql SELECT ${oldChartFieldList}, charts.isInheritanceEnabled, chart_configs.full AS config FROM charts JOIN chart_configs ON chart_configs.id = charts.configId @@ -251,297 +258,374 @@ getRouteWithROTransaction( WHERE cd.variableId = ? GROUP BY charts.id `, - [variableId] - ) - - // check for parent indicators - const charts = rawCharts.map((chart) => { - const parentIndicatorId = getParentVariableIdFromChartConfig( - parseChartConfig(chart.config) - ) - const hasParentIndicator = parentIndicatorId !== undefined - return omit({ ...chart, hasParentIndicator }, "config") - }) - - await assignTagsForCharts(trx, charts) + [variableId] + ) - const variableWithConfigs = await getGrapherConfigsForVariable( - trx, - variableId + // check for parent indicators + const charts = rawCharts.map((chart) => { + const parentIndicatorId = getParentVariableIdFromChartConfig( + parseChartConfig(chart.config) ) - const grapherConfigETL = variableWithConfigs?.etl?.patchConfig - const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig - const mergedGrapherConfig = - variableWithConfigs?.admin?.fullConfig ?? - variableWithConfigs?.etl?.fullConfig - - // add the variable's display field to the merged grapher config - if (mergedGrapherConfig) { - const [varDims, otherDims] = lodash.partition( - mergedGrapherConfig.dimensions ?? [], - (dim) => dim.variableId === variableId - ) - const varDimsWithDisplay = varDims.map((dim) => ({ - display: variable.display, - ...dim, - })) - mergedGrapherConfig.dimensions = [ - ...varDimsWithDisplay, - ...otherDims, - ] - } + const hasParentIndicator = parentIndicatorId !== undefined + return omit({ ...chart, hasParentIndicator }, "config") + }) - const variableWithCharts: OwidVariableWithSource & { - charts: Record<string, any> - grapherConfig: GrapherInterface | undefined - grapherConfigETL: GrapherInterface | undefined - grapherConfigAdmin: GrapherInterface | undefined - } = { - ...variable, - charts, - grapherConfig: mergedGrapherConfig, - grapherConfigETL, - grapherConfigAdmin, - } + await assignTagsForCharts(trx, charts) - return { - variable: variableWithCharts, - } /*, vardata: await getVariableData([variableId]) }*/ + const variableWithConfigs = await getGrapherConfigsForVariable( + trx, + variableId + ) + const grapherConfigETL = variableWithConfigs?.etl?.patchConfig + const grapherConfigAdmin = variableWithConfigs?.admin?.patchConfig + const mergedGrapherConfig = + variableWithConfigs?.admin?.fullConfig ?? + variableWithConfigs?.etl?.fullConfig + + // add the variable's display field to the merged grapher config + if (mergedGrapherConfig) { + const [varDims, otherDims] = lodash.partition( + mergedGrapherConfig.dimensions ?? [], + (dim) => dim.variableId === variableId + ) + const varDimsWithDisplay = varDims.map((dim) => ({ + display: variable.display, + ...dim, + })) + mergedGrapherConfig.dimensions = [...varDimsWithDisplay, ...otherDims] } -) -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } + const variableWithCharts: OwidVariableWithSource & { + charts: Record<string, any> + grapherConfig: GrapherInterface | undefined + grapherConfigETL: GrapherInterface | undefined + grapherConfigAdmin: GrapherInterface | undefined + } = { + ...variable, + charts, + grapherConfig: mergedGrapherConfig, + grapherConfigETL, + grapherConfigAdmin, + } - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) + return { + variable: variableWithCharts, + } /*, vardata: await getVariableData([variableId]) }*/ +} + +export async function putVariablesVariableIdGrapherConfigETL( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), } + } - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigETLOfVariable(trx, variable, validConfig) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigETLOfVariable(trx, variable, validConfig) - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true, savedPatch } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) } -) -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) + return { success: true, savedPatch } +} - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } +export async function deleteVariablesVariableIdGrapherConfigETL( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) - // no-op if the variable doesn't have an ETL config - if (!variable.etl) return { success: true } + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - const now = new Date() + // no-op if the variable doesn't have an ETL config + if (!variable.etl) return { success: true } - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql UPDATE variables SET grapherConfigIdETL = NULL WHERE id = ? `, - [variableId] - ) + [variableId] + ) - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql DELETE FROM chart_configs WHERE id = ? `, - [variable.etl.configId] - ) - - // update admin config if there is one - if (variable.admin) { - await updateExistingFullConfig(trx, { - configId: variable.admin.configId, - config: variable.admin.patchConfig, - updatedAt: now, - }) - } + [variable.etl.configId] + ) - const updates = { - patchConfigAdmin: variable.admin?.patchConfig, + // update admin config if there is one + if (variable.admin) { + await updateExistingFullConfig(trx, { + configId: variable.admin.configId, + config: variable.admin.patchConfig, updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( + }) + } + + const updates = { + patchConfigAdmin: variable.admin?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( trx, variableId, updates ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating ETL config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating ETL config for variable ${variableId}` + ) } -) -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - - let validConfig: GrapherInterface - try { - validConfig = migrateGrapherConfigToLatestVersion(req.body) - } catch (err) { - return { - success: false, - error: String(err), - } - } + return { success: true } +} - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) +export async function putVariablesVariableIdGrapherConfigAdmin( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) + + let validConfig: GrapherInterface + try { + validConfig = migrateGrapherConfigToLatestVersion(req.body) + } catch (err) { + return { + success: false, + error: String(err), } + } - const { savedPatch, updatedCharts, updatedMultiDimViews } = - await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] + const { savedPatch, updatedCharts, updatedMultiDimViews } = + await updateGrapherConfigAdminOfVariable(trx, variable, validConfig) - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true, savedPatch } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) } -) -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) + return { success: true, savedPatch } +} - const variable = await getGrapherConfigsForVariable(trx, variableId) - if (!variable) { - throw new JsonError(`Variable with id ${variableId} not found`, 500) - } +export async function deleteVariablesVariableIdGrapherConfigAdmin( + req: Request, + res: e.Response<any, Record<string, any>>, + trx: db.KnexReadWriteTransaction +) { + const variableId = expectInt(req.params.variableId) - // no-op if the variable doesn't have an admin-authored config - if (!variable.admin) return { success: true } + const variable = await getGrapherConfigsForVariable(trx, variableId) + if (!variable) { + throw new JsonError(`Variable with id ${variableId} not found`, 500) + } - const now = new Date() + // no-op if the variable doesn't have an admin-authored config + if (!variable.admin) return { success: true } - // remove reference in the variables table - await db.knexRaw( - trx, - `-- sql + const now = new Date() + + // remove reference in the variables table + await db.knexRaw( + trx, + `-- sql UPDATE variables SET grapherConfigIdAdmin = NULL WHERE id = ? `, - [variableId] - ) + [variableId] + ) - // delete row in the chart_configs table - await db.knexRaw( - trx, - `-- sql + // delete row in the chart_configs table + await db.knexRaw( + trx, + `-- sql DELETE FROM chart_configs WHERE id = ? `, - [variable.admin.configId] - ) + [variable.admin.configId] + ) - const updates = { - patchConfigETL: variable.etl?.patchConfig, - updatedAt: now, - } - const updatedCharts = await updateAllChartsThatInheritFromIndicator( + const updates = { + patchConfigETL: variable.etl?.patchConfig, + updatedAt: now, + } + const updatedCharts = await updateAllChartsThatInheritFromIndicator( + trx, + variableId, + updates + ) + const updatedMultiDimViews = + await updateAllMultiDimViewsThatInheritFromIndicator( trx, variableId, updates ) - const updatedMultiDimViews = - await updateAllMultiDimViewsThatInheritFromIndicator( - trx, - variableId, - updates - ) - await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) - const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - - if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { - await triggerStaticBuild( - res.locals.user, - `Updating admin-authored config for variable ${variableId}` - ) - } + await updateGrapherConfigsInR2(trx, updatedCharts, updatedMultiDimViews) + const allUpdatedConfigs = [...updatedCharts, ...updatedMultiDimViews] - return { success: true } + if (allUpdatedConfigs.some(({ isPublished }) => isPublished)) { + await triggerStaticBuild( + res.locals.user, + `Updating admin-authored config for variable ${variableId}` + ) } + + return { success: true } +} + +export async function getVariablesVariableIdChartsJson( + req: Request, + _res: e.Response<any, Record<string, any>>, + trx: db.KnexReadonlyTransaction +) { + const variableId = expectInt(req.params.variableId) + const charts = await getAllChartsForIndicator(trx, variableId) + return charts.map((chart) => ({ + id: chart.chartId, + title: chart.config.title, + variantName: chart.config.variantName, + isChild: chart.isChild, + isInheritanceEnabled: chart.isInheritanceEnabled, + isPublished: chart.isPublished, + })) +} + +getRouteWithROTransaction( + apiRouter, + "/editorData/variables.json", + getEditorVariablesJson +) + +getRouteWithROTransaction( + apiRouter, + "/data/variables/data/:variableStr.json", + getVariableDataJson +) + +getRouteWithROTransaction( + apiRouter, + "/data/variables/metadata/:variableStr.json", + getVariableMetadataJson +) + +getRouteWithROTransaction(apiRouter, "/variables.json", getVariablesJson) + +getRouteWithROTransaction( + apiRouter, + "/variables.usages.json", + getVariablesUsagesJson +) + +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigETL/:variableId.patchConfig.json", + getVariablesGrapherConfigETLPatchConfigJson +) + +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigAdmin/:variableId.patchConfig.json", + getVariablesGrapherConfigAdminPatchConfigJson +) + +getRouteWithROTransaction( + apiRouter, + "/variables/mergedGrapherConfig/:variableId.json", + getVariablesMergedGrapherConfigJson +) + +// Used in VariableEditPage +getRouteWithROTransaction( + apiRouter, + "/variables/:variableId.json", + getVariablesVariableIdJson +) + +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + putVariablesVariableIdGrapherConfigETL +) + +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + deleteVariablesVariableIdGrapherConfigETL +) + +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + putVariablesVariableIdGrapherConfigAdmin +) + +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + deleteVariablesVariableIdGrapherConfigAdmin ) getRouteWithROTransaction( apiRouter, "/variables/:variableId/charts.json", - async (req, res, trx) => { - const variableId = expectInt(req.params.variableId) - const charts = await getAllChartsForIndicator(trx, variableId) - return charts.map((chart) => ({ - id: chart.chartId, - title: chart.config.title, - variantName: chart.config.variantName, - isChild: chart.isChild, - isInheritanceEnabled: chart.isInheritanceEnabled, - isPublished: chart.isPublished, - })) - } + getVariablesVariableIdChartsJson ) From 9fd9512990d4957cdd6bc2c9229beab267f9ffd4 Mon Sep 17 00:00:00 2001 From: Daniel Bachler <daniel@danielbachler.de> Date: Fri, 20 Dec 2024 16:10:52 +0100 Subject: [PATCH 38/39] =?UTF-8?q?=F0=9F=94=A8=20finish=20refactoring=20of?= =?UTF-8?q?=20api=20routes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRouter.ts | 429 +++++++++++++++++++++++ adminSiteServer/apiRoutes/bulkUpdates.ts | 24 -- adminSiteServer/apiRoutes/chartViews.ts | 17 - adminSiteServer/apiRoutes/charts.ts | 79 +---- adminSiteServer/apiRoutes/datasets.ts | 23 -- adminSiteServer/apiRoutes/explorer.ts | 13 - adminSiteServer/apiRoutes/gdocs.ts | 22 -- adminSiteServer/apiRoutes/images.ts | 26 -- adminSiteServer/apiRoutes/mdims.ts | 6 - adminSiteServer/apiRoutes/misc.ts | 20 -- adminSiteServer/apiRoutes/posts.ts | 27 -- adminSiteServer/apiRoutes/redirects.ts | 38 -- adminSiteServer/apiRoutes/routeUtils.ts | 7 - adminSiteServer/apiRoutes/suggest.ts | 15 - adminSiteServer/apiRoutes/tagGraph.ts | 13 - adminSiteServer/apiRoutes/tags.ts | 13 - adminSiteServer/apiRoutes/users.ts | 29 -- adminSiteServer/apiRoutes/variables.ts | 89 ----- 18 files changed, 442 insertions(+), 448 deletions(-) diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 48ae2b306e..90afc08798 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -1,6 +1,435 @@ /* eslint @typescript-eslint/no-unused-vars: [ "warn", { argsIgnorePattern: "^(res|req)$" } ] */ +import { TaggableType } from "@ourworldindata/types" +import { DeployQueueServer } from "../baker/DeployQueueServer.js" +import { + updateVariableAnnotations, + getChartBulkUpdate, + updateBulkChartConfigs, + getVariableAnnotations, +} from "./apiRoutes/bulkUpdates.js" +import { + getChartViews, + getChartViewById, + createChartView, + updateChartView, + deleteChartView, +} from "./apiRoutes/chartViews.js" +import { + getDatasets, + getDataset, + updateDataset, + setArchived, + setTags, + deleteDataset, + republishCharts, +} from "./apiRoutes/datasets.js" +import { addExplorerTags, deleteExplorerTags } from "./apiRoutes/explorer.js" +import { + getAllGdocIndexItems, + getIndividualGdoc, + createOrUpdateGdoc, + deleteGdoc, + setGdocTags, +} from "./apiRoutes/gdocs.js" +import { + getImagesHandler, + postImageHandler, + putImageHandler, + patchImageHandler, + deleteImageHandler, + getImageUsageHandler, +} from "./apiRoutes/images.js" +import { handleMultiDimDataPageRequest } from "./apiRoutes/mdims.js" +import { + fetchAllWork, + fetchNamespaces, + fetchSourceById, +} from "./apiRoutes/misc.js" +import { + handleGetPostsJson, + handleSetTagsForPost, + handleGetPostById, + handleCreateGdoc, + handleUnlinkGdoc, +} from "./apiRoutes/posts.js" +import { + handleGetSiteRedirects, + handlePostNewSiteRedirect, + handleDeleteSiteRedirect, + handleGetRedirects, + handlePostNewChartRedirect, + handleDeleteChartRedirect, +} from "./apiRoutes/redirects.js" +import { triggerStaticBuild } from "./apiRoutes/routeUtils.js" +import { suggestGptTopics, suggestGptAltText } from "./apiRoutes/suggest.js" +import { + handleGetFlatTagGraph, + handlePostTagGraph, +} from "./apiRoutes/tagGraph.js" +import { + getTagById, + updateTag, + createTag, + getAllTags, + deleteTag, +} from "./apiRoutes/tags.js" +import { + getUsers, + getUserByIdHandler, + deleteUser, + updateUserHandler, + addUser, + addImageToUser, + removeUserImage, +} from "./apiRoutes/users.js" +import { + getEditorVariablesJson, + getVariableDataJson, + getVariableMetadataJson, + getVariablesJson, + getVariablesUsagesJson, + getVariablesGrapherConfigETLPatchConfigJson, + getVariablesGrapherConfigAdminPatchConfigJson, + getVariablesMergedGrapherConfigJson, + getVariablesVariableIdJson, + putVariablesVariableIdGrapherConfigETL, + deleteVariablesVariableIdGrapherConfigETL, + putVariablesVariableIdGrapherConfigAdmin, + deleteVariablesVariableIdGrapherConfigAdmin, + getVariablesVariableIdChartsJson, +} from "./apiRoutes/variables.js" import { FunctionalRouter } from "./FunctionalRouter.js" +import { + patchRouteWithRWTransaction, + getRouteWithROTransaction, + postRouteWithRWTransaction, + putRouteWithRWTransaction, + deleteRouteWithRWTransaction, + getRouteNonIdempotentWithRWTransaction, +} from "./functionalRouterHelpers.js" +import { + getChartsJson, + getChartsCsv, + getChartConfigJson, + getChartParentJson, + getChartPatchConfigJson, + getChartLogsJson, + getChartReferencesJson, + getChartRedirectsJson, + getChartPageviewsJson, + createChart, + setChartTagsHandler, + updateChart, + deleteChart, +} from "./apiRoutes/charts.js" const apiRouter = new FunctionalRouter() + +// Bulk chart update routes +patchRouteWithRWTransaction( + apiRouter, + "/variable-annotations", + updateVariableAnnotations +) +getRouteWithROTransaction(apiRouter, "/chart-bulk-update", getChartBulkUpdate) +patchRouteWithRWTransaction( + apiRouter, + "/chart-bulk-update", + updateBulkChartConfigs +) +getRouteWithROTransaction( + apiRouter, + "/variable-annotations", + getVariableAnnotations +) + +// Chart routes +getRouteWithROTransaction(apiRouter, "/charts.json", getChartsJson) +getRouteWithROTransaction(apiRouter, "/charts.csv", getChartsCsv) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.config.json", + getChartConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.parent.json", + getChartParentJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.patchConfig.json", + getChartPatchConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.logs.json", + getChartLogsJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.references.json", + getChartReferencesJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.redirects.json", + getChartRedirectsJson +) +getRouteWithROTransaction( + apiRouter, + "/charts/:chartId.pageviews.json", + getChartPageviewsJson +) +postRouteWithRWTransaction(apiRouter, "/charts", createChart) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/setTags", + setChartTagsHandler +) +putRouteWithRWTransaction(apiRouter, "/charts/:chartId", updateChart) +deleteRouteWithRWTransaction(apiRouter, "/charts/:chartId", deleteChart) + +// Chart view routes +getRouteWithROTransaction(apiRouter, "/chartViews", getChartViews) +getRouteWithROTransaction(apiRouter, "/chartViews/:id", getChartViewById) +postRouteWithRWTransaction(apiRouter, "/chartViews", createChartView) +putRouteWithRWTransaction(apiRouter, "/chartViews/:id", updateChartView) +deleteRouteWithRWTransaction(apiRouter, "/chartViews/:id", deleteChartView) + +// Dataset routes +getRouteWithROTransaction(apiRouter, "/datasets.json", getDatasets) +getRouteWithROTransaction(apiRouter, "/datasets/:datasetId.json", getDataset) +putRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", updateDataset) +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/setArchived", + setArchived +) +postRouteWithRWTransaction(apiRouter, "/datasets/:datasetId/setTags", setTags) +deleteRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", deleteDataset) +postRouteWithRWTransaction( + apiRouter, + "/datasets/:datasetId/charts", + republishCharts +) + +// explorer routes +postRouteWithRWTransaction(apiRouter, "/explorer/:slug/tags", addExplorerTags) +deleteRouteWithRWTransaction( + apiRouter, + "/explorer/:slug/tags", + deleteExplorerTags +) + +// Gdoc routes +getRouteWithROTransaction(apiRouter, "/gdocs", getAllGdocIndexItems) +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/gdocs/:id", + getIndividualGdoc +) +putRouteWithRWTransaction(apiRouter, "/gdocs/:id", createOrUpdateGdoc) +deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", deleteGdoc) +postRouteWithRWTransaction(apiRouter, "/gdocs/:gdocId/setTags", setGdocTags) + +// Images routes +getRouteNonIdempotentWithRWTransaction( + apiRouter, + "/images.json", + getImagesHandler +) +postRouteWithRWTransaction(apiRouter, "/images", postImageHandler) +putRouteWithRWTransaction(apiRouter, "/images/:id", putImageHandler) +// Update alt text via patch +patchRouteWithRWTransaction(apiRouter, "/images/:id", patchImageHandler) +deleteRouteWithRWTransaction(apiRouter, "/images/:id", deleteImageHandler) +getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler) + +// Mdim routes +putRouteWithRWTransaction( + apiRouter, + "/multi-dim/:slug", + handleMultiDimDataPageRequest +) + +// Misc routes +getRouteWithROTransaction(apiRouter, "/all-work", fetchAllWork) +getRouteWithROTransaction( + apiRouter, + "/editorData/namespaces.json", + fetchNamespaces +) +getRouteWithROTransaction(apiRouter, "/sources/:sourceId.json", fetchSourceById) + +// Wordpress posts routes +getRouteWithROTransaction(apiRouter, "/posts.json", handleGetPostsJson) +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/setTags", + handleSetTagsForPost +) +getRouteWithROTransaction(apiRouter, "/posts/:postId.json", handleGetPostById) +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/createGdoc", + handleCreateGdoc +) +postRouteWithRWTransaction( + apiRouter, + "/posts/:postId/unlinkGdoc", + handleUnlinkGdoc +) + +// Redirects routes +getRouteWithROTransaction( + apiRouter, + "/site-redirects.json", + handleGetSiteRedirects +) +postRouteWithRWTransaction( + apiRouter, + "/site-redirects/new", + handlePostNewSiteRedirect +) +deleteRouteWithRWTransaction( + apiRouter, + "/site-redirects/:id", + handleDeleteSiteRedirect +) +getRouteWithROTransaction(apiRouter, "/redirects.json", handleGetRedirects) +postRouteWithRWTransaction( + apiRouter, + "/charts/:chartId/redirects/new", + handlePostNewChartRedirect +) +deleteRouteWithRWTransaction( + apiRouter, + "/redirects/:id", + handleDeleteChartRedirect +) + +// GPT routes +getRouteWithROTransaction( + apiRouter, + `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, + suggestGptTopics +) +getRouteWithROTransaction( + apiRouter, + `/gpt/suggest-alt-text/:imageId`, + suggestGptAltText +) + +// Tag graph routes +getRouteWithROTransaction( + apiRouter, + "/flatTagGraph.json", + handleGetFlatTagGraph +) +postRouteWithRWTransaction(apiRouter, "/tagGraph", handlePostTagGraph) +getRouteWithROTransaction(apiRouter, "/tags/:tagId.json", getTagById) +putRouteWithRWTransaction(apiRouter, "/tags/:tagId", updateTag) +postRouteWithRWTransaction(apiRouter, "/tags/new", createTag) +getRouteWithROTransaction(apiRouter, "/tags.json", getAllTags) +deleteRouteWithRWTransaction(apiRouter, "/tags/:tagId/delete", deleteTag) + +// User routes +getRouteWithROTransaction(apiRouter, "/users.json", getUsers) +getRouteWithROTransaction(apiRouter, "/users/:userId.json", getUserByIdHandler) +deleteRouteWithRWTransaction(apiRouter, "/users/:userId", deleteUser) +putRouteWithRWTransaction(apiRouter, "/users/:userId", updateUserHandler) +postRouteWithRWTransaction(apiRouter, "/users/add", addUser) +postRouteWithRWTransaction( + apiRouter, + "/users/:userId/images/:imageId", + addImageToUser +) +deleteRouteWithRWTransaction( + apiRouter, + "/users/:userId/images/:imageId", + removeUserImage +) + +// Variable routes +getRouteWithROTransaction( + apiRouter, + "/editorData/variables.json", + getEditorVariablesJson +) +getRouteWithROTransaction( + apiRouter, + "/data/variables/data/:variableStr.json", + getVariableDataJson +) +getRouteWithROTransaction( + apiRouter, + "/data/variables/metadata/:variableStr.json", + getVariableMetadataJson +) +getRouteWithROTransaction(apiRouter, "/variables.json", getVariablesJson) +getRouteWithROTransaction( + apiRouter, + "/variables.usages.json", + getVariablesUsagesJson +) +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigETL/:variableId.patchConfig.json", + getVariablesGrapherConfigETLPatchConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/variables/grapherConfigAdmin/:variableId.patchConfig.json", + getVariablesGrapherConfigAdminPatchConfigJson +) +getRouteWithROTransaction( + apiRouter, + "/variables/mergedGrapherConfig/:variableId.json", + getVariablesMergedGrapherConfigJson +) +// Used in VariableEditPage +getRouteWithROTransaction( + apiRouter, + "/variables/:variableId.json", + getVariablesVariableIdJson +) +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + putVariablesVariableIdGrapherConfigETL +) +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigETL", + deleteVariablesVariableIdGrapherConfigETL +) +// inserts a new config or updates an existing one +putRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + putVariablesVariableIdGrapherConfigAdmin +) +deleteRouteWithRWTransaction( + apiRouter, + "/variables/:variableId/grapherConfigAdmin", + deleteVariablesVariableIdGrapherConfigAdmin +) +getRouteWithROTransaction( + apiRouter, + "/variables/:variableId/charts.json", + getVariablesVariableIdChartsJson +) + +// Deploy helpers +apiRouter.get("/deploys.json", async () => ({ + deploys: await new DeployQueueServer().getDeploys(), +})) + +apiRouter.put("/deploy", async (req, res) => { + return triggerStaticBuild(res.locals.user, "Manually triggered deploy") +}) + export { apiRouter } diff --git a/adminSiteServer/apiRoutes/bulkUpdates.ts b/adminSiteServer/apiRoutes/bulkUpdates.ts index 364146238c..82aa5b598c 100644 --- a/adminSiteServer/apiRoutes/bulkUpdates.ts +++ b/adminSiteServer/apiRoutes/bulkUpdates.ts @@ -22,14 +22,9 @@ import { getGrapherConfigsForVariable, updateGrapherConfigAdminOfVariable, } from "../../db/model/Variable.js" -import { - getRouteWithROTransaction, - patchRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import { saveGrapher } from "./charts.js" import * as db from "../../db/db.js" import * as lodash from "lodash" -import { apiRouter } from "../apiRouter.js" import { Request } from "../authentication.js" import e from "express" @@ -245,22 +240,3 @@ export async function updateVariableAnnotations( return { success: true } } - -patchRouteWithRWTransaction( - apiRouter, - "/variable-annotations", - updateVariableAnnotations -) - -getRouteWithROTransaction(apiRouter, "/chart-bulk-update", getChartBulkUpdate) - -patchRouteWithRWTransaction( - apiRouter, - "/chart-bulk-update", - updateBulkChartConfigs -) -getRouteWithROTransaction( - apiRouter, - "/variable-annotations", - getVariableAnnotations -) diff --git a/adminSiteServer/apiRoutes/chartViews.ts b/adminSiteServer/apiRoutes/chartViews.ts index 4eda8ff3aa..1bb86557bd 100644 --- a/adminSiteServer/apiRoutes/chartViews.ts +++ b/adminSiteServer/apiRoutes/chartViews.ts @@ -19,18 +19,11 @@ import { diffGrapherConfigs, mergeGrapherConfigs } from "@ourworldindata/utils" import { omit, pick } from "lodash" import { ApiChartViewOverview } from "../../adminShared/AdminTypes.js" import { expectInt } from "../../serverUtils/serverUtil.js" -import { apiRouter } from "../apiRouter.js" import { saveNewChartConfigInDbAndR2, updateChartConfigInDbAndR2, } from "../chartConfigHelpers.js" import { deleteGrapherConfigFromR2ByUUID } from "../chartConfigR2Helpers.js" -import { - getRouteWithROTransaction, - postRouteWithRWTransaction, - putRouteWithRWTransaction, - deleteRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import { expectChartById } from "./charts.js" @@ -288,13 +281,3 @@ export async function deleteChartView( return { success: true } } - -getRouteWithROTransaction(apiRouter, "/chartViews", getChartViews) - -getRouteWithROTransaction(apiRouter, "/chartViews/:id", getChartViewById) - -postRouteWithRWTransaction(apiRouter, "/chartViews", createChartView) - -putRouteWithRWTransaction(apiRouter, "/chartViews/:id", updateChartView) - -deleteRouteWithRWTransaction(apiRouter, "/chartViews/:id", deleteChartView) diff --git a/adminSiteServer/apiRoutes/charts.ts b/adminSiteServer/apiRoutes/charts.ts index 0ab2670cdd..2b19a204a8 100644 --- a/adminSiteServer/apiRoutes/charts.ts +++ b/adminSiteServer/apiRoutes/charts.ts @@ -45,7 +45,6 @@ import { BAKED_BASE_URL, ADMIN_BASE_URL, } from "../../settings/clientSettings.js" -import { apiRouter } from "../apiRouter.js" import { retrieveChartConfigFromDbAndSaveToR2, updateChartConfigInDbAndR2, @@ -55,12 +54,6 @@ import { deleteGrapherConfigFromR2ByUUID, saveGrapherConfigToR2ByUUID, } from "../chartConfigR2Helpers.js" -import { - deleteRouteWithRWTransaction, - getRouteWithROTransaction, - postRouteWithRWTransaction, - putRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" import { getLogsByChartId } from "../getLogsByChartId.js" @@ -503,7 +496,7 @@ export async function updateGrapherConfigsInR2( } } -async function getChartsJson( +export async function getChartsJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -529,7 +522,7 @@ async function getChartsJson( return { charts } } -async function getChartsCsv( +export async function getChartsCsv( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -589,7 +582,7 @@ async function getChartsCsv( return csv } -async function getChartConfigJson( +export async function getChartConfigJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -597,7 +590,7 @@ async function getChartConfigJson( return expectChartById(trx, req.params.chartId) } -async function getChartParentJson( +export async function getChartParentJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -615,7 +608,7 @@ async function getChartParentJson( }) } -async function getChartPatchConfigJson( +export async function getChartPatchConfigJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -625,7 +618,7 @@ async function getChartPatchConfigJson( return config } -async function getChartLogsJson( +export async function getChartLogsJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -638,7 +631,7 @@ async function getChartLogsJson( } } -async function getChartReferencesJson( +export async function getChartReferencesJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -652,7 +645,7 @@ async function getChartReferencesJson( return references } -async function getChartRedirectsJson( +export async function getChartRedirectsJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -665,7 +658,7 @@ async function getChartRedirectsJson( } } -async function getChartPageviewsJson( +export async function getChartPageviewsJson( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadonlyTransaction @@ -692,7 +685,7 @@ async function getChartPageviewsJson( } } -async function createChart( +export async function createChart( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadWriteTransaction @@ -715,7 +708,7 @@ async function createChart( } } -async function setChartTagsHandler( +export async function setChartTagsHandler( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadWriteTransaction @@ -727,7 +720,7 @@ async function setChartTagsHandler( return { success: true } } -async function updateChart( +export async function updateChart( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadWriteTransaction @@ -762,7 +755,7 @@ async function updateChart( } } -async function deleteChart( +export async function deleteChart( req: Request, res: e.Response<any, Record<string, any>>, trx: db.KnexReadWriteTransaction @@ -814,49 +807,3 @@ async function deleteChart( return { success: true } } - -getRouteWithROTransaction(apiRouter, "/charts.json", getChartsJson) -getRouteWithROTransaction(apiRouter, "/charts.csv", getChartsCsv) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.config.json", - getChartConfigJson -) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.parent.json", - getChartParentJson -) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.patchConfig.json", - getChartPatchConfigJson -) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.logs.json", - getChartLogsJson -) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.references.json", - getChartReferencesJson -) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.redirects.json", - getChartRedirectsJson -) -getRouteWithROTransaction( - apiRouter, - "/charts/:chartId.pageviews.json", - getChartPageviewsJson -) -postRouteWithRWTransaction(apiRouter, "/charts", createChart) -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/setTags", - setChartTagsHandler -) -putRouteWithRWTransaction(apiRouter, "/charts/:chartId", updateChart) -deleteRouteWithRWTransaction(apiRouter, "/charts/:chartId", deleteChart) diff --git a/adminSiteServer/apiRoutes/datasets.ts b/adminSiteServer/apiRoutes/datasets.ts index d6bac477a2..fb490bc42e 100644 --- a/adminSiteServer/apiRoutes/datasets.ts +++ b/adminSiteServer/apiRoutes/datasets.ts @@ -14,13 +14,6 @@ import { import { getDatasetById, setTagsForDataset } from "../../db/model/Dataset.js" import { logErrorAndMaybeSendToBugsnag } from "../../serverUtils/errorLog.js" import { expectInt } from "../../serverUtils/serverUtil.js" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - putRouteWithRWTransaction, - postRouteWithRWTransaction, - deleteRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import { syncDatasetToGitRepo, removeDatasetFromGitRepo, @@ -413,19 +406,3 @@ export async function republishCharts( return { success: true } } - -getRouteWithROTransaction(apiRouter, "/datasets.json", getDatasets) -getRouteWithROTransaction(apiRouter, "/datasets/:datasetId.json", getDataset) -putRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", updateDataset) -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/setArchived", - setArchived -) -postRouteWithRWTransaction(apiRouter, "/datasets/:datasetId/setTags", setTags) -deleteRouteWithRWTransaction(apiRouter, "/datasets/:datasetId", deleteDataset) -postRouteWithRWTransaction( - apiRouter, - "/datasets/:datasetId/charts", - republishCharts -) diff --git a/adminSiteServer/apiRoutes/explorer.ts b/adminSiteServer/apiRoutes/explorer.ts index f0228fafff..44b9caf630 100644 --- a/adminSiteServer/apiRoutes/explorer.ts +++ b/adminSiteServer/apiRoutes/explorer.ts @@ -1,9 +1,4 @@ import { JsonError } from "@ourworldindata/types" -import { apiRouter } from "../apiRouter.js" -import { - postRouteWithRWTransaction, - deleteRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import { Request } from "express" import * as e from "express" @@ -36,11 +31,3 @@ export async function deleteExplorerTags( await trx.table("explorer_tags").where({ explorerSlug: slug }).delete() return { success: true } } - -postRouteWithRWTransaction(apiRouter, "/explorer/:slug/tags", addExplorerTags) - -deleteRouteWithRWTransaction( - apiRouter, - "/explorer/:slug/tags", - deleteExplorerTags -) diff --git a/adminSiteServer/apiRoutes/gdocs.ts b/adminSiteServer/apiRoutes/gdocs.ts index ed96cb2417..fbeb412e0d 100644 --- a/adminSiteServer/apiRoutes/gdocs.ts +++ b/adminSiteServer/apiRoutes/gdocs.ts @@ -42,14 +42,6 @@ import { } from "../../db/model/Gdoc/GdocFactory.js" import { GdocHomepage } from "../../db/model/Gdoc/GdocHomepage.js" import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - getRouteNonIdempotentWithRWTransaction, - putRouteWithRWTransaction, - deleteRouteWithRWTransaction, - postRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import { triggerStaticBuild, enqueueLightningChange } from "./routeUtils.js" import * as db from "../../db/db.js" import * as lodash from "lodash" @@ -295,17 +287,3 @@ export async function setGdocTags( return { success: true } } - -getRouteWithROTransaction(apiRouter, "/gdocs", getAllGdocIndexItems) - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/gdocs/:id", - getIndividualGdoc -) - -putRouteWithRWTransaction(apiRouter, "/gdocs/:id", createOrUpdateGdoc) - -deleteRouteWithRWTransaction(apiRouter, "/gdocs/:id", deleteGdoc) - -postRouteWithRWTransaction(apiRouter, "/gdocs/:gdocId/setTags", setGdocTags) diff --git a/adminSiteServer/apiRoutes/images.ts b/adminSiteServer/apiRoutes/images.ts index b8b3b3db07..7fc71c08e7 100644 --- a/adminSiteServer/apiRoutes/images.ts +++ b/adminSiteServer/apiRoutes/images.ts @@ -1,14 +1,5 @@ import { DbEnrichedImage, JsonError } from "@ourworldindata/types" import pMap from "p-map" -import { apiRouter } from "../apiRouter.js" -import { - getRouteNonIdempotentWithRWTransaction, - postRouteWithRWTransaction, - putRouteWithRWTransaction, - patchRouteWithRWTransaction, - deleteRouteWithRWTransaction, - getRouteWithROTransaction, -} from "../functionalRouterHelpers.js" import { validateImagePayload, processImageContent, @@ -270,20 +261,3 @@ export async function getImageUsageHandler( usage, } } - -getRouteNonIdempotentWithRWTransaction( - apiRouter, - "/images.json", - getImagesHandler -) - -postRouteWithRWTransaction(apiRouter, "/images", postImageHandler) - -putRouteWithRWTransaction(apiRouter, "/images/:id", putImageHandler) - -// Update alt text via patch -patchRouteWithRWTransaction(apiRouter, "/images/:id", patchImageHandler) - -deleteRouteWithRWTransaction(apiRouter, "/images/:id", deleteImageHandler) - -getRouteWithROTransaction(apiRouter, "/images/usage", getImageUsageHandler) diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts index 34a05595d2..39ec7ab35c 100644 --- a/adminSiteServer/apiRoutes/mdims.ts +++ b/adminSiteServer/apiRoutes/mdims.ts @@ -35,9 +35,3 @@ export async function handleMultiDimDataPageRequest( } return { success: true, id } } - -putRouteWithRWTransaction( - apiRouter, - "/multi-dim/:slug", - handleMultiDimDataPageRequest -) diff --git a/adminSiteServer/apiRoutes/misc.ts b/adminSiteServer/apiRoutes/misc.ts index 6d6d448dd9..f31d6c2d6f 100644 --- a/adminSiteServer/apiRoutes/misc.ts +++ b/adminSiteServer/apiRoutes/misc.ts @@ -4,15 +4,11 @@ // [.secondary] section of the {.research-and-writing} block of author pages import { DbRawPostGdoc, JsonError } from "@ourworldindata/types" -import { apiRouter } from "../apiRouter.js" -import { getRouteWithROTransaction } from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" import path from "path" -import { DeployQueueServer } from "../../baker/DeployQueueServer.js" import { expectInt } from "../../serverUtils/serverUtil.js" -import { triggerStaticBuild } from "./routeUtils.js" import { Request } from "../authentication.js" import e from "express" // using the alternate template, which highlights topics rather than articles. @@ -179,19 +175,3 @@ export async function fetchSourceById( return { source: source } } - -apiRouter.get("/deploys.json", async () => ({ - deploys: await new DeployQueueServer().getDeploys(), -})) - -apiRouter.put("/deploy", async (req, res) => { - return triggerStaticBuild(res.locals.user, "Manually triggered deploy") -}) - -getRouteWithROTransaction(apiRouter, "/all-work", fetchAllWork) -getRouteWithROTransaction( - apiRouter, - "/editorData/namespaces.json", - fetchNamespaces -) -getRouteWithROTransaction(apiRouter, "/sources/:sourceId.json", fetchSourceById) diff --git a/adminSiteServer/apiRoutes/posts.ts b/adminSiteServer/apiRoutes/posts.ts index efd31d99db..f34714fc30 100644 --- a/adminSiteServer/apiRoutes/posts.ts +++ b/adminSiteServer/apiRoutes/posts.ts @@ -13,11 +13,6 @@ import { upsertGdoc, setTagsForGdoc } from "../../db/model/Gdoc/GdocFactory.js" import { GdocPost } from "../../db/model/Gdoc/GdocPost.js" import { setTagsForPost, getTagsByPostId } from "../../db/model/Post.js" import { expectInt } from "../../serverUtils/serverUtil.js" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - postRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import { Request } from "../authentication.js" import e from "express" @@ -218,25 +213,3 @@ export async function handleUnlinkGdoc( return { success: true } } - -getRouteWithROTransaction(apiRouter, "/posts.json", handleGetPostsJson) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/setTags", - handleSetTagsForPost -) - -getRouteWithROTransaction(apiRouter, "/posts/:postId.json", handleGetPostById) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/createGdoc", - handleCreateGdoc -) - -postRouteWithRWTransaction( - apiRouter, - "/posts/:postId/unlinkGdoc", - handleUnlinkGdoc -) diff --git a/adminSiteServer/apiRoutes/redirects.ts b/adminSiteServer/apiRoutes/redirects.ts index 00f8971b07..44452fe026 100644 --- a/adminSiteServer/apiRoutes/redirects.ts +++ b/adminSiteServer/apiRoutes/redirects.ts @@ -6,12 +6,6 @@ import { getRedirectById, } from "../../db/model/Redirect.js" import { expectInt } from "../../serverUtils/serverUtil.js" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - postRouteWithRWTransaction, - deleteRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import { triggerStaticBuild } from "./routeUtils.js" import * as db from "../../db/db.js" import { Request } from "../authentication.js" @@ -151,35 +145,3 @@ export async function handleDeleteChartRedirect( return { success: true } } - -getRouteWithROTransaction( - apiRouter, - "/site-redirects.json", - handleGetSiteRedirects -) - -postRouteWithRWTransaction( - apiRouter, - "/site-redirects/new", - handlePostNewSiteRedirect -) - -deleteRouteWithRWTransaction( - apiRouter, - "/site-redirects/:id", - handleDeleteSiteRedirect -) - -getRouteWithROTransaction(apiRouter, "/redirects.json", handleGetRedirects) - -postRouteWithRWTransaction( - apiRouter, - "/charts/:chartId/redirects/new", - handlePostNewChartRedirect -) - -deleteRouteWithRWTransaction( - apiRouter, - "/redirects/:id", - handleDeleteChartRedirect -) diff --git a/adminSiteServer/apiRoutes/routeUtils.ts b/adminSiteServer/apiRoutes/routeUtils.ts index c9a8bbc908..0e647f290c 100644 --- a/adminSiteServer/apiRoutes/routeUtils.ts +++ b/adminSiteServer/apiRoutes/routeUtils.ts @@ -1,13 +1,6 @@ import { DbPlainUser } from "@ourworldindata/types" import { DeployQueueServer } from "../../baker/DeployQueueServer.js" import { BAKE_ON_CHANGE } from "../../settings/serverSettings.js" -import { References } from "../../adminSiteClient/AbstractChartEditor.js" -import { ChartViewMinimalInformation } from "../../adminSiteClient/ChartEditor.js" -import * as db from "../../db/db.js" -import { - getWordpressPostReferencesByChartId, - getGdocsPostReferencesByChartId, -} from "../../db/model/Post.js" // Call this to trigger build and deployment of static charts on change export const triggerStaticBuild = async ( diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts index 657d0b6b1f..4a294d4328 100644 --- a/adminSiteServer/apiRoutes/suggest.ts +++ b/adminSiteServer/apiRoutes/suggest.ts @@ -1,5 +1,4 @@ import { - TaggableType, DbChartTagJoin, JsonError, DbEnrichedImage, @@ -7,8 +6,6 @@ import { import { parseIntOrUndefined } from "@ourworldindata/utils" import { getGptTopicSuggestions } from "../../db/model/Chart.js" import { CLOUDFLARE_IMAGES_URL } from "../../settings/clientSettings.js" -import { apiRouter } from "../apiRouter.js" -import { getRouteWithROTransaction } from "../functionalRouterHelpers.js" import { fetchGptGeneratedAltText } from "../imagesHelpers.js" import * as db from "../../db/db.js" import e from "express" @@ -65,15 +62,3 @@ export async function suggestGptAltText( return { success: true, altText } } - -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-topics/${TaggableType.Charts}/:chartId.json`, - suggestGptTopics -) - -getRouteWithROTransaction( - apiRouter, - `/gpt/suggest-alt-text/:imageId`, - suggestGptAltText -) diff --git a/adminSiteServer/apiRoutes/tagGraph.ts b/adminSiteServer/apiRoutes/tagGraph.ts index f4dfc8b7b2..bafeec6d51 100644 --- a/adminSiteServer/apiRoutes/tagGraph.ts +++ b/adminSiteServer/apiRoutes/tagGraph.ts @@ -1,10 +1,5 @@ import { JsonError, FlatTagGraph } from "@ourworldindata/types" import { checkIsPlainObjectWithGuard } from "@ourworldindata/utils" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - postRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" import { Request } from "../authentication.js" @@ -65,11 +60,3 @@ export async function handlePostTagGraph( await db.updateTagGraph(trx, tagGraph) res.send({ success: true }) } - -getRouteWithROTransaction( - apiRouter, - "/flatTagGraph.json", - handleGetFlatTagGraph -) - -postRouteWithRWTransaction(apiRouter, "/tagGraph", handlePostTagGraph) diff --git a/adminSiteServer/apiRoutes/tags.ts b/adminSiteServer/apiRoutes/tags.ts index 578209cfe2..40bec68cec 100644 --- a/adminSiteServer/apiRoutes/tags.ts +++ b/adminSiteServer/apiRoutes/tags.ts @@ -12,13 +12,6 @@ import { } from "../../db/model/Chart.js" import { expectInt } from "../../serverUtils/serverUtil.js" import { UNCATEGORIZED_TAG_ID } from "../../settings/serverSettings.js" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - putRouteWithRWTransaction, - postRouteWithRWTransaction, - deleteRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import * as lodash from "lodash" import e from "express" @@ -266,9 +259,3 @@ export async function deleteTag( return { success: true } } - -getRouteWithROTransaction(apiRouter, "/tags/:tagId.json", getTagById) -putRouteWithRWTransaction(apiRouter, "/tags/:tagId", updateTag) -postRouteWithRWTransaction(apiRouter, "/tags/new", createTag) -getRouteWithROTransaction(apiRouter, "/tags.json", getAllTags) -deleteRouteWithRWTransaction(apiRouter, "/tags/:tagId/delete", deleteTag) diff --git a/adminSiteServer/apiRoutes/users.ts b/adminSiteServer/apiRoutes/users.ts index ea2016608e..e232fd15a3 100644 --- a/adminSiteServer/apiRoutes/users.ts +++ b/adminSiteServer/apiRoutes/users.ts @@ -3,13 +3,6 @@ import { parseIntOrUndefined } from "@ourworldindata/utils" import { pick } from "lodash" import { getUserById, updateUser, insertUser } from "../../db/model/User.js" import { expectInt } from "../../serverUtils/serverUtil.js" -import { apiRouter } from "../apiRouter.js" -import { - getRouteWithROTransaction, - deleteRouteWithRWTransaction, - putRouteWithRWTransaction, - postRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import { Request } from "../authentication.js" import e from "express" @@ -120,25 +113,3 @@ export async function removeUserImage( await trx("images").where({ id: imageId, userId }).update({ userId: null }) return { success: true } } - -getRouteWithROTransaction(apiRouter, "/users.json", getUsers) - -getRouteWithROTransaction(apiRouter, "/users/:userId.json", getUserByIdHandler) - -deleteRouteWithRWTransaction(apiRouter, "/users/:userId", deleteUser) - -putRouteWithRWTransaction(apiRouter, "/users/:userId", updateUserHandler) - -postRouteWithRWTransaction(apiRouter, "/users/add", addUser) - -postRouteWithRWTransaction( - apiRouter, - "/users/:userId/images/:imageId", - addImageToUser -) - -deleteRouteWithRWTransaction( - apiRouter, - "/users/:userId/images/:imageId", - removeUserImage -) diff --git a/adminSiteServer/apiRoutes/variables.ts b/adminSiteServer/apiRoutes/variables.ts index f8f21a65ab..d8f21c20ed 100644 --- a/adminSiteServer/apiRoutes/variables.ts +++ b/adminSiteServer/apiRoutes/variables.ts @@ -26,12 +26,6 @@ import { updateGrapherConfigETLOfVariable, } from "../../db/model/Variable.js" import { DATA_API_URL } from "../../settings/clientSettings.js" -import { apiRouter } from "../apiRouter.js" -import { - deleteRouteWithRWTransaction, - getRouteWithROTransaction, - putRouteWithRWTransaction, -} from "../functionalRouterHelpers.js" import * as db from "../../db/db.js" import { getParentVariableIdFromChartConfig, @@ -546,86 +540,3 @@ export async function getVariablesVariableIdChartsJson( isPublished: chart.isPublished, })) } - -getRouteWithROTransaction( - apiRouter, - "/editorData/variables.json", - getEditorVariablesJson -) - -getRouteWithROTransaction( - apiRouter, - "/data/variables/data/:variableStr.json", - getVariableDataJson -) - -getRouteWithROTransaction( - apiRouter, - "/data/variables/metadata/:variableStr.json", - getVariableMetadataJson -) - -getRouteWithROTransaction(apiRouter, "/variables.json", getVariablesJson) - -getRouteWithROTransaction( - apiRouter, - "/variables.usages.json", - getVariablesUsagesJson -) - -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigETL/:variableId.patchConfig.json", - getVariablesGrapherConfigETLPatchConfigJson -) - -getRouteWithROTransaction( - apiRouter, - "/variables/grapherConfigAdmin/:variableId.patchConfig.json", - getVariablesGrapherConfigAdminPatchConfigJson -) - -getRouteWithROTransaction( - apiRouter, - "/variables/mergedGrapherConfig/:variableId.json", - getVariablesMergedGrapherConfigJson -) - -// Used in VariableEditPage -getRouteWithROTransaction( - apiRouter, - "/variables/:variableId.json", - getVariablesVariableIdJson -) - -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - putVariablesVariableIdGrapherConfigETL -) - -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigETL", - deleteVariablesVariableIdGrapherConfigETL -) - -// inserts a new config or updates an existing one -putRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - putVariablesVariableIdGrapherConfigAdmin -) - -deleteRouteWithRWTransaction( - apiRouter, - "/variables/:variableId/grapherConfigAdmin", - deleteVariablesVariableIdGrapherConfigAdmin -) - -getRouteWithROTransaction( - apiRouter, - "/variables/:variableId/charts.json", - getVariablesVariableIdChartsJson -) From 50fb028407daf9e6f5a1dd1858e47bee993d5eaa Mon Sep 17 00:00:00 2001 From: Daniel Bachler <daniel@danielbachler.de> Date: Fri, 20 Dec 2024 16:20:47 +0100 Subject: [PATCH 39/39] =?UTF-8?q?=F0=9F=90=9D=20fix=20unused=20imports?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/apiRoutes/mdims.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/adminSiteServer/apiRoutes/mdims.ts b/adminSiteServer/apiRoutes/mdims.ts index 39ec7ab35c..7880662928 100644 --- a/adminSiteServer/apiRoutes/mdims.ts +++ b/adminSiteServer/apiRoutes/mdims.ts @@ -5,8 +5,6 @@ import { FEATURE_FLAGS, FeatureFlagFeature, } from "../../settings/clientSettings.js" -import { apiRouter } from "../apiRouter.js" -import { putRouteWithRWTransaction } from "../functionalRouterHelpers.js" import { createMultiDimConfig } from "../multiDim.js" import { triggerStaticBuild } from "./routeUtils.js" import { Request } from "../authentication.js"