Data pages: Update citations

baker/GrapherBaker.tsx

@@ -24,6 +24,8 @@
     FaqEntryData,
     FaqDictionary,
     partition,
+    PrimaryTopic,
+    sortBy,
 } from "@ourworldindata/utils"
 import {
     getRelatedArticles,
@@ -45,7 +47,7 @@
 import * as db from "../db/db.js"
 import { glob } from "glob"
 import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js"
-import { getPostBySlug } from "../db/model/Post.js"
+import { bySlug, getPostBySlug, parsePostAuthors } from "../db/model/Post.js"
 import { GrapherInterface } from "@ourworldindata/grapher"
 import workerpool from "workerpool"
 import ProgressBar from "progress"
@@ -60,11 +62,14 @@
     getDatapageJson,
     parseGdocContentFromAllowedLevelOneHeadings,
 } from "../datapage/Datapage.js"
+import { slugify_topic } from "../site/DataPageV2Content.js"
 import { ExplorerProgram } from "../explorer/ExplorerProgram.js"
 import { Image } from "../db/model/Image.js"
 import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
 
 import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js"
+import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
+import { getShortPageCitation } from "../site/gdocs/utils.js"
 
 /**
  *
@@ -322,6 +327,42 @@
         grapherConfigForVariable ?? {}
     )
 
+    const firstTopicTag = datapageData.topicTagsLinks?.[0]
+
+    if (firstTopicTag) {
+        const gdoc = await Gdoc.findOne({
+            where: {
+                slug: slugify_topic(firstTopicTag),
+            },
+            relations: ["tags"],
+        })
+        if (gdoc) {
+            const citation = getShortPageCitation(
+                gdoc.content.authors,
+                gdoc.content.title ?? "",
+                gdoc?.publishedAt
+            )
+            datapageData.primaryTopic = {
+                topicTag: firstTopicTag,
+                citation,
+            }
+        } else {
+            const post = await bySlug(slugify_topic(firstTopicTag))
+            if (post) {
+                const authors = parsePostAuthors(post.authors)
+                const citation = getShortPageCitation(
+                    authors,
+                    post.title,
+                    post.published_at
+                )
+                datapageData.primaryTopic = {
+                    topicTag: firstTopicTag,
+                    citation,
+                }
+            }
+        }
+    }
+
     // Get the charts this variable is being used in (aka "related charts")
     // and exclude the current chart to avoid duplicates
     datapageData.allCharts = await getRelatedChartsForVariable(

datapage/Datapage.ts

@@ -16,9 +16,9 @@ import {
     DataPageDataV2,
     OwidVariableWithSource,
     dayjs,
-    getAttributionFromVariable,
     gdocIdRegex,
     getETLPathComponents,
+    getAttributionFragmentsFromVariable,
 } from "@ourworldindata/utils"
 import { ExplorerProgram } from "../explorer/ExplorerProgram.js"
 import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
@@ -29,7 +29,7 @@ export const getDatapageDataV2 = async (
     partialGrapherConfig: GrapherInterface
 ): Promise<DataPageDataV2> => {
     {
-        const processingLevel = variableMetadata.processingLevel ?? "major"
+        const processingLevel = variableMetadata.processingLevel ?? "minor"
         const version =
             getETLPathComponents(variableMetadata.catalogPath ?? "")?.version ??
             ""
@@ -59,7 +59,7 @@ export const getDatapageDataV2 = async (
             attributionShort: variableMetadata.presentation?.attributionShort,
             titleVariant: variableMetadata.presentation?.titleVariant,
             topicTagsLinks: variableMetadata.presentation?.topicTagsLinks ?? [],
-            attribution: getAttributionFromVariable(variableMetadata),
+            attributions: getAttributionFragmentsFromVariable(variableMetadata),
             faqs: [],
             descriptionKey: variableMetadata.descriptionKey ?? [],
             descriptionProcessing: variableMetadata.descriptionProcessing,

db/migrateWpPostsToArchieMl.ts

@@ -4,7 +4,6 @@
 import {
     OwidGdocPublicationContext,
     OwidGdocInterface,
-    sortBy,
     OwidArticleBackportingStatistics,
     OwidGdocType,
     RelatedChart,
@@ -19,6 +18,7 @@
     adjustHeadingLevels,
 } from "./model/Gdoc/htmlToEnriched.js"
 import { getRelatedCharts } from "./wpdb.js"
+import { parsePostAuthors } from "./model/Post.js"
 
 // slugs from all the linear entries we want to migrate from @edomt
 const entries = new Set([
@@ -149,7 +149,7 @@
                 ? post.published_at.toLocaleDateString("en-US", options)
                 : ""
 
             const authors: { author: string; order: number }[] = JSON.parse(
                 post.authors
             )
 
@@ -162,9 +162,7 @@
                     title: post.title,
                     subtitle: post.excerpt,
                     excerpt: post.excerpt,
-                    authors: sortBy(authors, ["order"]).map(
-                        (author) => author.author
-                    ),
+                    authors: parsePostAuthors(post.authors),
                     dateline: dateline,
                     // TODO: this discards block level elements - those might be needed?
                     refs: undefined,

db/model/Post.ts

@@ -1,6 +1,6 @@
 import * as db from "../db.js"
 import { Knex } from "knex"
-import { PostRow } from "@ourworldindata/utils"
+import { PostRow, sortBy } from "@ourworldindata/utils"
 
 export const postsTable = "posts"
 
@@ -49,6 +49,14 @@ export const setTags = async (
 export const bySlug = async (slug: string): Promise<PostRow | undefined> =>
     (await db.knexTable("posts").where({ slug: slug }))[0]
 
+/** The authors field in the posts table is a json column that contains an array of
+    { order: 1, authors: "Max Mustermann" } like records. This function parses the
+    string and returns a simple string array of author names in the correct order  */
+export const parsePostAuthors = (authorsJson: string): string[] => {
+    const authors = JSON.parse(authorsJson)
+    return sortBy(authors, ["order"]).map((author) => author.author)
+}
+
 export const setTagsForPost = async (
     postId: number,
     tagIds: number[]

packages/@ourworldindata/grapher/src/core/Grapher.tsx

@@ -1535,7 +1535,8 @@ export class Grapher
 
         const uniqueAttributions = uniq(compact(attributions))
 
-        if (uniqueAttributions.length > 3) return "Multiple sources"
+        if (uniqueAttributions.length > 3)
+            return `${attributions[0]} and other sources`
 
         return uniqueAttributions.join("; ")
     }

packages/@ourworldindata/utils/src/Util.ts

@@ -1743,16 +1743,22 @@ export function getOriginAttributionFragments(
 export function getAttributionFromVariable(
     variable: OwidVariableWithSource
 ): string {
+    return getAttributionFragmentsFromVariable(variable).join("; ")
+}
+
+export function getAttributionFragmentsFromVariable(
+    variable: OwidVariableWithSource
+): string[] {
     if (
         variable.presentation?.attribution &&
         variable.presentation?.attribution !== ""
     )
-        return variable.presentation?.attribution
+        return [variable.presentation?.attribution]
     const originAttributionFragments = getOriginAttributionFragments(
         variable.origins
     )
     const sourceName = variable.source?.name
-    return uniq(compact([sourceName, ...originAttributionFragments])).join("; ")
+    return uniq(compact([sourceName, ...originAttributionFragments]))
 }
 
 interface ETLPathComponents {
@@ -1769,3 +1775,22 @@ export const getETLPathComponents = (path: string): ETLPathComponents => {
         path.split("/")
     return { channel, producer, version, dataset, table, indicator }
 }
+
+export const formatAuthors = ({
+    authors,
+    requireMax,
+    forBibtex,
+}: {
+    authors: string[]
+    requireMax?: boolean
+    forBibtex?: boolean
+}): string => {
+    if (requireMax && !authors.includes("Max Roser"))
+        authors = [...authors, "Max Roser"]
+
+    let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ")
+    if (authorsText.length === 0) authorsText = authors[0]
+    else authorsText += ` and ${last(authors)}`
+
+    return authorsText
+}

packages/@ourworldindata/utils/src/index.ts

@@ -215,6 +215,7 @@ export {
     type DataPageDataV2,
     type DataPageRelatedData,
     type DataPageRelatedResearch,
+    type PrimaryTopic,
     type FaqLink,
     type FaqDictionary,
     type RawBlockResearchAndWritingRow,
@@ -340,8 +341,10 @@ export {
     mergePartialGrapherConfigs,
     getOriginAttributionFragments,
     getAttributionFromVariable,
+    getAttributionFragmentsFromVariable,
     copyToClipboard,
     getETLPathComponents,
+    formatAuthors,
 } from "./Util.js"
 
 export {

packages/@ourworldindata/utils/src/owidTypes.ts

@@ -1464,13 +1464,19 @@ export interface FaqLink {
     fragmentId: string
 }
 
+export interface PrimaryTopic {
+    topicTag: string
+    citation: string
+}
+
 export interface DataPageDataV2 {
     status: "published" | "draft"
     title: string
     titleVariant?: string
     attributionShort?: string
     topicTagsLinks?: string[]
-    attribution: string
+    primaryTopic?: PrimaryTopic
+    attributions: string[]
     descriptionShort?: string
     descriptionFromProducer?: string
     faqs: FaqLink[] // Todo: resolve these at this level to the point where we can preview them