Data pages: improve citations (#2806)

This PR overhauls how we do citations on datapages. Max and Joe came up with a new way for how to cite data pages themselves (the last thing in the citation section) that needs the citation information of the related topic page. 

There are also various minor improvements to the display of sources/origins.

This is related to #2755 and fixes these points:
* [x] Change the wording of "processed" or "adapated" by under the chart to "with minor/major processing by Our World In Data"
* [x] Switch to new data page citation format described by Max and Joe

baker/GrapherBaker.tsx

@@ -45,7 +45,7 @@ import {
 import * as db from "../db/db.js"
 import { glob } from "glob"
 import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js"
-import { getPostBySlug } from "../db/model/Post.js"
+import { bySlug, getPostBySlug, parsePostAuthors } from "../db/model/Post.js"
 import { GrapherInterface } from "@ourworldindata/grapher"
 import workerpool from "workerpool"
 import ProgressBar from "progress"
@@ -60,11 +60,14 @@ import {
     getDatapageJson,
     parseGdocContentFromAllowedLevelOneHeadings,
 } from "../datapage/Datapage.js"
+import { slugify_topic } from "../site/DataPageV2Content.js"
 import { ExplorerProgram } from "../explorer/ExplorerProgram.js"
 import { Image } from "../db/model/Image.js"
 import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
 
 import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js"
+import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
+import { getShortPageCitation } from "../site/gdocs/utils.js"
 
 /**
  *
@@ -322,6 +325,42 @@ export async function renderDataPageV2({
         grapherConfigForVariable ?? {}
     )
 
+    const firstTopicTag = datapageData.topicTagsLinks?.[0]
+
+    if (firstTopicTag) {
+        const gdoc = await Gdoc.findOne({
+            where: {
+                slug: slugify_topic(firstTopicTag),
+            },
+            relations: ["tags"],
+        })
+        if (gdoc) {
+            const citation = getShortPageCitation(
+                gdoc.content.authors,
+                gdoc.content.title ?? "",
+                gdoc?.publishedAt
+            )
+            datapageData.primaryTopic = {
+                topicTag: firstTopicTag,
+                citation,
+            }
+        } else {
+            const post = await bySlug(slugify_topic(firstTopicTag))
+            if (post) {
+                const authors = parsePostAuthors(post.authors)
+                const citation = getShortPageCitation(
+                    authors,
+                    post.title,
+                    post.published_at
+                )
+                datapageData.primaryTopic = {
+                    topicTag: firstTopicTag,
+                    citation,
+                }
+            }
+        }
+    }
+
     // Get the charts this variable is being used in (aka "related charts")
     // and exclude the current chart to avoid duplicates
     datapageData.allCharts = await getRelatedChartsForVariable(

datapage/Datapage.ts

@@ -16,9 +16,9 @@ import {
     DataPageDataV2,
     OwidVariableWithSource,
     dayjs,
-    getAttributionFromVariable,
     gdocIdRegex,
     getETLPathComponents,
+    getAttributionFragmentsFromVariable,
 } from "@ourworldindata/utils"
 import { ExplorerProgram } from "../explorer/ExplorerProgram.js"
 import { Gdoc } from "../db/model/Gdoc/Gdoc.js"
@@ -29,7 +29,7 @@ export const getDatapageDataV2 = async (
     partialGrapherConfig: GrapherInterface
 ): Promise<DataPageDataV2> => {
     {
-        const processingLevel = variableMetadata.processingLevel ?? "major"
+        const processingLevel = variableMetadata.processingLevel ?? "minor"
         const version =
             getETLPathComponents(variableMetadata.catalogPath ?? "")?.version ??
             ""
@@ -59,7 +59,7 @@ export const getDatapageDataV2 = async (
             attributionShort: variableMetadata.presentation?.attributionShort,
             titleVariant: variableMetadata.presentation?.titleVariant,
             topicTagsLinks: variableMetadata.presentation?.topicTagsLinks ?? [],
-            attribution: getAttributionFromVariable(variableMetadata),
+            attributions: getAttributionFragmentsFromVariable(variableMetadata),
             faqs: [],
             descriptionKey: variableMetadata.descriptionKey ?? [],
             descriptionProcessing: variableMetadata.descriptionProcessing,

db/migrateWpPostsToArchieMl.ts

@@ -4,7 +4,6 @@ import cheerio from "cheerio"
 import {
     OwidGdocPublicationContext,
     OwidGdocInterface,
-    sortBy,
     OwidArticleBackportingStatistics,
     OwidGdocType,
     RelatedChart,
@@ -19,6 +18,7 @@ import {
     adjustHeadingLevels,
 } from "./model/Gdoc/htmlToEnriched.js"
 import { getRelatedCharts } from "./wpdb.js"
+import { parsePostAuthors } from "./model/Post.js"
 
 // slugs from all the linear entries we want to migrate from @edomt
 const entries = new Set([
@@ -149,10 +149,6 @@ const migrate = async (): Promise<void> => {
                 ? post.published_at.toLocaleDateString("en-US", options)
                 : ""
 
-            const authors: { author: string; order: number }[] = JSON.parse(
-                post.authors
-            )
-
             const archieMlFieldContent: OwidGdocInterface = {
                 id: `wp-${post.id}`,
                 slug: post.slug,
@@ -162,9 +158,7 @@ const migrate = async (): Promise<void> => {
                     title: post.title,
                     subtitle: post.excerpt,
                     excerpt: post.excerpt,
-                    authors: sortBy(authors, ["order"]).map(
-                        (author) => author.author
-                    ),
+                    authors: parsePostAuthors(post.authors),
                     dateline: dateline,
                     // TODO: this discards block level elements - those might be needed?
                     refs: undefined,

db/model/Post.ts

@@ -1,6 +1,6 @@
 import * as db from "../db.js"
 import { Knex } from "knex"
-import { PostRow } from "@ourworldindata/utils"
+import { PostRow, sortBy } from "@ourworldindata/utils"
 
 export const postsTable = "posts"
 
@@ -49,6 +49,14 @@ export const setTags = async (
 export const bySlug = async (slug: string): Promise<PostRow | undefined> =>
     (await db.knexTable("posts").where({ slug: slug }))[0]
 
+/** The authors field in the posts table is a json column that contains an array of
+    { order: 1, authors: "Max Mustermann" } like records. This function parses the
+    string and returns a simple string array of author names in the correct order  */
+export const parsePostAuthors = (authorsJson: string): string[] => {
+    const authors = JSON.parse(authorsJson)
+    return sortBy(authors, ["order"]).map((author) => author.author)
+}
+
 export const setTagsForPost = async (
     postId: number,
     tagIds: number[]

packages/@ourworldindata/grapher/src/core/Grapher.tsx

@@ -1535,7 +1535,8 @@ export class Grapher
 
         const uniqueAttributions = uniq(compact(attributions))
 
-        if (uniqueAttributions.length > 3) return "Multiple sources"
+        if (uniqueAttributions.length > 3)
+            return `${attributions[0]} and other sources`
 
         return uniqueAttributions.join("; ")
     }

packages/@ourworldindata/utils/src/Util.ts

@@ -1743,16 +1743,22 @@ export function getOriginAttributionFragments(
 export function getAttributionFromVariable(
     variable: OwidVariableWithSource
 ): string {
+    return getAttributionFragmentsFromVariable(variable).join("; ")
+}
+
+export function getAttributionFragmentsFromVariable(
+    variable: OwidVariableWithSource
+): string[] {
     if (
         variable.presentation?.attribution &&
         variable.presentation?.attribution !== ""
     )
-        return variable.presentation?.attribution
+        return [variable.presentation?.attribution]
     const originAttributionFragments = getOriginAttributionFragments(
         variable.origins
     )
     const sourceName = variable.source?.name
-    return uniq(compact([sourceName, ...originAttributionFragments])).join("; ")
+    return uniq(compact([sourceName, ...originAttributionFragments]))
 }
 
 interface ETLPathComponents {
@@ -1769,3 +1775,22 @@ export const getETLPathComponents = (path: string): ETLPathComponents => {
         path.split("/")
     return { channel, producer, version, dataset, table, indicator }
 }
+
+export const formatAuthors = ({
+    authors,
+    requireMax,
+    forBibtex,
+}: {
+    authors: string[]
+    requireMax?: boolean
+    forBibtex?: boolean
+}): string => {
+    if (requireMax && !authors.includes("Max Roser"))
+        authors = [...authors, "Max Roser"]
+
+    let authorsText = authors.slice(0, -1).join(forBibtex ? " and " : ", ")
+    if (authorsText.length === 0) authorsText = authors[0]
+    else authorsText += ` and ${last(authors)}`
+
+    return authorsText
+}

packages/@ourworldindata/utils/src/index.ts

@@ -215,6 +215,7 @@ export {
     type DataPageDataV2,
     type DataPageRelatedData,
     type DataPageRelatedResearch,
+    type PrimaryTopic,
     type FaqLink,
     type FaqDictionary,
     type RawBlockResearchAndWritingRow,
@@ -340,8 +341,10 @@ export {
     mergePartialGrapherConfigs,
     getOriginAttributionFragments,
     getAttributionFromVariable,
+    getAttributionFragmentsFromVariable,
     copyToClipboard,
     getETLPathComponents,
+    formatAuthors,
 } from "./Util.js"
 
 export {

packages/@ourworldindata/utils/src/owidTypes.ts

@@ -1464,13 +1464,19 @@ export interface FaqLink {
     fragmentId: string
 }
 
+export interface PrimaryTopic {
+    topicTag: string
+    citation: string
+}
+
 export interface DataPageDataV2 {
     status: "published" | "draft"
     title: string
     titleVariant?: string
     attributionShort?: string
     topicTagsLinks?: string[]
-    attribution: string
+    primaryTopic?: PrimaryTopic
+    attributions: string[]
     descriptionShort?: string
     descriptionFromProducer?: string
     faqs: FaqLink[] // Todo: resolve these at this level to the point where we can preview them
@@ -1602,6 +1608,7 @@ export interface DataPageV2ContentFields {
     faqEntries: FaqEntryData | undefined
     // TODO: add gdocs for FAQs
     isPreviewing?: boolean
+    canonicalUrl: string
 }
 
 export interface UserCountryInformation {

site/DataPageV2.tsx

@@ -146,6 +146,7 @@ export const DataPageV2 = (props: {
                                     grapherConfig={grapherConfig}
                                     isPreviewing={isPreviewing}
                                     faqEntries={faqEntries}
+                                    canonicalUrl={canonicalUrl}
                                 />
                             </DebugProvider>
                         </div>