🎉 lightning chart deploys

adminSiteServer/apiRouter.ts

@@ -114,7 +114,7 @@ const triggerStaticBuild = async (user: CurrentUser, commitMessage: string) => {
     })
 }
 
-const enqueueLightningChange = async (
+const enqueueLightningGDocChange = async (
     user: CurrentUser,
     commitMessage: string,
     slug: string
@@ -131,7 +131,28 @@ const enqueueLightningChange = async (
         authorName: user.fullName,
         authorEmail: user.email,
         message: commitMessage,
-        slug,
+        gdocSlug: slug,
+    })
+}
+
+const enqueueLightningChartChange = async (
+    user: CurrentUser,
+    commitMessage: string,
+    slug: string
+) => {
+    if (!BAKE_ON_CHANGE) {
+        console.log(
+            "Not triggering static build because BAKE_ON_CHANGE is false"
+        )
+        return
+    }
+
+    return new DeployQueueServer().enqueueChange({
+        timeISOString: new Date().toISOString(),
+        authorName: user.fullName,
+        authorEmail: user.email,
+        message: commitMessage,
+        chartSlug: slug,
     })
 }
 
@@ -406,7 +427,12 @@ const saveGrapher = async (
             `UPDATE charts SET publishedAt=?, publishedByUserId=? WHERE id = ? `,
             [now, user.id, chartId]
         )
-        await triggerStaticBuild(user, `Publishing chart ${newConfig.slug}`)
+        // await triggerStaticBuild(user, `Publishing chart ${newConfig.slug}`)
+        await enqueueLightningChartChange(
+            user,
+            `(Lightning) Publishing chart ${newConfig.slug}`,
+            newConfig.slug!
+        )
     } else if (
         !newConfig.isPublished &&
         existingConfig &&
@@ -417,9 +443,15 @@ const saveGrapher = async (
             `DELETE FROM chart_slug_redirects WHERE chart_id = ?`,
             [existingConfig.id]
         )
+        // This doesn't trigger lightning build, it should happen quite rarely
         await triggerStaticBuild(user, `Unpublishing chart ${newConfig.slug}`)
     } else if (newConfig.isPublished)
-        await triggerStaticBuild(user, `Updating chart ${newConfig.slug}`)
+        // await triggerStaticBuild(user, `Updating chart ${newConfig.slug}`)
+        await enqueueLightningChartChange(
+            user,
+            `(Lightning) Updating chart ${newConfig.slug}`,
+            newConfig.slug!
+        )
 
     return chartId
 }
@@ -2207,8 +2239,8 @@ apiRouter.put("/tags/:tagId", async (req: Request) => {
         if (!gdoc.length) {
             return {
                 success: true,
-                tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug. 
-                    
+                tagUpdateWarning: `The tag's slug has been updated, but there isn't a published Gdoc page with the same slug.
+
 Are you sure you haven't made a typo?`,
             }
         }
@@ -2602,9 +2634,9 @@ apiRouter.put("/gdocs/:id", async (req, res) => {
 
     const hasChanges = checkHasChanges(prevGdoc, nextGdoc)
     if (checkIsLightningUpdate(prevGdoc, nextGdoc, hasChanges)) {
-        await enqueueLightningChange(
+        await enqueueLightningGDocChange(
             res.locals.user,
-            `Lightning update ${nextGdoc.slug}`,
+            `(Lightning) Gdoc update ${nextGdoc.slug}`,
             nextGdoc.slug
         )
     } else if (checkFullDeployFallback(prevGdoc, nextGdoc, hasChanges)) {

baker/BuildkiteTrigger.ts

@@ -101,16 +101,32 @@ export class BuildkiteTrigger {
         }
     }
 
-    async runLightningBuild(
+    async runLightningGdocBuild(
         message: string,
         gdocSlugs: string[]
     ): Promise<void> {
+        if (!gdocSlugs.length) {
+            return
+        }
         const buildNumber = await this.triggerBuild(message, {
             LIGHTNING_GDOC_SLUGS: gdocSlugs.join(" "),
         })
         await this.waitForBuildToFinish(buildNumber)
     }
 
+    async runLightningChartBuild(
+        message: string,
+        chartSlugs: string[]
+    ): Promise<void> {
+        if (!chartSlugs.length) {
+            return
+        }
+        const buildNumber = await this.triggerBuild(message, {
+            LIGHTNING_CHART_SLUGS: chartSlugs.join(" "),
+        })
+        await this.waitForBuildToFinish(buildNumber)
+    }
+
     async runFullBuild(message: string): Promise<void> {
         const buildNumber = await this.triggerBuild(message, {})
         await this.waitForBuildToFinish(buildNumber)

baker/DeployQueueServer.test.ts

@@ -28,7 +28,7 @@ describe("parseQueueContent", () => {
         )
         expect(output[1].message).toEqual("something one")
         expect(output[2].message).toEqual("something two")
-        expect(output[3].slug).toEqual("article-lightning-deploy")
+        expect(output[3].gdocSlug).toEqual("article-lightning-deploy")
     })
 })
 

baker/DeployUtils.ts

@@ -40,10 +40,23 @@ const triggerBakeAndDeploy = async (
     if (BUILDKITE_API_ACCESS_TOKEN) {
         const buildkite = new BuildkiteTrigger()
         if (lightningQueue?.length) {
+            // first run lightning gdocs
             await buildkite
-                .runLightningBuild(
+                .runLightningGdocBuild(
                     message!,
-                    lightningQueue.map((change) => change.slug!)
+                    lightningQueue
+                        .map((change) => change.gdocSlug!)
+                        .filter(Boolean)
+                )
+                .catch(logErrorAndMaybeSendToBugsnag)
+
+            // then run chart builds
+            await buildkite
+                .runLightningChartBuild(
+                    message!,
+                    lightningQueue
+                        .map((change) => change.chartSlug!)
+                        .filter(Boolean)
                 )
                 .catch(logErrorAndMaybeSendToBugsnag)
         } else {
@@ -55,10 +68,12 @@ const triggerBakeAndDeploy = async (
         // otherwise, bake locally. This is used for local development or staging servers
         const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL)
         if (lightningQueue?.length) {
-            if (!lightningQueue.every((change) => change.slug))
-                throw new Error("Lightning deploy is missing a slug")
-
-            await baker.bakeGDocPosts(lightningQueue.map((c) => c.slug!))
+            await baker.bakeGDocPosts(
+                lightningQueue.map((c) => c.gdocSlug!).filter(Boolean)
+            )
+            await baker.bakeCharts(
+                lightningQueue.map((c) => c.chartSlug!).filter(Boolean)
+            )
         } else {
             await baker.bakeAll()
         }
@@ -135,4 +150,5 @@ export const deployIfQueueIsNotEmpty = async () => {
     deploying = false
 }
 
-const isLightningChange = (item: DeployChange) => item.slug
+const isLightningChange = (item: DeployChange) =>
+    item.gdocSlug || item.chartSlug

baker/GrapherBaker.tsx

@@ -498,15 +498,22 @@ export const bakeSingleGrapherChart = async (
 }
 
 export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
-    async (bakedSiteDir: string) => {
-        const chartsToBake: { id: number; config: string; slug: string }[] =
+    async (bakedSiteDir: string, slugsToBake: string[] = []) => {
+        let chartsToBake: { id: number; config: string; slug: string }[] =
             await db.queryMysql(`
                 SELECT
                     id, config, config->>'$.slug' as slug
                 FROM charts WHERE JSON_EXTRACT(config, "$.isPublished")=true
                 ORDER BY JSON_EXTRACT(config, "$.slug") ASC
                 `)
 
+        // If we are passed a list of slugs, only bake those charts
+        if (slugsToBake.length) {
+            chartsToBake = chartsToBake.filter((row) =>
+                slugsToBake.includes(row.slug)
+            )
+        }
+
         const newSlugs = chartsToBake.map((row) => row.slug)
         await fs.mkdirp(bakedSiteDir + "/grapher")
 
@@ -535,7 +542,8 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
             }
         )
 
-        if (MAX_NUM_BAKE_PROCESSES === 1) {
+        // run async if we use single process or bake specific slugs
+        if (MAX_NUM_BAKE_PROCESSES === 1 || slugsToBake.length) {
             await Promise.all(
                 jobs.map(async (job) => {
                     await bakeSingleGrapherChart(job)
@@ -563,6 +571,9 @@ export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers =
             }
         }
 
-        await deleteOldGraphers(bakedSiteDir, excludeUndefined(newSlugs))
-        progressBar.tick({ name: `✅ Deleted old graphers` })
+        // only delete old graphers if we don't bake selected slugs
+        if (!slugsToBake.length) {
+            await deleteOldGraphers(bakedSiteDir, excludeUndefined(newSlugs))
+            progressBar.tick({ name: `✅ Deleted old graphers` })
+        }
     }

baker/SiteBaker.tsx

@@ -407,6 +407,17 @@ export class SiteBaker {
         this.progressBar.tick({ name: "✅ baked google doc posts" })
     }
 
+    async bakeCharts(slugsToBake: string[] = []) {
+        if (!this.bakeSteps.has("charts")) return
+        await bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers(
+            this.bakedSiteDir,
+            slugsToBake
+        )
+        this.progressBar.tick({
+            name: "✅ bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers",
+        })
+    }
+
     // Bake unique individual pages
     private async bakeSpecialPages() {
         if (!this.bakeSteps.has("specialPages")) return
@@ -678,14 +689,7 @@ export class SiteBaker {
         await this.bakeSpecialPages()
         await this.bakeCountryProfiles()
         await this.bakeExplorers()
-        if (this.bakeSteps.has("charts")) {
-            await bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers(
-                this.bakedSiteDir
-            )
-            this.progressBar.tick({
-                name: "✅ bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers",
-            })
-        }
+        await this.bakeCharts()
         await this.bakeDetailsOnDemand()
         await this.validateGrapherDodReferences()
         await this.bakeGDocPosts()

baker/bakeCharts.ts

@@ -0,0 +1,34 @@
+#! /usr/bin/env node
+
+import yargs from "yargs"
+import { hideBin } from "yargs/helpers"
+import { SiteBaker } from "./SiteBaker.js"
+import { BAKED_SITE_DIR, BAKED_BASE_URL } from "../settings/serverSettings.js"
+import * as db from "../db/db.js"
+
+yargs(hideBin(process.argv))
+    .command<{ slugs: string[] }>(
+        "$0 [slug]",
+        "Bake multiple Charts",
+        (yargs) => {
+            yargs
+                .option("slugs", {
+                    type: "array",
+                    describe: "Chart slugs",
+                })
+                .demandOption(
+                    ["slugs"],
+                    "Please provide slugs using --slugs slug1 slug2"
+                )
+        },
+        async ({ slugs }) => {
+            const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL)
+
+            await db.getConnection()
+            await baker.bakeCharts(slugs)
+            process.exit(0)
+        }
+    )
+    .help()
+    .alias("help", "h")
+    .strict().argv

ops/buildkite/deploy-content

@@ -31,6 +31,11 @@ deploy_content () {
     # right now lightning bake has to wait if there's a regular bake
     if [ -n "${LIGHTNING_GDOC_SLUGS:-}" ]; then
         bake_gdoc_posts "$LIGHTNING_GDOC_SLUGS"
+    # Lightning updates for charts
+    elif [ -n "${LIGHTNING_CHART_SLUGS:-}" ]; then
+        bake_charts "$LIGHTNING_CHART_SLUGS"
+        # We could only sync selected slugs, but syncing everything takes only 6s anyway
+        sync_to_r2_aws grapher/exports
     else
         update_owid_content_repo
         sync_wordpress_uploads
@@ -171,4 +176,13 @@ bake_gdoc_posts() {
     )
 }
 
+bake_charts() {
+    local slugs="$1"
+    echo "--- Baking Charts ${slugs}"
+    (
+        cd owid-grapher
+        yarn bakeCharts --slugs ${slugs}
+    )
+}
+
 deploy_content

package.json

@@ -19,6 +19,7 @@
         "buildVite": "vite build",
         "buildWordpressPlugin": "cd ./wordpress/web/app/plugins/owid && yarn && yarn build",
         "bakeGdocPosts": "node --enable-source-maps ./itsJustJavascript/baker/bakeGdocPosts.js",
+        "bakeCharts": "node --enable-source-maps ./itsJustJavascript/baker/bakeCharts.js",
         "cleanTsc": "rm -rf itsJustJavascript && tsc -b -clean",
         "deployWordpress": "./wordpress/scripts/deploy.sh",
         "fetchServerStatus": "node --enable-source-maps ./itsJustJavascript/baker/liveCommit.js",

packages/@ourworldindata/utils/src/owidTypes.ts

@@ -434,7 +434,8 @@ export interface DeployChange {
     authorName?: string
     authorEmail?: string
     message?: string
-    slug?: string
+    gdocSlug?: string
+    chartSlug?: string
 }
 
 export interface Deploy {