Skip to content

Commit

Permalink
✨ remove remaining exports references
Browse files Browse the repository at this point in the history
  • Loading branch information
@ikesau
ikesau committed Jan 22, 2025
1 parent d38cba1 commit b425637
Show file tree
Hide file tree
Showing 6 changed files with 23 additions and 149 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

The monorepo we use at [Our World in Data](https://ourworldindata.org) to create and publish embeddable, interactive visualizations like this one:

[![A Grapher chart showing world-wide life expectancy at birth. Click for interactive.](https://ourworldindata.org/grapher/exports/life-expectancy.svg)](https://ourworldindata.org/grapher/life-expectancy)
[![A Grapher chart showing world-wide life expectancy at birth. Click for interactive.](https://ourworldindata.org/grapher/life-expectancy.svg)](https://ourworldindata.org/grapher/life-expectancy)

## ✋ Disclaimer

Expand Down
17 changes: 11 additions & 6 deletions adminSiteServer/testPageRouter.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import {
ColorSchemes,
GrapherProgrammaticInterface,
} from "@ourworldindata/grapher"
import { GRAPHER_DYNAMIC_THUMBNAIL_URL } from "../settings/clientSettings.js"

const IS_LIVE = ADMIN_BASE_URL === "https://owid.cloud"
const DEFAULT_COMPARISON_URL = "https://ourworldindata.org"
Expand Down Expand Up @@ -467,7 +468,7 @@ getPlainRouteWithROTransaction(
Pick<DbPlainChart, "id"> & { config: DbRawChartConfig["full"] }
>(
trx,
`--sql
`-- sql
select ca.id, cc.full as config
from charts ca
join chart_configs cc
Expand Down Expand Up @@ -539,11 +540,15 @@ function PreviewTestPage(props: { charts: any[] }) {
href={`https://ourworldindata.org/grapher/${chart.slug}`}
>
<img
src={`https://ourworldindata.org/grapher/exports/${chart.slug}.svg`}
src={`https://ourworldindata.org/grapher/${chart.slug}.svg`}
/>
</a>
<a href={`/grapher/${chart.slug}`}>
<img src={`/grapher/exports/${chart.slug}.svg`} />
<a
href={`${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${chart.slug}.svg`}
>
<img
src={`${GRAPHER_DYNAMIC_THUMBNAIL_URL}/${chart.slug}.svg`}
/>
</a>
</div>
))}
Expand Down Expand Up @@ -732,7 +737,7 @@ getPlainRouteWithROTransaction(
async (req, res, trx) => {
const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
trx,
`--sql
`-- sql
SELECT cc.full as config
FROM charts ca
JOIN chart_configs cc
Expand All @@ -752,7 +757,7 @@ getPlainRouteWithROTransaction(
async (req, res, trx) => {
const rows = await db.knexRaw<{ config: DbRawChartConfig["full"] }>(
trx,
`--sql
`-- sql
SELECT cc.full as config
FROM charts ca
JOIN chart_configs cc
Expand Down
65 changes: 9 additions & 56 deletions baker/GrapherBaker.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@ import {
excludeUndefined,
urlToSlug,
without,
deserializeJSONFromHTML,
uniq,
keyBy,
compact,
mergeGrapherConfigs,
} from "@ourworldindata/utils"
import fs from "fs-extra"
import * as lodash from "lodash"
import { bakeGrapherToSvgAndPng } from "./GrapherImageBaker.js"
import {
BAKED_BASE_URL,
BAKED_GRAPHER_URL,
Expand All @@ -29,7 +27,6 @@ import {
} from "../db/model/Post.js"
import {
GrapherInterface,
OwidVariableDataMetadataDimensions,
DimensionProperty,
OwidVariableWithSource,
OwidChartDimensionInterface,
Expand All @@ -41,7 +38,6 @@ import {
} from "@ourworldindata/types"
import ProgressBar from "progress"
import {
getVariableData,
getMergedGrapherConfigForVariable,
getVariableOfDatapageIfApplicable,
} from "../db/model/Variable.js"
Expand Down Expand Up @@ -267,29 +263,12 @@ const renderGrapherPage = async (
)
}

const chartIsSameVersion = async (
htmlPath: string,
grapherVersion: number | undefined
): Promise<boolean> => {
if (fs.existsSync(htmlPath)) {
// If the chart is the same version, we can potentially skip baking the data and exports (which is by far the slowest part)
const html = await fs.readFile(htmlPath, "utf8")
const savedVersion = deserializeJSONFromHTML(html)
return savedVersion?.version === grapherVersion
} else {
return false
}
}

const bakeGrapherPageAndVariablesPngAndSVGIfChanged = async (
const bakeGrapherPage = async (
bakedSiteDir: string,
imageMetadataDictionary: Record<string, DbEnrichedImage>,
grapher: GrapherInterface,
knex: db.KnexReadonlyTransaction
) => {
const htmlPath = `${bakedSiteDir}/grapher/${grapher.slug}.html`
const isSameVersion = await chartIsSameVersion(htmlPath, grapher.version)

// Need to set up the connection for using TypeORM in
// renderDataPageOrGrapherPage() when baking using multiple worker threads
// (MAX_NUM_BAKE_PROCESSES > 1). It could be done in
Expand All @@ -308,28 +287,6 @@ const bakeGrapherPageAndVariablesPngAndSVGIfChanged = async (
)
)
console.log(outPath)

const variableIds = lodash.uniq(
grapher.dimensions?.map((d) => d.variableId)
)
if (!variableIds.length) return

await fs.mkdirp(`${bakedSiteDir}/grapher/exports/`)
const svgPath = `${bakedSiteDir}/grapher/exports/${grapher.slug}.svg`
const pngPath = `${bakedSiteDir}/grapher/exports/${grapher.slug}.png`
if (!isSameVersion || !fs.existsSync(svgPath) || !fs.existsSync(pngPath)) {
const loadDataMetadataPromises: Promise<OwidVariableDataMetadataDimensions>[] =
variableIds.map(getVariableData)
const variableDataMetadata = await Promise.all(loadDataMetadataPromises)
const variableDataMedadataMap = new Map(
variableDataMetadata.map((item) => [item.metadata.id, item])
)
await bakeGrapherToSvgAndPng(
`${bakedSiteDir}/grapher/exports`,
grapher,
variableDataMedadataMap
)
}
}

const deleteOldGraphers = async (bakedSiteDir: string, newSlugs: string[]) => {
Expand All @@ -343,17 +300,13 @@ const deleteOldGraphers = async (bakedSiteDir: string, newSlugs: string[]) => {
// do not delete grapher slugs redirected to explorers
.filter((slug) => !isPathRedirectedToExplorer(`/grapher/${slug}`))
for (const slug of toRemove) {
console.log(`DELETING ${slug}`)
try {
const paths = [
`${bakedSiteDir}/grapher/${slug}.html`,
`${bakedSiteDir}/grapher/exports/${slug}.png`,
] //, `${BAKED_SITE_DIR}/grapher/exports/${slug}.svg`]
await Promise.all(paths.map((p) => fs.unlink(p)))
paths.map((p) => console.log(p))
} catch (err) {
console.error(err)
}
const path = `${bakedSiteDir}/grapher/${slug}.html`
console.log(`DELETING ${path}`)
await fs.unlink(path, (err) =>
err
? console.error(`Error deleting ${path}`, err)
: console.log(`Deleted ${path}`)
)
}
}

Expand All @@ -379,7 +332,7 @@ export const bakeSingleGrapherChart = async (
return
}

await bakeGrapherPageAndVariablesPngAndSVGIfChanged(
await bakeGrapherPage(
args.bakedSiteDir,
args.imageMetadataDictionary,
grapher,
Expand Down
84 changes: 0 additions & 84 deletions baker/GrapherImageBaker.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,8 @@ import {
} from "@ourworldindata/types"
import { Grapher, GrapherProgrammaticInterface } from "@ourworldindata/grapher"
import { MultipleOwidVariableDataDimensionsMap } from "@ourworldindata/utils"
import fs from "fs-extra"
import path from "path"
import sharp from "sharp"
import svgo from "svgo"
import * as db from "../db/db.js"
import { getDataForMultipleVariables } from "../db/model/Variable.js"
import { grapherSlugToExportFileKey } from "./GrapherBakingUtils.js"
import { BAKED_GRAPHER_URL } from "../settings/clientSettings.js"

Expand All @@ -23,29 +19,6 @@ interface SvgFilenameFragments {
queryStr?: string
}

export async function bakeGrapherToSvgAndPng(
outDir: string,
jsonConfig: GrapherInterface,
vardata: MultipleOwidVariableDataDimensionsMap
) {
const grapher = initGrapherForSvgExport(jsonConfig)
grapher.receiveOwidData(vardata)
const outPath = path.join(outDir, grapher.slug as string)

const svgCode = grapher.staticSVG

return Promise.all([
fs
.writeFile(`${outPath}.svg`, svgCode)
.then(() => console.log(`${outPath}.svg`)),
sharp(Buffer.from(grapher.staticSVG), { density: 144 })
.png()
.resize(grapher.defaultBounds.width, grapher.defaultBounds.height)
.flatten({ background: "#ffffff" })
.toFile(`${outPath}.png`),
])
}

export async function getGraphersAndRedirectsBySlug(
knex: db.KnexReadonlyTransaction
) {
Expand Down Expand Up @@ -93,41 +66,6 @@ export async function getPublishedGraphersBySlug(
return { graphersBySlug, graphersById }
}

export async function bakeGrapherToSvg(
jsonConfig: GrapherInterface,
outDir: string,
slug: string,
queryStr = "",
optimizeSvgs = false,
overwriteExisting = false,
verbose = true
) {
const grapher = initGrapherForSvgExport(jsonConfig, queryStr)
const { width, height } = grapher.defaultBounds
const outPath = buildSvgOutFilepath(
outDir,
{
slug,
version: jsonConfig.version ?? 0,
width,
height,
queryStr,
},
verbose
)

if (fs.existsSync(outPath) && !overwriteExisting) return
const variableIds = grapher.dimensions.map((d) => d.variableId)
const vardata = await getDataForMultipleVariables(variableIds)
grapher.receiveOwidData(vardata)

let svgCode = grapher.staticSVG
if (optimizeSvgs) svgCode = await optimizeSvg(svgCode)

await fs.writeFile(outPath, svgCode)
return svgCode
}

export function initGrapherForSvgExport(
jsonConfig: GrapherProgrammaticInterface,
queryStr: string = ""
Expand Down Expand Up @@ -170,28 +108,6 @@ export function buildSvgOutFilepath(
return outPath
}

const svgoConfig: svgo.Config = {
floatPrecision: 2,
plugins: [
{
name: "preset-default",
params: {
overrides: {
// disable certain plugins
collapseGroups: false, // breaks the "Our World in Data" logo in the upper right
removeUnknownsAndDefaults: false, // would remove hrefs from links (<a>)
removeViewBox: false,
},
},
},
],
}

async function optimizeSvg(svgString: string): Promise<string> {
const optimizedSvg = await svgo.optimize(svgString, svgoConfig)
return optimizedSvg.data
}

export async function grapherToSVG(
jsonConfig: GrapherInterface,
vardata: MultipleOwidVariableDataDimensionsMap
Expand Down
2 changes: 1 addition & 1 deletion baker/redirects.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ export const getRedirects = async (knex: db.KnexReadonlyTransaction) => {

// Automatic static grapher exports for every grapher chart
// Example: https://assets.ourworldindata.org/grapher/exports/absolute-change-co2.svg
"/grapher/exports/* https://assets.ourworldindata.org/grapher/exports/:splat 301",
"/grapher/exports/* https://ourworldindata.org/grapher/:splat 301",
]

// TODO: Fix this transaction locking up the DB for too long.
Expand Down
2 changes: 1 addition & 1 deletion ops/buildkite/deploy-content
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ deploy_to_cloudflare() {
create_dist() {
echo '--- Creating dist/ folder'
# Define a list of excluded directories for rsync
EXCLUDES=(grapher/data/variables/ .git/ grapher/exports/ exports/ uploads/ assets-admin/)
EXCLUDES=(grapher/data/variables/ .git/ exports/ uploads/ assets-admin/)

# Build rsync command with excluded directories
RSYNC_COMMAND=("rsync" "-havzq" "--delete")
Expand Down

0 comments on commit b425637

Please sign in to comment.