From 210b4d4b82a83002c865063dd1ec132841ab4d89 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Wed, 20 Mar 2024 21:19:54 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=94=A8=20images=20need=20a=20RW=20tra?= =?UTF-8?q?nsaction=20in=20case=20they=20need=20to=20update=20the=20db=20?= =?UTF-8?q?=F0=9F=98=A2=20-=20bubble=20this=20up?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/adminRouter.tsx | 9 ++-- adminSiteServer/app.tsx | 2 +- adminSiteServer/mockSiteRouter.tsx | 69 +++++++++++++++++------------- baker/DatapageHelpers.ts | 4 +- baker/DeployUtils.ts | 6 +-- baker/GrapherBaker.tsx | 16 +++---- baker/SiteBaker.tsx | 20 ++++----- baker/algolia/indexToAlgolia.tsx | 4 +- baker/bakeGdocPost.ts | 2 +- baker/bakeGdocPosts.ts | 2 +- baker/buildLocalBake.ts | 2 +- baker/runBakeGraphers.ts | 2 +- baker/siteRenderers.tsx | 19 +++++--- baker/sitemap.ts | 2 +- baker/startDeployQueueServer.ts | 2 +- db/model/Gdoc/GdocAuthor.ts | 6 +-- db/model/Gdoc/GdocBase.ts | 6 +-- db/model/Gdoc/GdocDataInsight.ts | 2 +- db/model/Gdoc/GdocFactory.ts | 14 +++--- db/model/Gdoc/GdocPost.ts | 8 +++- db/model/Image.ts | 12 +++--- db/model/Post.ts | 2 +- devTools/markdownTest/markdown.ts | 8 +++- 23 files changed, 123 insertions(+), 96 deletions(-) diff --git a/adminSiteServer/adminRouter.tsx b/adminSiteServer/adminRouter.tsx index 2e738c12954..cb920859d3d 100644 --- a/adminSiteServer/adminRouter.tsx +++ b/adminSiteServer/adminRouter.tsx @@ -45,7 +45,10 @@ import { import { getChartConfigBySlug } from "../db/model/Chart.js" import { getVariableMetadata } from "../db/model/Variable.js" import { DbPlainDatasetFile, DbPlainDataset } from "@ourworldindata/types" -import { getPlainRouteWithROTransaction } from "./plainRouterHelpers.js" +import { + getPlainRouteNonIdempotentWithRWTransaction, + getPlainRouteWithROTransaction, +} from "./plainRouterHelpers.js" // Used for rate-limiting important endpoints (login, register) to prevent brute force attacks const limiterMiddleware = ( @@ -315,7 +318,7 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( adminRouter, "/datapage-preview/:id", async (req, res, trx) => { @@ -337,7 +340,7 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( adminRouter, "/grapher/:slug", async (req, res, trx) => { diff --git a/adminSiteServer/app.tsx b/adminSiteServer/app.tsx index ad5ab374481..8a9e5ba7a72 100644 --- a/adminSiteServer/app.tsx +++ b/adminSiteServer/app.tsx @@ -126,7 +126,7 @@ export class OwidAdminApp { // Public preview of a Gdoc document app.get("/gdocs/:id/preview", async (req, res) => { try { - await db.knexReadonlyTransaction(async (knex) => { + await db.knexReadWriteTransaction(async (knex) => { const gdoc = await getAndLoadGdocById( knex, req.params.id, diff --git a/adminSiteServer/mockSiteRouter.tsx b/adminSiteServer/mockSiteRouter.tsx index 20c10c76bd0..3e1f470839b 100644 --- a/adminSiteServer/mockSiteRouter.tsx +++ b/adminSiteServer/mockSiteRouter.tsx @@ -57,7 +57,10 @@ import { GdocPost } from "../db/model/Gdoc/GdocPost.js" import { GdocDataInsight } from "../db/model/Gdoc/GdocDataInsight.js" import * as db from "../db/db.js" import { calculateDataInsightIndexPageCount } from "../db/model/Gdoc/gdocUtils.js" -import { getPlainRouteWithROTransaction } from "./plainRouterHelpers.js" +import { + getPlainRouteNonIdempotentWithRWTransaction, + getPlainRouteWithROTransaction, +} from "./plainRouterHelpers.js" import { DEFAULT_LOCAL_BAKE_DIR } from "../site/SiteConstants.js" require("express-async-errors") @@ -68,7 +71,7 @@ const mockSiteRouter = Router() mockSiteRouter.use(express.urlencoded({ extended: true })) mockSiteRouter.use(express.json()) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/sitemap.xml", async (req, res, trx) => { @@ -78,7 +81,7 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/atom.xml", async (req, res, trx) => { @@ -88,7 +91,7 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/atom-no-topic-pages.xml", async (req, res, trx) => { @@ -188,7 +191,7 @@ mockSiteRouter.get("/collection/custom", async (_, res) => { return res.send(await renderDynamicCollectionPage()) }) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/grapher/:slug", async (req, res, trx) => { @@ -204,12 +207,16 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction(mockSiteRouter, "/", async (req, res, trx) => { - const frontPage = await renderFrontPage(trx) - res.send(frontPage) -}) +getPlainRouteNonIdempotentWithRWTransaction( + mockSiteRouter, + "/", + async (req, res, trx) => { + const frontPage = await renderFrontPage(trx) + res.send(frontPage) + } +) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/donate", async (req, res, trx) => res.send(await renderDonatePage(trx)) @@ -219,7 +226,7 @@ mockSiteRouter.get("/thank-you", async (req, res) => res.send(await renderThankYouPage()) ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/data-insights/:pageNumberOrSlug?", async (req, res, trx) => { @@ -278,7 +285,7 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/datapage-preview/:id", async (req, res, trx) => { @@ -318,7 +325,7 @@ mockSiteRouter.get("/search", async (req, res) => res.send(await renderSearchPage()) ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/latest", async (req, res, trx) => { @@ -327,7 +334,7 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction( +getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/latest/page/:pageno", async (req, res, trx) => { @@ -444,23 +451,27 @@ getPlainRouteWithROTransaction( } ) -getPlainRouteWithROTransaction(mockSiteRouter, "/*", async (req, res, trx) => { - const slug = req.path.replace(/^\//, "") +getPlainRouteNonIdempotentWithRWTransaction( + mockSiteRouter, + "/*", + async (req, res, trx) => { + const slug = req.path.replace(/^\//, "") - try { - const page = await renderGdocsPageBySlug(trx, slug) - res.send(page) - } catch (e) { - console.error(e) - } + try { + const page = await renderGdocsPageBySlug(trx, slug) + res.send(page) + } catch (e) { + console.error(e) + } - try { - const page = await renderPageBySlug(slug, trx) - res.send(page) - } catch (e) { - console.error(e) - res.status(404).send(await renderNotFoundPage()) + try { + const page = await renderPageBySlug(slug, trx) + res.send(page) + } catch (e) { + console.error(e) + res.status(404).send(await renderNotFoundPage()) + } } -}) +) export { mockSiteRouter } diff --git a/baker/DatapageHelpers.ts b/baker/DatapageHelpers.ts index e7f0611ca91..eeae249e9b3 100644 --- a/baker/DatapageHelpers.ts +++ b/baker/DatapageHelpers.ts @@ -16,7 +16,7 @@ import { } from "../db/model/Gdoc/GdocFactory.js" import { OwidGoogleAuth } from "../db/OwidGoogleAuth.js" import { GrapherInterface, OwidGdocBaseInterface } from "@ourworldindata/types" -import { KnexReadonlyTransaction } from "../db/db.js" +import { KnexReadWriteTransaction } from "../db/db.js" export const getDatapageDataV2 = async ( variableMetadata: OwidVariableWithSource, @@ -78,7 +78,7 @@ export const getDatapageDataV2 = async ( * see https://github.com/owid/owid-grapher/issues/2121#issue-1676097164 */ export const getDatapageGdoc = async ( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, googleDocEditLinkOrId: string, isPreviewing: boolean ): Promise => { diff --git a/baker/DeployUtils.ts b/baker/DeployUtils.ts index db3ba5dcc6c..08741e98f1d 100644 --- a/baker/DeployUtils.ts +++ b/baker/DeployUtils.ts @@ -11,7 +11,7 @@ import { import { SiteBaker } from "../baker/SiteBaker.js" import { WebClient } from "@slack/web-api" import { DeployChange, DeployMetadata } from "@ourworldindata/utils" -import { KnexReadonlyTransaction } from "../db/db.js" +import { KnexReadWriteTransaction } from "../db/db.js" const deployQueueServer = new DeployQueueServer() @@ -35,7 +35,7 @@ export const defaultCommitMessage = async (): Promise => { */ const triggerBakeAndDeploy = async ( deployMetadata: DeployMetadata, - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, lightningQueue?: DeployChange[] ) => { // deploy to Buildkite if we're on master and BUILDKITE_API_ACCESS_TOKEN is set @@ -160,7 +160,7 @@ let deploying = false * If there are no changes in the queue, a deploy won't be initiated. */ export const deployIfQueueIsNotEmpty = async ( - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ) => { if (deploying) return deploying = true diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index 74d5543f3f0..62c3eb48bd5 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -66,7 +66,7 @@ import { getGdocBaseObjectBySlug } from "../db/model/Gdoc/GdocFactory.js" const renderDatapageIfApplicable = async ( grapher: GrapherInterface, isPreviewing: boolean, - knex: db.KnexReadonlyTransaction, + knex: db.KnexReadWriteTransaction, imageMetadataDictionary?: Record ) => { const variable = await getVariableOfDatapageIfApplicable(grapher) @@ -92,7 +92,7 @@ const renderDatapageIfApplicable = async ( */ export const renderDataPageOrGrapherPage = async ( grapher: GrapherInterface, - knex: db.KnexReadonlyTransaction, + knex: db.KnexReadWriteTransaction, imageMetadataDictionary?: Record ): Promise => { const datapage = await renderDatapageIfApplicable( @@ -133,7 +133,7 @@ export async function renderDataPageV2( pageGrapher?: GrapherInterface imageMetadataDictionary?: Record }, - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ) { const grapherConfigForVariable = await getMergedGrapherConfigForVariable( variableId, @@ -174,7 +174,7 @@ export async function renderDataPageV2( ) const imageMetadata: OwidGdocPostInterface["imageMetadata"] = merge( {}, - imageMetadataDictionary, + // imageMetadataDictionary, ...compact(gdocs.map((gdoc) => gdoc?.imageMetadata)) ) const relatedCharts: OwidGdocPostInterface["relatedCharts"] = gdocs.flatMap( @@ -317,7 +317,7 @@ export async function renderDataPageV2( */ export const renderPreviewDataPageOrGrapherPage = async ( grapher: GrapherInterface, - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ) => { const datapage = await renderDatapageIfApplicable(grapher, true, knex) if (datapage) return datapage @@ -370,7 +370,7 @@ const bakeGrapherPageAndVariablesPngAndSVGIfChanged = async ( bakedSiteDir: string, imageMetadataDictionary: Record, grapher: GrapherInterface, - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ) => { const htmlPath = `${bakedSiteDir}/grapher/${grapher.slug}.html` const isSameVersion = await chartIsSameVersion(htmlPath, grapher.version) @@ -453,7 +453,7 @@ export interface BakeSingleGrapherChartArguments { export const bakeSingleGrapherChart = async ( args: BakeSingleGrapherChartArguments, - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ) => { const grapher: GrapherInterface = JSON.parse(args.config) grapher.id = args.id @@ -475,7 +475,7 @@ export const bakeSingleGrapherChart = async ( } export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers = - async (bakedSiteDir: string, knex: db.KnexReadonlyTransaction) => { + async (bakedSiteDir: string, knex: db.KnexReadWriteTransaction) => { const chartsToBake: { id: number; config: string; slug: string }[] = await knexRaw( knex, diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 4a7d68caada..8d931e4b977 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -480,7 +480,7 @@ export class SiteBaker { } // Bake all GDoc posts, or a subset of them if slugs are provided - async bakeGDocPosts(knex: db.KnexReadonlyTransaction, slugs?: string[]) { + async bakeGDocPosts(knex: db.KnexReadWriteTransaction, slugs?: string[]) { if (!this.bakeSteps.has("gdocPosts")) return const publishedGdocs = await GdocPost.getPublishedGdocPosts(knex) @@ -544,7 +544,7 @@ export class SiteBaker { } // Bake unique individual pages - private async bakeSpecialPages(knex: db.KnexReadonlyTransaction) { + private async bakeSpecialPages(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("specialPages")) return await this.stageWrite( `${this.bakedSiteDir}/index.html`, @@ -700,7 +700,7 @@ export class SiteBaker { } } - private async bakeDataInsights(knex: db.KnexReadonlyTransaction) { + private async bakeDataInsights(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("dataInsights")) return const latestDataInsights = await db.getPublishedDataInsights(knex, 5) const publishedDataInsights = @@ -768,7 +768,7 @@ export class SiteBaker { } } - private async bakeAuthors(knex: db.KnexReadonlyTransaction) { + private async bakeAuthors(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("authors")) return const publishedAuthors = await GdocAuthor.getPublishedAuthors(knex) @@ -855,7 +855,7 @@ export class SiteBaker { } // Bake the blog index - private async bakeBlogIndex(knex: db.KnexReadonlyTransaction) { + private async bakeBlogIndex(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("blogIndex")) return const allPosts = await getBlogIndex(knex) const numPages = Math.ceil(allPosts.length / BLOG_POSTS_PER_PAGE) @@ -869,7 +869,7 @@ export class SiteBaker { } // Bake the RSS feed - private async bakeRSS(knex: db.KnexReadonlyTransaction) { + private async bakeRSS(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("rss")) return await this.stageWrite( `${this.bakedSiteDir}/atom.xml`, @@ -954,7 +954,7 @@ export class SiteBaker { this.progressBar.tick({ name: "✅ baked redirects" }) } - async bakeWordpressPages(knex: db.KnexReadonlyTransaction) { + async bakeWordpressPages(knex: db.KnexReadWriteTransaction) { await this.bakeRedirects(knex) await this.bakeEmbeds(knex) await this.bakeBlogIndex(knex) @@ -964,7 +964,7 @@ export class SiteBaker { await this.bakePosts(knex) } - private async _bakeNonWordpressPages(knex: db.KnexReadonlyTransaction) { + private async _bakeNonWordpressPages(knex: db.KnexReadWriteTransaction) { if (this.bakeSteps.has("countries")) { await bakeCountries(this, knex) } @@ -988,7 +988,7 @@ export class SiteBaker { await this.bakeDriveImages(knex) } - async bakeNonWordpressPages(knex: db.KnexReadonlyTransaction) { + async bakeNonWordpressPages(knex: db.KnexReadWriteTransaction) { const progressBarTotal = nonWordpressSteps .map((step) => this.bakeSteps.has(step)) .filter((hasStep) => hasStep).length @@ -1001,7 +1001,7 @@ export class SiteBaker { await this._bakeNonWordpressPages(knex) } - async bakeAll(knex: db.KnexReadonlyTransaction) { + async bakeAll(knex: db.KnexReadWriteTransaction) { // Ensure caches are correctly initialized this.flushCache() await this.removeDeletedPosts(knex) diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index 97771a8cb45..876a46f1004 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -195,7 +195,7 @@ function generateGdocRecords( } // Generate records for countries, WP posts (not including posts that have been succeeded by Gdocs equivalents), and Gdocs -const getPagesRecords = async (knex: db.KnexReadonlyTransaction) => { +const getPagesRecords = async (knex: db.KnexReadWriteTransaction) => { const pageviews = await getAnalyticsPageviewsByUrlObj(knex) const gdocs = await GdocPost.getPublishedGdocPosts(knex) const publishedGdocsBySlug = keyBy(gdocs, "slug") @@ -234,7 +234,7 @@ const indexToAlgolia = async () => { } const index = client.initIndex(getIndexName(SearchIndexName.Pages)) - const records = await db.knexReadonlyTransaction( + const records = await db.knexReadWriteTransaction( getPagesRecords, db.TransactionCloseMode.Close ) diff --git a/baker/bakeGdocPost.ts b/baker/bakeGdocPost.ts index 97deb5caac3..29056c602a3 100644 --- a/baker/bakeGdocPost.ts +++ b/baker/bakeGdocPost.ts @@ -19,7 +19,7 @@ void yargs(hideBin(process.argv)) async ({ slug }) => { const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) - await db.knexReadonlyTransaction( + await db.knexReadWriteTransaction( (trx) => baker.bakeGDocPosts(trx, [slug]), db.TransactionCloseMode.Close ) diff --git a/baker/bakeGdocPosts.ts b/baker/bakeGdocPosts.ts index 1b1638208ea..3d75c402a82 100644 --- a/baker/bakeGdocPosts.ts +++ b/baker/bakeGdocPosts.ts @@ -24,7 +24,7 @@ void yargs(hideBin(process.argv)) async ({ slugs }) => { const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) - await db.knexReadonlyTransaction( + await db.knexReadWriteTransaction( (trx) => baker.bakeGDocPosts(trx, slugs), db.TransactionCloseMode.Close ) diff --git a/baker/buildLocalBake.ts b/baker/buildLocalBake.ts index b988c7b510a..98810dbd51e 100644 --- a/baker/buildLocalBake.ts +++ b/baker/buildLocalBake.ts @@ -17,7 +17,7 @@ const bakeDomainToFolder = async ( await fs.mkdirp(dir) const baker = new SiteBaker(dir, baseUrl, bakeSteps) console.log(`Baking site locally with baseUrl '${baseUrl}' to dir '${dir}'`) - await db.knexReadonlyTransaction( + await db.knexReadWriteTransaction( (trx) => baker.bakeAll(trx), db.TransactionCloseMode.Close ) diff --git a/baker/runBakeGraphers.ts b/baker/runBakeGraphers.ts index 02f862ea2e9..eda4ada9c37 100755 --- a/baker/runBakeGraphers.ts +++ b/baker/runBakeGraphers.ts @@ -9,7 +9,7 @@ import * as db from "../db/db.js" */ const main = async (folder: string) => { - return db.knexReadonlyTransaction( + return db.knexReadWriteTransaction( (trx) => bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers( folder, diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index bee79e54823..b9128a9f83b 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -58,7 +58,12 @@ import { import { FormattingOptions, GrapherInterface } from "@ourworldindata/types" import { CountryProfileSpec } from "../site/countryProfileProjects.js" import { formatPost } from "./formatWordpressPost.js" -import { getHomepageId, knexRaw, KnexReadonlyTransaction } from "../db/db.js" +import { + getHomepageId, + knexRaw, + KnexReadonlyTransaction, + KnexReadWriteTransaction, +} from "../db/db.js" import { getPageOverrides, isPageOverridesCitable } from "./pageOverrides.js" import { ProminentLink } from "../site/blocks/ProminentLink.js" import { @@ -185,7 +190,7 @@ export function renderDynamicCollectionPage() { } export const renderGdocsPageBySlug = async ( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, slug: string, isPreviewing: boolean = false ): Promise => { @@ -274,7 +279,7 @@ export const renderPost = async ( ) } -export const renderFrontPage = async (knex: KnexReadonlyTransaction) => { +export const renderFrontPage = async (knex: KnexReadWriteTransaction) => { const gdocHomepageId = await getHomepageId(knex) if (gdocHomepageId) { @@ -291,7 +296,7 @@ export const renderFrontPage = async (knex: KnexReadonlyTransaction) => { } } -export const renderDonatePage = async (knex: KnexReadonlyTransaction) => { +export const renderDonatePage = async (knex: KnexReadWriteTransaction) => { const faqsGdoc = (await getAndLoadGdocById( knex, GDOCS_DONATE_FAQS_DOCUMENT_ID @@ -333,7 +338,7 @@ export const renderDataInsightsIndexPage = ( export const renderBlogByPageNum = async ( pageNum: number, - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ) => { const allPosts = await getBlogIndex(knex) @@ -359,7 +364,7 @@ export const renderSearchPage = () => export const renderNotFoundPage = () => renderToHtmlPage() -export async function makeAtomFeed(knex: KnexReadonlyTransaction) { +export async function makeAtomFeed(knex: KnexReadWriteTransaction) { const posts = (await getBlogIndex(knex)).slice(0, 10) return makeAtomFeedFromPosts(posts) } @@ -367,7 +372,7 @@ export async function makeAtomFeed(knex: KnexReadonlyTransaction) { // We don't want to include topic pages in the atom feed that is being consumed // by Mailchimp for sending the "immediate update" newsletter. Instead topic // pages announcements are sent out manually. -export async function makeAtomFeedNoTopicPages(knex: KnexReadonlyTransaction) { +export async function makeAtomFeedNoTopicPages(knex: KnexReadWriteTransaction) { const posts = (await getBlogIndex(knex)) .filter((post: IndexPost) => post.type !== OwidGdocType.TopicPage) .slice(0, 10) diff --git a/baker/sitemap.ts b/baker/sitemap.ts index 67766140e40..98c770b8fe7 100644 --- a/baker/sitemap.ts +++ b/baker/sitemap.ts @@ -64,7 +64,7 @@ const explorerToSitemapUrl = (program: ExplorerProgram): SitemapUrl[] => { export const makeSitemap = async ( explorerAdminServer: ExplorerAdminServer, - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ) => { const alreadyPublishedViaGdocsSlugsSet = await db.getSlugsWithPublishedGdocsSuccessors(knex) diff --git a/baker/startDeployQueueServer.ts b/baker/startDeployQueueServer.ts index 5bc45f755d4..499910adf0c 100644 --- a/baker/startDeployQueueServer.ts +++ b/baker/startDeployQueueServer.ts @@ -33,7 +33,7 @@ const main = async () => { setTimeout(deployIfQueueIsNotEmpty, 10 * 1000) }) - await db.knexReadonlyTransaction( + await db.knexReadWriteTransaction( deployIfQueueIsNotEmpty, db.TransactionCloseMode.Close ) diff --git a/db/model/Gdoc/GdocAuthor.ts b/db/model/Gdoc/GdocAuthor.ts index 4a8af87c073..f5cbdc5083c 100644 --- a/db/model/Gdoc/GdocAuthor.ts +++ b/db/model/Gdoc/GdocAuthor.ts @@ -41,13 +41,13 @@ export class GdocAuthor extends GdocBase implements OwidGdocAuthorInterface { } _loadSubclassAttachments = ( - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ): Promise => { return this.loadLatestWorkImages(knex) } loadLatestWorkImages = async ( - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ): Promise => { if (!this.content.title) return @@ -123,7 +123,7 @@ export class GdocAuthor extends GdocBase implements OwidGdocAuthorInterface { } static async getPublishedAuthors( - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ): Promise { return loadPublishedGdocAuthors(knex) } diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index c56e283f6bd..7fa892207b1 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -86,7 +86,7 @@ export class GdocBase implements OwidGdocBaseInterface { ) => Promise = async () => [] _omittableFields: string[] = [] _loadSubclassAttachments: ( - knex: db.KnexReadonlyTransaction + knex: db.KnexReadWriteTransaction ) => Promise = async () => undefined constructor(id?: string) { @@ -674,7 +674,7 @@ export class GdocBase implements OwidGdocBaseInterface { } async loadImageMetadata( - knex: db.KnexReadonlyTransaction, + knex: db.KnexReadWriteTransaction, filenames?: string[] ): Promise { const imagesFilenames = filenames ?? this.linkedImageFilenames @@ -815,7 +815,7 @@ export class GdocBase implements OwidGdocBaseInterface { ] } - async loadState(knex: db.KnexReadonlyTransaction): Promise { + async loadState(knex: db.KnexReadWriteTransaction): Promise { await this.loadLinkedDocuments(knex) await this.loadImageMetadata(knex) await this.loadLinkedCharts(knex) diff --git a/db/model/Gdoc/GdocDataInsight.ts b/db/model/Gdoc/GdocDataInsight.ts index 4756131b52d..4934986148f 100644 --- a/db/model/Gdoc/GdocDataInsight.ts +++ b/db/model/Gdoc/GdocDataInsight.ts @@ -59,7 +59,7 @@ export class GdocDataInsight } static async getPublishedDataInsights( - knex: db.KnexReadonlyTransaction, + knex: db.KnexReadWriteTransaction, page?: number ): Promise { return getAndLoadPublishedDataInsights(knex, page) diff --git a/db/model/Gdoc/GdocFactory.ts b/db/model/Gdoc/GdocFactory.ts index 320a247da8d..6afafaf9503 100644 --- a/db/model/Gdoc/GdocFactory.ts +++ b/db/model/Gdoc/GdocFactory.ts @@ -257,7 +257,7 @@ export async function getGdocBaseObjectBySlug( } export async function getAndLoadGdocBySlug( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, slug: string ): Promise { const base = await getGdocBaseObjectBySlug(knex, slug, true) @@ -270,7 +270,7 @@ export async function getAndLoadGdocBySlug( } export async function getAndLoadGdocById( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, id: string, contentSource?: GdocsContentSource ): Promise { @@ -283,7 +283,7 @@ export async function getAndLoadGdocById( // From an ID, get a Gdoc object with all its metadata and state loaded, in its correct subclass. // If contentSource is Gdocs, use live data from Google, otherwise use the data in the DB. export async function loadGdocFromGdocBase( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, base: OwidGdocBaseInterface, contentSource?: GdocsContentSource ): Promise { @@ -325,7 +325,7 @@ export async function loadGdocFromGdocBase( } export async function getAndLoadPublishedDataInsights( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, page?: number ): Promise { const limitOffsetClause = @@ -370,7 +370,7 @@ export async function getAndLoadPublishedDataInsights( } export async function getAndLoadPublishedGdocPosts( - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ): Promise { const rows = await knexRaw( knex, @@ -415,7 +415,7 @@ export async function getAndLoadPublishedGdocPosts( } export async function loadPublishedGdocAuthors( - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ): Promise { const rows = await knexRaw( knex, @@ -441,7 +441,7 @@ export async function loadPublishedGdocAuthors( } export async function getAndLoadListedGdocPosts( - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ): Promise { // TODO: Check if we shouldn't also restrict the types of gdocs here const rows = await knexRaw( diff --git a/db/model/Gdoc/GdocPost.ts b/db/model/Gdoc/GdocPost.ts index 4dd3a0153ee..3ffa58b731b 100644 --- a/db/model/Gdoc/GdocPost.ts +++ b/db/model/Gdoc/GdocPost.ts @@ -23,7 +23,11 @@ import { ADMIN_BASE_URL } from "../../../settings/clientSettings.js" import { parseDetails, parseFaqs } from "./rawToEnriched.js" import { htmlToEnrichedTextBlock } from "./htmlToEnriched.js" import { GdocBase } from "./GdocBase.js" -import { KnexReadonlyTransaction, knexRaw } from "../../db.js" +import { + KnexReadWriteTransaction, + KnexReadonlyTransaction, + knexRaw, +} from "../../db.js" import { getGdocBaseObjectById, getAndLoadPublishedGdocPosts, @@ -233,7 +237,7 @@ export class GdocPost extends GdocBase implements OwidGdocPostInterface { } static async getPublishedGdocPosts( - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ): Promise { // #gdocsvalidation this cast means that we trust the admin code and // workflow to provide published articles that have all the required content diff --git a/db/model/Image.ts b/db/model/Image.ts index 6513b13bead..3d9f11144c2 100644 --- a/db/model/Image.ts +++ b/db/model/Image.ts @@ -29,7 +29,7 @@ import { GDOCS_CLIENT_EMAIL, GDOCS_SHARED_DRIVE_ID, } from "../../settings/serverSettings.js" -import { KnexReadonlyTransaction } from "../db.js" +import { KnexReadWriteTransaction, KnexReadonlyTransaction } from "../db.js" class ImageStore { images: Record | undefined @@ -114,7 +114,7 @@ class ImageStore { } async syncImagesToS3( - knex: KnexReadonlyTransaction + knex: KnexReadWriteTransaction ): Promise<(Image | undefined)[]> { const images = this.images if (!images) return [] @@ -172,7 +172,7 @@ export class Image implements ImageMetadata { // If it is, upload it and update our record // If we're not aware of it, upload and record it static async syncImage( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, metadata: ImageMetadata ): Promise { const fresh = new Image(metadata) @@ -276,7 +276,7 @@ export async function getAllImages( } export async function updateImage( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, id: number, updates: Partial ): Promise { @@ -284,14 +284,14 @@ export async function updateImage( } export async function insertImageClass( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, image: Image ): Promise { return insertImageObject(knex, serializeImageRow({ ...image })) } export async function insertImageObject( - knex: KnexReadonlyTransaction, + knex: KnexReadWriteTransaction, image: DbInsertImage ): Promise { const [id] = await knex.table("images").insert(image) diff --git a/db/model/Post.ts b/db/model/Post.ts index fd2add24d1e..1ed5ef2ba0c 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -263,7 +263,7 @@ const selectHomepagePosts: FilterFnPostRestApi = (post) => post.meta?.owid_publication_context_meta_field?.homepage === true export const getBlogIndex = memoize( - async (knex: db.KnexReadonlyTransaction): Promise => { + async (knex: db.KnexReadWriteTransaction): Promise => { const gdocPosts = await getAndLoadListedGdocPosts(knex) const wpPosts = await Promise.all( await getPostsFromSnapshots( diff --git a/devTools/markdownTest/markdown.ts b/devTools/markdownTest/markdown.ts index febd66e07d7..c5f4c541228 100644 --- a/devTools/markdownTest/markdown.ts +++ b/devTools/markdownTest/markdown.ts @@ -1,4 +1,8 @@ -import { TransactionCloseMode, knexReadonlyTransaction } from "../../db/db.js" +import { + TransactionCloseMode, + knexReadWriteTransaction, + knexReadonlyTransaction, +} from "../../db/db.js" import { getPostRawBySlug } from "../../db/model/Post.js" import { enrichedBlocksToMarkdown } from "../../db/model/Gdoc/enrichedToMarkdown.js" @@ -11,7 +15,7 @@ import { getAndLoadGdocBySlug } from "../../db/model/Gdoc/GdocFactory.js" async function main(parsedArgs: parseArgs.ParsedArgs) { try { - await knexReadonlyTransaction(async (trx) => { + await knexReadWriteTransaction(async (trx) => { const gdoc = await getAndLoadGdocBySlug(trx, parsedArgs._[0]) let archieMlContent: OwidEnrichedGdocBlock[] | null let contentToShowOnError: any From 90dc009871c5ea6f2ecf7f9472d0c3ef6b2fb4c0 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Wed, 20 Mar 2024 21:31:34 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9C=20mark=20todos=20everywhere=20?= =?UTF-8?q?we=20might=20want=20to=20revert=20the=20rw=20transaction=20just?= =?UTF-8?q?=20for=20images?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adminSiteServer/adminRouter.tsx | 4 ++-- adminSiteServer/app.tsx | 1 + adminSiteServer/mockSiteRouter.tsx | 11 +++++++++++ baker/DatapageHelpers.ts | 1 + baker/DeployUtils.ts | 3 +++ baker/GrapherBaker.tsx | 11 ++++++++++- baker/SiteBaker.tsx | 14 ++++++++++++++ baker/algolia/indexToAlgolia.tsx | 2 ++ baker/bakeGdocPost.ts | 1 + baker/bakeGdocPosts.ts | 1 + baker/buildLocalBake.ts | 2 ++ baker/runBakeGraphers.ts | 1 + baker/siteRenderers.tsx | 7 +++++++ baker/sitemap.ts | 1 + baker/startDeployQueueServer.ts | 1 + db/model/Gdoc/GdocAuthor.ts | 3 +++ db/model/Gdoc/GdocBase.ts | 4 ++++ db/model/Gdoc/GdocDataInsight.ts | 1 + db/model/Gdoc/GdocFactory.ts | 8 ++++++++ db/model/Gdoc/GdocPost.ts | 1 + db/model/Post.ts | 1 + devTools/markdownTest/markdown.ts | 1 + 22 files changed, 77 insertions(+), 3 deletions(-) diff --git a/adminSiteServer/adminRouter.tsx b/adminSiteServer/adminRouter.tsx index cb920859d3d..de77928ca7c 100644 --- a/adminSiteServer/adminRouter.tsx +++ b/adminSiteServer/adminRouter.tsx @@ -317,7 +317,7 @@ getPlainRouteWithROTransaction( return res.send(explorerPage) } ) - +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( adminRouter, "/datapage-preview/:id", @@ -339,7 +339,7 @@ getPlainRouteNonIdempotentWithRWTransaction( ) } ) - +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( adminRouter, "/grapher/:slug", diff --git a/adminSiteServer/app.tsx b/adminSiteServer/app.tsx index 8a9e5ba7a72..4cd4c119114 100644 --- a/adminSiteServer/app.tsx +++ b/adminSiteServer/app.tsx @@ -126,6 +126,7 @@ export class OwidAdminApp { // Public preview of a Gdoc document app.get("/gdocs/:id/preview", async (req, res) => { try { + // TODO: this transaction is only RW because somewhere inside it we fetch images await db.knexReadWriteTransaction(async (knex) => { const gdoc = await getAndLoadGdocById( knex, diff --git a/adminSiteServer/mockSiteRouter.tsx b/adminSiteServer/mockSiteRouter.tsx index 3e1f470839b..72003e185b8 100644 --- a/adminSiteServer/mockSiteRouter.tsx +++ b/adminSiteServer/mockSiteRouter.tsx @@ -71,6 +71,7 @@ const mockSiteRouter = Router() mockSiteRouter.use(express.urlencoded({ extended: true })) mockSiteRouter.use(express.json()) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/sitemap.xml", @@ -81,6 +82,7 @@ getPlainRouteNonIdempotentWithRWTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/atom.xml", @@ -91,6 +93,7 @@ getPlainRouteNonIdempotentWithRWTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/atom-no-topic-pages.xml", @@ -191,6 +194,7 @@ mockSiteRouter.get("/collection/custom", async (_, res) => { return res.send(await renderDynamicCollectionPage()) }) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/grapher/:slug", @@ -207,6 +211,7 @@ getPlainRouteNonIdempotentWithRWTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/", @@ -216,6 +221,7 @@ getPlainRouteNonIdempotentWithRWTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/donate", @@ -226,6 +232,7 @@ mockSiteRouter.get("/thank-you", async (req, res) => res.send(await renderThankYouPage()) ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/data-insights/:pageNumberOrSlug?", @@ -285,6 +292,7 @@ getPlainRouteWithROTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/datapage-preview/:id", @@ -325,6 +333,7 @@ mockSiteRouter.get("/search", async (req, res) => res.send(await renderSearchPage()) ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/latest", @@ -334,6 +343,7 @@ getPlainRouteNonIdempotentWithRWTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/latest/page/:pageno", @@ -451,6 +461,7 @@ getPlainRouteWithROTransaction( } ) +// TODO: this transaction is only RW because somewhere inside it we fetch images getPlainRouteNonIdempotentWithRWTransaction( mockSiteRouter, "/*", diff --git a/baker/DatapageHelpers.ts b/baker/DatapageHelpers.ts index eeae249e9b3..386be565230 100644 --- a/baker/DatapageHelpers.ts +++ b/baker/DatapageHelpers.ts @@ -78,6 +78,7 @@ export const getDatapageDataV2 = async ( * see https://github.com/owid/owid-grapher/issues/2121#issue-1676097164 */ export const getDatapageGdoc = async ( + // TODO: this transaction is only RW because somewhere inside it we fetch images knex: KnexReadWriteTransaction, googleDocEditLinkOrId: string, isPreviewing: boolean diff --git a/baker/DeployUtils.ts b/baker/DeployUtils.ts index 08741e98f1d..6c5aacb5a44 100644 --- a/baker/DeployUtils.ts +++ b/baker/DeployUtils.ts @@ -33,6 +33,8 @@ export const defaultCommitMessage = async (): Promise => { /** * Initiate a deploy, without any checks. Throws error on failure. */ + +// TODO: this transaction is only RW because somewhere inside it we fetch images const triggerBakeAndDeploy = async ( deployMetadata: DeployMetadata, knex: KnexReadWriteTransaction, @@ -160,6 +162,7 @@ let deploying = false * If there are no changes in the queue, a deploy won't be initiated. */ export const deployIfQueueIsNotEmpty = async ( + // TODO: this transaction is only RW because somewhere inside it we fetch images knex: KnexReadWriteTransaction ) => { if (deploying) return diff --git a/baker/GrapherBaker.tsx b/baker/GrapherBaker.tsx index 62c3eb48bd5..0b29e38753f 100644 --- a/baker/GrapherBaker.tsx +++ b/baker/GrapherBaker.tsx @@ -90,6 +90,8 @@ const renderDatapageIfApplicable = async ( * * Render a datapage if available, otherwise render a grapher page. */ + +// TODO: this transaction is only RW because somewhere inside it we fetch images export const renderDataPageOrGrapherPage = async ( grapher: GrapherInterface, knex: db.KnexReadWriteTransaction, @@ -133,6 +135,8 @@ export async function renderDataPageV2( pageGrapher?: GrapherInterface imageMetadataDictionary?: Record }, + + // TODO: this transaction is only RW because somewhere inside it we fetch images knex: db.KnexReadWriteTransaction ) { const grapherConfigForVariable = await getMergedGrapherConfigForVariable( @@ -174,7 +178,7 @@ export async function renderDataPageV2( ) const imageMetadata: OwidGdocPostInterface["imageMetadata"] = merge( {}, - // imageMetadataDictionary, + imageMetadataDictionary, ...compact(gdocs.map((gdoc) => gdoc?.imageMetadata)) ) const relatedCharts: OwidGdocPostInterface["relatedCharts"] = gdocs.flatMap( @@ -317,6 +321,8 @@ export async function renderDataPageV2( */ export const renderPreviewDataPageOrGrapherPage = async ( grapher: GrapherInterface, + + // TODO: this transaction is only RW because somewhere inside it we fetch images knex: db.KnexReadWriteTransaction ) => { const datapage = await renderDatapageIfApplicable(grapher, true, knex) @@ -453,6 +459,8 @@ export interface BakeSingleGrapherChartArguments { export const bakeSingleGrapherChart = async ( args: BakeSingleGrapherChartArguments, + + // TODO: this transaction is only RW because somewhere inside it we fetch images knex: db.KnexReadWriteTransaction ) => { const grapher: GrapherInterface = JSON.parse(args.config) @@ -475,6 +483,7 @@ export const bakeSingleGrapherChart = async ( } export const bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers = + // TODO: this transaction is only RW because somewhere inside it we fetch images async (bakedSiteDir: string, knex: db.KnexReadWriteTransaction) => { const chartsToBake: { id: number; config: string; slug: string }[] = await knexRaw( diff --git a/baker/SiteBaker.tsx b/baker/SiteBaker.tsx index 8d931e4b977..1cc0cd1b3a8 100644 --- a/baker/SiteBaker.tsx +++ b/baker/SiteBaker.tsx @@ -480,6 +480,8 @@ export class SiteBaker { } // Bake all GDoc posts, or a subset of them if slugs are provided + + // TODO: this transaction is only RW because somewhere inside it we fetch images async bakeGDocPosts(knex: db.KnexReadWriteTransaction, slugs?: string[]) { if (!this.bakeSteps.has("gdocPosts")) return const publishedGdocs = await GdocPost.getPublishedGdocPosts(knex) @@ -544,6 +546,8 @@ export class SiteBaker { } // Bake unique individual pages + + // TODO: this transaction is only RW because somewhere inside it we fetch images private async bakeSpecialPages(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("specialPages")) return await this.stageWrite( @@ -700,6 +704,7 @@ export class SiteBaker { } } + // TODO: this transaction is only RW because somewhere inside it we fetch images private async bakeDataInsights(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("dataInsights")) return const latestDataInsights = await db.getPublishedDataInsights(knex, 5) @@ -768,6 +773,7 @@ export class SiteBaker { } } + // TODO: this transaction is only RW because somewhere inside it we fetch images private async bakeAuthors(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("authors")) return @@ -855,6 +861,8 @@ export class SiteBaker { } // Bake the blog index + + // TODO: this transaction is only RW because somewhere inside it we fetch images private async bakeBlogIndex(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("blogIndex")) return const allPosts = await getBlogIndex(knex) @@ -869,6 +877,8 @@ export class SiteBaker { } // Bake the RSS feed + + // TODO: this transaction is only RW because somewhere inside it we fetch images private async bakeRSS(knex: db.KnexReadWriteTransaction) { if (!this.bakeSteps.has("rss")) return await this.stageWrite( @@ -954,6 +964,7 @@ export class SiteBaker { this.progressBar.tick({ name: "✅ baked redirects" }) } + // TODO: this transaction is only RW because somewhere inside it we fetch images async bakeWordpressPages(knex: db.KnexReadWriteTransaction) { await this.bakeRedirects(knex) await this.bakeEmbeds(knex) @@ -964,6 +975,7 @@ export class SiteBaker { await this.bakePosts(knex) } + // TODO: this transaction is only RW because somewhere inside it we fetch images private async _bakeNonWordpressPages(knex: db.KnexReadWriteTransaction) { if (this.bakeSteps.has("countries")) { await bakeCountries(this, knex) @@ -988,6 +1000,7 @@ export class SiteBaker { await this.bakeDriveImages(knex) } + // TODO: this transaction is only RW because somewhere inside it we fetch images async bakeNonWordpressPages(knex: db.KnexReadWriteTransaction) { const progressBarTotal = nonWordpressSteps .map((step) => this.bakeSteps.has(step)) @@ -1001,6 +1014,7 @@ export class SiteBaker { await this._bakeNonWordpressPages(knex) } + // TODO: this transaction is only RW because somewhere inside it we fetch images async bakeAll(knex: db.KnexReadWriteTransaction) { // Ensure caches are correctly initialized this.flushCache() diff --git a/baker/algolia/indexToAlgolia.tsx b/baker/algolia/indexToAlgolia.tsx index 876a46f1004..44a243700a3 100644 --- a/baker/algolia/indexToAlgolia.tsx +++ b/baker/algolia/indexToAlgolia.tsx @@ -194,6 +194,7 @@ function generateGdocRecords( return records } +// TODO: this transaction is only RW because somewhere inside it we fetch images // Generate records for countries, WP posts (not including posts that have been succeeded by Gdocs equivalents), and Gdocs const getPagesRecords = async (knex: db.KnexReadWriteTransaction) => { const pageviews = await getAnalyticsPageviewsByUrlObj(knex) @@ -234,6 +235,7 @@ const indexToAlgolia = async () => { } const index = client.initIndex(getIndexName(SearchIndexName.Pages)) + // TODO: this transaction is only RW because somewhere inside it we fetch images const records = await db.knexReadWriteTransaction( getPagesRecords, db.TransactionCloseMode.Close diff --git a/baker/bakeGdocPost.ts b/baker/bakeGdocPost.ts index 29056c602a3..d63c6561395 100644 --- a/baker/bakeGdocPost.ts +++ b/baker/bakeGdocPost.ts @@ -19,6 +19,7 @@ void yargs(hideBin(process.argv)) async ({ slug }) => { const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) + // TODO: this transaction is only RW because somewhere inside it we fetch images await db.knexReadWriteTransaction( (trx) => baker.bakeGDocPosts(trx, [slug]), db.TransactionCloseMode.Close diff --git a/baker/bakeGdocPosts.ts b/baker/bakeGdocPosts.ts index 3d75c402a82..bf332cf237b 100644 --- a/baker/bakeGdocPosts.ts +++ b/baker/bakeGdocPosts.ts @@ -24,6 +24,7 @@ void yargs(hideBin(process.argv)) async ({ slugs }) => { const baker = new SiteBaker(BAKED_SITE_DIR, BAKED_BASE_URL) + // TODO: this transaction is only RW because somewhere inside it we fetch images await db.knexReadWriteTransaction( (trx) => baker.bakeGDocPosts(trx, slugs), db.TransactionCloseMode.Close diff --git a/baker/buildLocalBake.ts b/baker/buildLocalBake.ts index 98810dbd51e..6d401a63a9b 100644 --- a/baker/buildLocalBake.ts +++ b/baker/buildLocalBake.ts @@ -17,6 +17,8 @@ const bakeDomainToFolder = async ( await fs.mkdirp(dir) const baker = new SiteBaker(dir, baseUrl, bakeSteps) console.log(`Baking site locally with baseUrl '${baseUrl}' to dir '${dir}'`) + + // TODO: this transaction is only RW because somewhere inside it we fetch images await db.knexReadWriteTransaction( (trx) => baker.bakeAll(trx), db.TransactionCloseMode.Close diff --git a/baker/runBakeGraphers.ts b/baker/runBakeGraphers.ts index eda4ada9c37..898a4a1a78d 100755 --- a/baker/runBakeGraphers.ts +++ b/baker/runBakeGraphers.ts @@ -9,6 +9,7 @@ import * as db from "../db/db.js" */ const main = async (folder: string) => { + // TODO: this transaction is only RW because somewhere inside it we fetch images return db.knexReadWriteTransaction( (trx) => bakeAllChangedGrapherPagesVariablesPngSvgAndDeleteRemovedGraphers( diff --git a/baker/siteRenderers.tsx b/baker/siteRenderers.tsx index b9128a9f83b..9138f3c069a 100644 --- a/baker/siteRenderers.tsx +++ b/baker/siteRenderers.tsx @@ -189,6 +189,7 @@ export function renderDynamicCollectionPage() { return renderToHtmlPage() } +// TODO: this transaction is only RW because somewhere inside it we fetch images export const renderGdocsPageBySlug = async ( knex: KnexReadWriteTransaction, slug: string, @@ -279,6 +280,7 @@ export const renderPost = async ( ) } +// TODO: this transaction is only RW because somewhere inside it we fetch images export const renderFrontPage = async (knex: KnexReadWriteTransaction) => { const gdocHomepageId = await getHomepageId(knex) @@ -296,6 +298,7 @@ export const renderFrontPage = async (knex: KnexReadWriteTransaction) => { } } +// TODO: this transaction is only RW because somewhere inside it we fetch images export const renderDonatePage = async (knex: KnexReadWriteTransaction) => { const faqsGdoc = (await getAndLoadGdocById( knex, @@ -336,6 +339,7 @@ export const renderDataInsightsIndexPage = ( ) } +// TODO: this transaction is only RW because somewhere inside it we fetch images export const renderBlogByPageNum = async ( pageNum: number, knex: KnexReadWriteTransaction @@ -364,6 +368,7 @@ export const renderSearchPage = () => export const renderNotFoundPage = () => renderToHtmlPage() +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function makeAtomFeed(knex: KnexReadWriteTransaction) { const posts = (await getBlogIndex(knex)).slice(0, 10) return makeAtomFeedFromPosts(posts) @@ -372,6 +377,8 @@ export async function makeAtomFeed(knex: KnexReadWriteTransaction) { // We don't want to include topic pages in the atom feed that is being consumed // by Mailchimp for sending the "immediate update" newsletter. Instead topic // pages announcements are sent out manually. + +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function makeAtomFeedNoTopicPages(knex: KnexReadWriteTransaction) { const posts = (await getBlogIndex(knex)) .filter((post: IndexPost) => post.type !== OwidGdocType.TopicPage) diff --git a/baker/sitemap.ts b/baker/sitemap.ts index 98c770b8fe7..544727f9562 100644 --- a/baker/sitemap.ts +++ b/baker/sitemap.ts @@ -62,6 +62,7 @@ const explorerToSitemapUrl = (program: ExplorerProgram): SitemapUrl[] => { } } +// TODO: this transaction is only RW because somewhere inside it we fetch images export const makeSitemap = async ( explorerAdminServer: ExplorerAdminServer, knex: db.KnexReadWriteTransaction diff --git a/baker/startDeployQueueServer.ts b/baker/startDeployQueueServer.ts index 499910adf0c..c21935859de 100644 --- a/baker/startDeployQueueServer.ts +++ b/baker/startDeployQueueServer.ts @@ -33,6 +33,7 @@ const main = async () => { setTimeout(deployIfQueueIsNotEmpty, 10 * 1000) }) + // TODO: this transaction is only RW because somewhere inside it we fetch images await db.knexReadWriteTransaction( deployIfQueueIsNotEmpty, db.TransactionCloseMode.Close diff --git a/db/model/Gdoc/GdocAuthor.ts b/db/model/Gdoc/GdocAuthor.ts index f5cbdc5083c..2fbca80e54c 100644 --- a/db/model/Gdoc/GdocAuthor.ts +++ b/db/model/Gdoc/GdocAuthor.ts @@ -40,12 +40,14 @@ export class GdocAuthor extends GdocBase implements OwidGdocAuthorInterface { return blocks } + // TODO: this transaction is only RW because somewhere inside it we fetch images _loadSubclassAttachments = ( knex: db.KnexReadWriteTransaction ): Promise => { return this.loadLatestWorkImages(knex) } + // TODO: this transaction is only RW because somewhere inside it we fetch images loadLatestWorkImages = async ( knex: db.KnexReadWriteTransaction ): Promise => { @@ -122,6 +124,7 @@ export class GdocAuthor extends GdocBase implements OwidGdocAuthorInterface { return errors } + // TODO: this transaction is only RW because somewhere inside it we fetch images static async getPublishedAuthors( knex: db.KnexReadWriteTransaction ): Promise { diff --git a/db/model/Gdoc/GdocBase.ts b/db/model/Gdoc/GdocBase.ts index 7fa892207b1..245c4b5a461 100644 --- a/db/model/Gdoc/GdocBase.ts +++ b/db/model/Gdoc/GdocBase.ts @@ -85,6 +85,8 @@ export class GdocBase implements OwidGdocBaseInterface { gdoc: typeof this ) => Promise = async () => [] _omittableFields: string[] = [] + + // TODO: this transaction is only RW because somewhere inside it we fetch images _loadSubclassAttachments: ( knex: db.KnexReadWriteTransaction ) => Promise = async () => undefined @@ -673,6 +675,7 @@ export class GdocBase implements OwidGdocBaseInterface { this.linkedDocuments = keyBy(linkedDocuments, "id") } + // TODO: this transaction is only RW because somewhere inside it we fetch images async loadImageMetadata( knex: db.KnexReadWriteTransaction, filenames?: string[] @@ -815,6 +818,7 @@ export class GdocBase implements OwidGdocBaseInterface { ] } + // TODO: this transaction is only RW because somewhere inside it we fetch images async loadState(knex: db.KnexReadWriteTransaction): Promise { await this.loadLinkedDocuments(knex) await this.loadImageMetadata(knex) diff --git a/db/model/Gdoc/GdocDataInsight.ts b/db/model/Gdoc/GdocDataInsight.ts index 4934986148f..cb7ffb42a33 100644 --- a/db/model/Gdoc/GdocDataInsight.ts +++ b/db/model/Gdoc/GdocDataInsight.ts @@ -58,6 +58,7 @@ export class GdocDataInsight this.latestDataInsights = await db.getPublishedDataInsights(knex, 5) } + // TODO: this transaction is only RW because somewhere inside it we fetch images static async getPublishedDataInsights( knex: db.KnexReadWriteTransaction, page?: number diff --git a/db/model/Gdoc/GdocFactory.ts b/db/model/Gdoc/GdocFactory.ts index 6afafaf9503..d1c9bf82da2 100644 --- a/db/model/Gdoc/GdocFactory.ts +++ b/db/model/Gdoc/GdocFactory.ts @@ -256,6 +256,7 @@ export async function getGdocBaseObjectBySlug( return gdoc } +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function getAndLoadGdocBySlug( knex: KnexReadWriteTransaction, slug: string @@ -269,6 +270,7 @@ export async function getAndLoadGdocBySlug( return loadGdocFromGdocBase(knex, base) } +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function getAndLoadGdocById( knex: KnexReadWriteTransaction, id: string, @@ -282,6 +284,8 @@ export async function getAndLoadGdocById( // From an ID, get a Gdoc object with all its metadata and state loaded, in its correct subclass. // If contentSource is Gdocs, use live data from Google, otherwise use the data in the DB. + +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function loadGdocFromGdocBase( knex: KnexReadWriteTransaction, base: OwidGdocBaseInterface, @@ -324,6 +328,7 @@ export async function loadGdocFromGdocBase( return gdoc } +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function getAndLoadPublishedDataInsights( knex: KnexReadWriteTransaction, page?: number @@ -369,6 +374,7 @@ export async function getAndLoadPublishedDataInsights( return gdocs as GdocDataInsight[] } +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function getAndLoadPublishedGdocPosts( knex: KnexReadWriteTransaction ): Promise { @@ -414,6 +420,7 @@ export async function getAndLoadPublishedGdocPosts( return gdocs as GdocPost[] } +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function loadPublishedGdocAuthors( knex: KnexReadWriteTransaction ): Promise { @@ -440,6 +447,7 @@ export async function loadPublishedGdocAuthors( return gdocs as GdocAuthor[] } +// TODO: this transaction is only RW because somewhere inside it we fetch images export async function getAndLoadListedGdocPosts( knex: KnexReadWriteTransaction ): Promise { diff --git a/db/model/Gdoc/GdocPost.ts b/db/model/Gdoc/GdocPost.ts index 3ffa58b731b..b77254be481 100644 --- a/db/model/Gdoc/GdocPost.ts +++ b/db/model/Gdoc/GdocPost.ts @@ -236,6 +236,7 @@ export class GdocPost extends GdocBase implements OwidGdocPostInterface { return parseDetails(gdoc.content.details) } + // TODO: this transaction is only RW because somewhere inside it we fetch images static async getPublishedGdocPosts( knex: KnexReadWriteTransaction ): Promise { diff --git a/db/model/Post.ts b/db/model/Post.ts index 1ed5ef2ba0c..83038876fc5 100644 --- a/db/model/Post.ts +++ b/db/model/Post.ts @@ -262,6 +262,7 @@ export const getFullPost = async ( const selectHomepagePosts: FilterFnPostRestApi = (post) => post.meta?.owid_publication_context_meta_field?.homepage === true +// TODO: this transaction is only RW because somewhere inside it we fetch images export const getBlogIndex = memoize( async (knex: db.KnexReadWriteTransaction): Promise => { const gdocPosts = await getAndLoadListedGdocPosts(knex) diff --git a/devTools/markdownTest/markdown.ts b/devTools/markdownTest/markdown.ts index c5f4c541228..141ee0ebab1 100644 --- a/devTools/markdownTest/markdown.ts +++ b/devTools/markdownTest/markdown.ts @@ -15,6 +15,7 @@ import { getAndLoadGdocBySlug } from "../../db/model/Gdoc/GdocFactory.js" async function main(parsedArgs: parseArgs.ParsedArgs) { try { + // TODO: this transaction is only RW because somewhere inside it we fetch images await knexReadWriteTransaction(async (trx) => { const gdoc = await getAndLoadGdocBySlug(trx, parsedArgs._[0]) let archieMlContent: OwidEnrichedGdocBlock[] | null