pdf build workflows

.github/workflows/build-pages.yml

@@ -0,0 +1,94 @@
+name: Build pages
+on:
+  workflow_dispatch:
+    inputs:
+      ref: 
+        description: Ref name to build
+        required: true
+        type: string
+      sha:
+        description: SHA, should correspond to ref
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      ref: 
+        description: Ref name to build
+        required: true
+        type: string
+      sha:
+        description: SHA, should correspond to ref
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: docs-16c-64gb-600gb
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.sha }}
+          fetch-depth: 0 # fetch whole repo so git-restore-mtime can work
+          lfs: true
+
+      - name: Adjust file watchers limit
+        run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "npm"
+          cache-dependency-path: "package-lock.json"
+        env:
+          NODE_ENV: ${{ secrets.NODE_ENV }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Install dependencies
+        run: |
+          npm run presetup
+          npm ci --ignore-scripts
+        env:
+          NODE_ENV: ${{ secrets.NODE_ENV }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+      - name: Run NPM install scripts
+        run: |
+          npm rebuild
+          
+      - name: Checking Gatsby cache
+        id: gatsby-cache-build
+        uses: actions/cache@v4
+        with:
+          path: |
+            public/*
+            !public/pdfs
+            .cache
+          key: ${{ runner.os }}-gatsby-build-${{ hashFiles('package.json', 'gatsby-config.js', 'gatsby-node.js') }}-${{ inputs.ref }}-${{ inputs.sha }}
+          restore-keys: |
+            ${{ runner.os }}-gatsby-build-${{ hashFiles('package.json', 'gatsby-config.js', 'gatsby-node.js') }}-${{ inputs.ref }}
+            ${{ runner.os }}-gatsby-build-${{ hashFiles('package.json', 'gatsby-config.js', 'gatsby-node.js') }}
+            ${{ runner.os }}-gatsby-build
+
+      - name: Fix mtimes
+        run: npm run fix-mtimes
+
+      - name: Gatsby build
+        run: npm run build
+        env:
+          APP_ENV: staging
+          NODE_ENV: ${{ vars.NODE_ENV }}
+          NODE_OPTIONS: --max-old-space-size=4096
+          FATHOM_SITE_ID: ${{ vars.FATHOM_SITE_ID }}
+          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
+          ALGOLIA_SEARCH_ONLY_KEY: ${{ vars.ALGOLIA_SEARCH_ONLY_KEY }}
+          ALGOLIA_APP_ID: ${{ vars.ALGOLIA_APP_ID }}
+          ALGOLIA_INDEX_NAME: edb-docs-staging
+          INDEX_ON_BUILD: ${{ inputs.ref == 'develop' }}
+
+      - name: Store pages
+        uses: actions/upload-artifact@v4
+        with: 
+          name: pages-build
+          path: |
+            public/*
+            !public/pdfs

.github/workflows/build-pdfs.yml

@@ -0,0 +1,91 @@
+name: Generate PDFs
+on:
+  workflow_dispatch:
+    inputs:
+      ref: 
+        description: Ref name to build
+        required: true
+        type: string
+      sha:
+        description: SHA, should correspond to ref
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      ref: 
+        description: Ref name to build
+        required: true
+        type: string
+      sha:
+        description: SHA, should correspond to ref
+        required: true
+        type: string
+
+jobs:
+  build:
+    runs-on: docs-16c-64gb-600gb
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.sha }}
+          lfs: true
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: "npm"
+          cache-dependency-path: "package-lock.json"
+        env:
+          NODE_ENV: ${{ vars.NODE_ENV }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.6"
+
+      - uses: r-lib/actions/setup-pandoc@v2
+        with:
+          pandoc-version: "2.14.1"
+
+      - name: Install wkhtmltopdf
+        run: |
+          curl -L https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-3/wkhtmltox_0.12.6.1-3.jammy_amd64.deb > wkhtmltopdf.deb
+          sudo apt update
+          sudo apt install -y ./wkhtmltopdf.deb
+          sudo apt install -y rsync
+
+      - name: Install Python dependencies
+        run: pip install -r requirements-ci.txt
+
+      - name: Install PDF Node dependencies
+        working-directory: ./scripts/pdf
+        run: |
+          npm install
+
+      - name: Checking PDF cache
+        id: pdf-cache-build
+        uses: actions/cache@v4
+        with:
+          path: |
+            product_docs/**/*.pdf
+            product_docs/**/*.pdf-hash
+            advocacy_docs/**/*.pdf
+            advocacy_docs/**/*.pdf-hash
+          key: ${{ runner.os }}-build-pdfs-${{ hashFiles('scripts/pdf/*', 'scripts/pdf/lib/*') }}-${{ inputs.sha }}
+          restore-keys: |
+            ${{ runner.os }}-build-pdfs-${{ hashFiles('scripts/pdf/*', 'scripts/pdf/lib/*') }}  
+        
+      - name: Build all pdfs
+        run: npm run pdf:build-all-ci
+
+      - name: Copy pdfs to build output
+        run: |
+          mkdir -p public/pdfs
+          shopt -s globstar
+          rsync -avm --filter="+ */" --filter="-! *.pdf" advocacy_docs/ public/pdfs/
+          rsync -avm --filter="+ */" --filter="-! *.pdf" product_docs/docs/ public/pdfs/
+
+      - name: Store pdfs
+        uses: actions/upload-artifact@v4
+        with: 
+          name: pdf-build
+          path: public/*

.github/workflows/deploy-draft.yml

@@ -9,90 +9,49 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-deploy:
+  build:
     if: |
       (
         (github.event.action == 'labeled' && github.event.label.name == 'deploy') ||
         (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'deploy'))
       ) && !github.event.pull_request.head.repo.fork
-    runs-on: docs-16c-64gb-600gb
-    steps:
-      - name: inject slug/short variables
-        uses: rlespinasse/github-slug-action@v4
-
-      - name: compose a name for the build environment
-        run: echo "BUILD_ENV_NAME=pr-${{ github.event.number }}-${{ env.GITHUB_HEAD_REF_SLUG }}" >> $GITHUB_ENV
-
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0 # fetch whole repo so git-restore-mtime can work
-          lfs: true
-
-      - name: Adjust file watchers limit
-        run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version-file: ".nvmrc"
-          cache: "npm"
-          cache-dependency-path: "package-lock.json"
-        env:
-          NODE_ENV: ${{ secrets.NODE_ENV }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-
-      - name: Install dependencies
-        run: |
-          npm run presetup
-          npm ci --ignore-scripts
-        env:
-          NODE_ENV: ${{ secrets.NODE_ENV }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+    uses: ./.github/workflows/build-pages.yml
+    with:
+      ref: ${{ github.head_ref }}
+      sha: ${{ github.event.pull_request.head.sha }}
+    secrets: inherit
 
-      - name: Run NPM install scripts
+  build-pdfs:
+    if: |
+      (
+        (github.event.action == 'labeled' && github.event.label.name == 'deploy-pdfs') ||
+        (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'deploy-pdfs'))
+      ) && !github.event.pull_request.head.repo.fork
+    uses: ./.github/workflows/build-pdfs.yml
+    with:
+      ref: ${{ github.head_ref }}
+      sha: ${{ github.event.pull_request.head.sha }}
+
+  # this serves to allow building PDFs to be optional without preventing deployment, 
+  # while still failing the workflow if we *try* to build PDFs and fail
+  # there are ways to do this in conditions on a single job, but this keeps things clear
+  # this job will fail only if build-pdfs failed (not if it was skipped)
+  # however, it will set an output - check-has-pdfs.steps.check.has-pdfs - true if PDFs were generated
+  check-has-pdfs:
+    needs: build-pdfs
+    if: ${{ always() }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: check
         run: |
-          npm rebuild
-          
-      - name: Checking Gatsby cache
-        id: gatsby-cache-build
-        uses: actions/cache@v4
-        with:
-          path: |
-            public/*
-            !public/pdfs
-            .cache
-          key: ${{ runner.os }}-gatsby-build-draft-${{ github.head_ref }}-${{ hashFiles('package.json', 'gatsby-config.js', 'gatsby-node.js') }}-${{ github.sha }}
-          restore-keys: |
-            ${{ runner.os }}-gatsby-build-draft-${{ github.head_ref }}
-            ${{ runner.os }}-gatsby-build-develop-${{ hashFiles('package.json', 'gatsby-config.js', 'gatsby-node.js') }}
-            ${{ runner.os }}-gatsby-build-develop
-
-      - name: Fix mtimes
-        run: npm run fix-mtimes
-
-      - name: Gatsby build
-        run: npm run build
-        env:
-          APP_ENV: staging
-          NODE_ENV: ${{ vars.NODE_ENV }}
-          NODE_OPTIONS: --max-old-space-size=4096
-          FATHOM_SITE_ID: ${{ vars.FATHOM_SITE_ID }}
-          ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
-          ALGOLIA_SEARCH_ONLY_KEY: ${{ vars.ALGOLIA_SEARCH_ONLY_KEY }}
-          ALGOLIA_APP_ID: ${{ vars.ALGOLIA_APP_ID }}
-          ALGOLIA_INDEX_NAME: edb-docs-staging
-          INDEX_ON_BUILD: false
-
-      - name: Deploy to Netlify
-        id: netlify-deploy
-        uses: nwtgck/actions-netlify@v3
-        with:
-          publish-dir: "./public"
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          deploy-message: ${{ github.event.pull_request.title }}
-          enable-commit-comment: false
-          github-deployment-environment: ${{ env.BUILD_ENV_NAME }}
-          alias: deploy-preview-${{ github.event.number }}
-        env:
-          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
-          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_DEVELOP_SITE_ID }}
+          echo "has-pdfs=${{ needs.build-pdfs.result == 'success' }}" >> "$GITHUB_OUTPUT"
+          exit ${{ needs.build-pdfs.result == 'failure' && 1 || 0 }}
+  
+  deploy:
+    needs: [build, check-has-pdfs]
+    # succeeded() (the default check) will be false at this point if PDFs were skipped
+    if: ${{ !cancelled() && needs.build.result == 'success' && needs.check-has-pdfs.result == 'success' }}
+    uses: ./.github/workflows/deploy-to-netlify.yml
+    with:
+      enable-commit-comment: false
+    secrets: inherit

.github/workflows/deploy-to-netlify.yml

@@ -0,0 +1,41 @@
+name: Deploy build to Netlify
+on:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      enable-pull-request-comment: 
+        description: whether to comment on the triggering PR
+        required: false
+        default: true
+        type: boolean
+      enable-commit-comment: 
+        description: whether to comment on the triggering commit
+        required: false
+        default: true
+        type: boolean
+
+jobs:
+  deploy:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Pull build artifacts
+        uses: actions/download-artifact@v4
+        with: 
+          path: ./public
+          merge-multiple: true
+
+      - name: list
+        run: ls -lfR ./public
+
+      - name: Deploy to Netlify
+        id: netlify-deploy
+        if: false
+        uses: nwtgck/actions-netlify@v3
+        with:
+          publish-dir: "./public"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          enable-pull-request-comment: ${{ inputs.enable-pull-request-comment }}
+          enable-commit-comment: ${{ inputs.enable-commit-comment }}
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          NETLIFY_SITE_ID: ${{ secrets.NETLIFY_DEVELOP_SITE_ID }}

.github/workflows/sync-and-process-files.yml

@@ -67,10 +67,7 @@ jobs:
       - name: setup node
         uses: actions/setup-node@v4
         with:
-          node-version: "18"
-
-      - name: update npm
-        run: npm install -g npm@10
+          node-version-file: "destination/.nvmrc"
 
       - name: Process changes
         id: changes

.nvmrc

@@ -1 +1 @@
-18
+22

README.md

@@ -120,7 +120,7 @@ These instructions are for members of the EnterpriseDB Github Org only. The icon
 
    - If you already have a different version of Node installed, you may want to consider using Node Version Manager (NVM) for a simpler way to manage multiple versions of Node.js. Follow the [directions to install NVM](https://github.com/nvm-sh/nvm#installing-and-updating), then run `nvm install` in the cloned repo directory, followed by `nvm use` which will auto-detect the correct version of Node.js to use (currently 20 LTS).
 
-1. Install Python 3 with `brew install python3`, if it's not already installed. (Use `python3 -V` to check that you have version 3.8 or higher.) Python is not needed for the core Gatsby system, but is required by several source scripts.
+1. Install Python 3 with `brew install python3`, if it's not already installed. (Use `python3 -V` to check that you have version 3.11 or higher.) Python is not needed for the core Gatsby system, but is required by several source scripts.
 
 1. NPM 10 is the package manager we're using for this project.