Merge pull request #2791 from owid/fix-fonts-in-exports-v2

🎉 (fonts): expand Playfair Display's character map to include super/subscripts & fix PNG exports

devTools/fonts/Makefile

@@ -2,14 +2,15 @@ VENV := .env
 PYTHON := $(VENV)/bin/python3
 PIP := $(VENV)/bin/pip3
 SUBSET := $(VENV)/bin/pyftsubset
-PRETTIER := node ../../node_modules/prettier/bin-prettier.js
+PRETTIER := node ../../node_modules/.bin/prettier
 .PHONY: all install report test clean
 
 default: all
 BUILD_DIR := $(VENV)/build
 FONTS_DIR := ../../public/fonts
 FONTS_CSS := ../../public/fonts.css
 FACES = $(BUILD_DIR)/fonts.css
+EMBEDS = $(BUILD_DIR)/embedded.css
 
 LATO_WOFFS = $(wildcard $(LATO)/Lato-*.woff2)
 LATO_LATIN_WOFFS = $(subst Lato-,LatoLatin-,$(LATO_WOFFS))
@@ -35,13 +36,15 @@ LatoLatin-%: Lato-%
 PLAYFAIR := $(BUILD_DIR)/playfair
 PLAYFAIR_URL := https://fonts.google.com/download/list?family=Playfair%20Display
 $(PLAYFAIR).tsv:
-	curl -sL $(PLAYFAIR_URL) |  tail -n +2 | \
+	curl -sL $(PLAYFAIR_URL) | tail -n +2 | \
 	jq -r '.manifest.fileRefs[] | select(.filename | contains("Variable") | not) | [.url, .filename] | @tsv' \
 	> $@
 $(PLAYFAIR): $(PLAYFAIR).tsv
 	mkdir -p $@
 	@cat $< | while IFS=$$'\t' read -r url filename; do \
-		curl "$$url" -o "$@/$$(basename $$filename)"; \
+		FONT="$@/$$(basename $$filename)"; \
+		curl -# "$$url" -o "$$FONT"; \
+		$(PYTHON) fix-numerals.py "$$FONT"; \
 	done
 PlayfairDisplay-%.woff2: PlayfairDisplay-%.ttf
 	$(SUBSET) $< --unicodes="*" --flavor=woff2 --layout-features+=$(OT_FEATURES) --name-IDs="*" --output-file="$@"
@@ -56,9 +59,13 @@ $(VENV):
 $(FACES): $(VENV) $(FONTS)
 	@$(PYTHON) make-faces.py $(FONTS) | $(PRETTIER) --parser css > $@
 
+$(EMBEDS): $(VENV) $(FONTS)
+	@$(PYTHON) make-faces.py --embed $(FONTS) | $(PRETTIER) --parser css > $@
+
 all: $(VENV) $(LATO) $(PLAYFAIR)
-	rm -f $(FACES)
+	rm -f $(FACES) $(EMBEDS)
 	$(MAKE) -j8 $(FACES)
+	$(MAKE) $(EMBEDS)
 	@for font in $(LATO)/*.woff2 $(PLAYFAIR)/*.woff2; do \
 	    diff -q "$$font" $(FONTS_DIR)/`basename "$$font"` || true; \
 	done
@@ -82,6 +89,7 @@ test:
 install: $(FONTS) $(FACES)
 	mkdir -p $(FONTS_DIR)
 	cp $(FONTS) $(FONTS_DIR)
+	cp $(EMBEDS) $(FONTS_DIR)
 	cp $(FACES) $(FONTS_CSS)
 
 clean:

devTools/fonts/README.md

@@ -19,6 +19,7 @@ You can also run `make report` to display the current division of characters bet
 
 -   Python3 (in order to use the [`fontTools.ttLib`](https://pypi.org/project/fonttools/) module for converting otf to woff2, creating the LatoLatin & PlayfairLatin fonts with its [subset](https://fonttools.readthedocs.io/en/latest/subset/index.html) tool, and unpacking the `cmap` table to calculate `unicode-range` settings for switching between the subset and full version in the browser)
 -   the repo's copies of prettier and express in `../../node_modules`
+-   the `jq` command line tool
 
 ## Typefaces
 

devTools/fonts/fix-numerals.py

@@ -0,0 +1,72 @@
+#!./env/bin/python3
+"""
+This script performs an in-place modification of the specified font file to insert missing
+character-map entries for superscript and subscript numerals. It's intended to be used on the
+PlayfairDisplay TTFs downloaded from google fonts since they lack those mappings. If invoked with
+the filename "-" it will read from stdin and write the modified font to stdout.
+"""
+from subprocess import run
+from os.path import exists, dirname, abspath
+import sys
+import re
+
+TTX = f'{dirname(abspath(__file__))}/.env/bin/ttx'
+
+# unicode codepoint -> glyph name mappings
+scripts = {
+    # subscripts
+    0x2080: "zero.subs",
+    0x2081: "one.subs",
+    0x2082: "two.subs",
+    0x2083: "three.subs",
+    0x2084: "four.subs",
+    0x2085: "five.subs",
+    0x2086: "six.subs",
+    0x2087: "seven.subs",
+    0x2088: "eight.subs",
+    0x2089: "nine.subs",
+
+    # superscripts
+    0x2070: "zero.sups",
+    0x00B9: "uni00B9", # one
+    0x00B2: "uni00B2", # two
+    0x00B3: "uni00B3", # three
+    0x2074: "four.sups",
+    0x2075: "five.sups",
+    0x2076: "six.sups",
+    0x2077: "seven.sups",
+    0x2078: "eight.sups",
+    0x2079: "nine.sups",
+}
+
+def update_cmap(path):
+    pipe_input = sys.stdin.buffer.read() if path=="-" else None
+    if not exists(path) and not pipe_input:
+        print("No such file:", path, file=sys.stderr)
+        sys.exit(1)
+
+    # decompile the TTF
+    print(f"Updating character tables in {path}...", end=' ', flush=True, file=sys.stderr)
+    ttx_orig = run([TTX, '-o', '-', path], capture_output=True, input=pipe_input).stdout.decode('utf-8')
+
+    # bail out if this font has already been modified
+    for m in re.findall(r'<cmap_format_\d.*?</cmap_format_\d>', ttx_orig, re.DOTALL):
+        if 'zero.subs' in m:
+            print("(already contains super/subscript definitions)", file=sys.stderr)
+            sys.exit(0)
+
+    # add missing definitions to all cmap tables in the font
+    cmap_additions = "".join([
+        f'<map code="{uni:#x}" name="{name}"/>' for uni, name in scripts.items()
+    ])
+    ttx_modified = re.sub(r'(</cmap_format_\d>)', cmap_additions + r'\1', ttx_orig).encode('utf-8')
+
+    # compile the updated font and overwrite the existing file
+    run([TTX, '-q', '-o', path, '-'], input=ttx_modified)
+    print("(done)", file=sys.stderr)
+
+if __name__ == "__main__":
+    try:
+        update_cmap(*sys.argv[1:2])
+    except TypeError:
+        print("Usage: fix-numerals.py <path-to-font>")

devTools/fonts/make-faces.py

@@ -16,6 +16,7 @@
 from more_itertools import consecutive_groups
 from operator import itemgetter
 from os.path import basename, splitext
+from base64 import b64encode
 import sys
 import re
 
@@ -52,7 +53,8 @@ def inspect_font(path):
         italic = italic,
         style = "italic" if italic else "normal",
         codepoints = codepoints,
-        url = f'/fonts/{basename(path)}'
+        url = f'/fonts/{basename(path)}',
+        path = path,
     )
 
 def find_ranges(iterable):
@@ -90,7 +92,7 @@ def make_face(font, subset=None, singleton=False):
 
     return "\n".join(css)
 
-def main(woff_files):
+def main_stylesheet(woff_files):
     fonts = sorted([inspect_font(f) for f in woff_files], key=itemgetter('weight', 'italic'))
     lato = [f for f in fonts if f['family']=='Lato' and not f['subset']]
     lato_latin = [f for f in fonts if f['family']=='Lato' and f['subset']]
@@ -111,6 +113,45 @@ def main(woff_files):
     ]
     print("\n\n".join(faces))
 
+def make_embedded_face(font):
+    font_data = open(font['path'], 'rb').read()
+    font_uri = 'data:font/woff2;base64,' + b64encode(font_data).decode('utf-8')
+    family = font['family']
+    weight = font['weight']
+    style = font['style']
+
+    css = [
+        '@font-face {',
+        'font-display: block;',
+        f'font-family: "{family}";',
+        f'font-weight: {weight};',
+        f'font-style: {style};',
+        f'src: url({font_uri}) format("woff2");',
+        "}",
+    ]
+    return " ".join(css)
+
+def embedded_stylesheet(woff_files):
+    font_info = [inspect_font(f) for f in woff_files]
+
+    # include just the fonts known to be used in static chart exports
+    embeddable_fonts =[
+        'Lato-Regular',
+        'Lato-Italic',
+        'Lato-Bold',
+        'PlayfairDisplay-Medium',
+    ]
+
+    # use the latin subsets to keep the size down
+    faces = [make_embedded_face(f) for f in font_info if f['subset'] and f['ps_name'] in embeddable_fonts]
+    print("\n".join(faces))
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    args = sys.argv[1:]
+    if '--embed' in args[:1]:
+        embedded_stylesheet(args[1:])
+    elif args:
+        main_stylesheet(args)
+    else:
+        print("Usage: make-faces.py [--embed] woff2-files...")
+

devTools/fonts/test.js

@@ -31,6 +31,8 @@ const FAMILIES = {
 }
 
 const SAMPLE_TEXT =
+    // `x<sup>0</sup>x<sup>1</sup>x<sup>2</sup>x<sup>3</sup>x<sup>4</sup>x<sup>5</sup>x<sup>6</sup>x<sup>7</sup>x<sup>8</sup>x<sup>9</sup>x<sub>0</sub>x<sub>1</sub>x<sub>2</sub>x<sub>3</sub>x<sub>4</sub>x<sub>5</sub>x<sub>6</sub>x<sub>7</sub>x<sub>8</sub>x<sub>9</sub>`
+    // `x⁰x¹x²x³x⁴x⁵x⁶x⁷x⁸x⁹x₀x₁x₂x₃x₄x₅x₆x₇x₈x₉`
     "hamburgefonstiv 0123<sub>456</sub><sup>789</sup> ←↑→↓↔↕↖↗↘↙"
 
 function pangram(family, postscriptNames = false) {

packages/@ourworldindata/components/src/MarkdownTextWrap/MarkdownTextWrap.tsx

@@ -8,11 +8,25 @@ import {
     sumBy,
     imemo,
     max,
+    get,
     Bounds,
     FontFamily,
 } from "@ourworldindata/utils"
 import { TextWrap } from "../TextWrap/TextWrap.js"
 
+const SUPERSCRIPT_NUMERALS = {
+    "0": "\u2070",
+    "1": "\u00b9",
+    "2": "\u00b2",
+    "3": "\u00b3",
+    "4": "\u2074",
+    "5": "\u2075",
+    "6": "\u2076",
+    "7": "\u2077",
+    "8": "\u2078",
+    "9": "\u2079",
+}
+
 export interface IRFontParams {
     fontSize?: number
     fontWeight?: number
@@ -208,23 +222,14 @@ export class IRSuperscript implements IRToken {
         return <sup key={key}>{this.text}</sup>
     }
     toSVG(key?: React.Key): JSX.Element {
+        // replace numerals with literals, for everything else let the font-feature handle it
+        const style = { fontFeatureSettings: '"sups"' }
+        const text = this.text.replace(/./g, (c) =>
+            get(SUPERSCRIPT_NUMERALS, c, c)
+        )
         return (
             <React.Fragment key={key}>
-                <tspan
-                    style={{
-                        fontSize: this.height / 2,
-                    }}
-                    dy={-this.height / 3}
-                >
-                    {this.text}
-                </tspan>
-                {/*
-                    can't use baseline-shift as it's not supported in firefox
-                    can't use transform translations on tspans
-                    so we use dy translations but they apply to all subsequent elements
-                    so we need a "reset" element to counteract each time
-                 */}
-                <tspan dy={this.height / 3}> </tspan>
+                <tspan style={style}>{text}</tspan>
             </React.Fragment>
         )
     }