fix(tsx): Collect multibytes characters in skipped script and styles (#…

…1033) * fix(tsx): Collect multibytes characters in skipped script and styles * chore: changeset
withastro · Jul 31, 2024 · af471f5 · af471f5
1 parent 0ebfc96
commit af471f5
Show file tree

Hide file tree

Showing 4 changed files with 94 additions and 16 deletions.
diff --git a/.changeset/two-ghosts-fly.md b/.changeset/two-ghosts-fly.md
@@ -0,0 +1,5 @@
+---
+"@astrojs/compiler": patch
+---
+
+Fixes positions for extracted tags being wrong when using IncludeStyles and IncludeScripts set to false
diff --git a/internal/printer/print-to-tsx.go b/internal/printer/print-to-tsx.go
@@ -456,6 +456,8 @@ declare const Astro: Readonly<import('astro').AstroGlobal<%s, typeof %s`, propsI
 				p.printTextWithSourcemap(n.Data, n.Loc[0])
 				p.addNilSourceMapping()
 				p.print("}}\n")
+			} else {
+				p.collectMultiByteCharacters(n.Data)
 			}
 			p.addSourceMapping(loc.Loc{Start: n.Loc[0].Start + len(n.Data)})
 		} else if textType == StyleText || textType == JsonScriptText || textType == RawText || textType == UnknownScriptText {
@@ -465,6 +467,8 @@ declare const Astro: Readonly<import('astro').AstroGlobal<%s, typeof %s`, propsI
 				p.printTextWithSourcemap(escapeText(n.Data), n.Loc[0])
 				p.addNilSourceMapping()
 				p.print("`}")
+			} else {
+				p.collectMultiByteCharacters(n.Data)
 			}
 			p.addSourceMapping(loc.Loc{Start: n.Loc[0].Start + len(n.Data)})
 		} else {
@@ -768,6 +772,14 @@ declare const Astro: Readonly<import('astro').AstroGlobal<%s, typeof %s`, propsI
 		}
 	}
 
+	if len(n.Loc) > 1 {
+		endLoc = n.Loc[1].Start - 2
+	} else if n.LastChild != nil && n.LastChild.Expression {
+		if len(n.LastChild.Loc) > 1 {
+			endLoc = n.LastChild.Loc[1].Start + 1
+		}
+	}
+
 	if n.FirstChild != nil && (n.DataAtom == atom.Script || n.DataAtom == atom.Style) {
 		if n.DataAtom == atom.Script {
 			p.addTSXScript(startTagEnd, endLoc-p.bytesToSkip, n.FirstChild.Data, getScriptTypeFromAttrs(n.Attr))
@@ -783,13 +795,6 @@ declare const Astro: Readonly<import('astro').AstroGlobal<%s, typeof %s`, propsI
 		return
 	}
 
-	if len(n.Loc) > 1 {
-		endLoc = n.Loc[1].Start - 2
-	} else if n.LastChild != nil && n.LastChild.Expression {
-		if len(n.LastChild.Loc) > 1 {
-			endLoc = n.LastChild.Loc[1].Start + 1
-		}
-	}
 	p.addSourceMapping(loc.Loc{Start: endLoc})
 	p.print("</")
 	if !isSelfClosing {

diff --git a/internal/printer/printer.go b/internal/printer/printer.go
@@ -159,6 +159,16 @@ func (p *printer) printEscapedJSXTextWithSourcemap(text string, l loc.Loc) {
 	}
 }
 
+// We normally collect multi-byte characters while printing, but this method can be used for skipped text
+func (p *printer) collectMultiByteCharacters(text string) {
+	for pos := range text {
+		_, nextCharByteSize := utf8.DecodeRuneInString(text[pos:])
+		if nextCharByteSize > 1 {
+			p.bytesToSkip += nextCharByteSize - 1
+		}
+	}
+}
+
 func (p *printer) printInternalImports(importSpecifier string, opts *RenderOptions) {
 	if p.hasInternalImports {
 		return

diff --git a/packages/compiler/test/tsx/meta.ts b/packages/compiler/test/tsx/meta.ts
@@ -48,7 +48,7 @@ test('extract scripts', async () => {
 			{
 				position: {
 					start: 22,
-					end: 87,
+					end: 54,
 				},
 				type: 'module',
 				content: 'console.log({ test: `literal` })',
@@ -57,7 +57,7 @@ test('extract scripts', async () => {
 			{
 				position: {
 					start: 93,
-					end: 158,
+					end: 125,
 				},
 				type: 'inline',
 				content: 'console.log({ test: `literal` })',
@@ -66,7 +66,7 @@ test('extract scripts', async () => {
 			{
 				position: {
 					start: 169,
-					end: 188,
+					end: 178,
 				},
 				type: 'json',
 				content: '{"a":"b"}',
@@ -75,7 +75,7 @@ test('extract scripts', async () => {
 			{
 				position: {
 					start: 205,
-					end: 246,
+					end: 225,
 				},
 				type: 'inline',
 				content: 'console.log("hello")',
@@ -93,7 +93,7 @@ test('extract scripts', async () => {
 			{
 				position: {
 					start: 281,
-					end: 346,
+					end: 313,
 				},
 				type: 'processed-module',
 				content: 'console.log({ test: `literal` })',
@@ -102,7 +102,7 @@ test('extract scripts', async () => {
 			{
 				position: {
 					start: 337,
-					end: 358,
+					end: 347,
 				},
 				type: 'raw',
 				content: 'something;',
@@ -123,7 +123,7 @@ test('extract styles', async () => {
 			{
 				position: {
 					start: 7,
-					end: 48,
+					end: 27,
 				},
 				type: 'tag',
 				content: 'body { color: red; }',
@@ -141,7 +141,7 @@ test('extract styles', async () => {
 			{
 				position: {
 					start: 86,
-					end: 127,
+					end: 106,
 				},
 				type: 'tag',
 				content: 'body { color: red; }',
@@ -150,7 +150,7 @@ test('extract styles', async () => {
 			{
 				position: {
 					start: 133,
-					end: 174,
+					end: 153,
 				},
 				type: 'tag',
 				content: 'body { color: red; }',
@@ -161,4 +161,62 @@ test('extract styles', async () => {
 	);
 });
 
+test('extract scripts and styles with multibyte characters', async () => {
+	const scripts = "<script>console.log('🦄')</script><script>console.log('Hey');</script>";
+	const styles =
+		"<style>body { background: url('🦄.png'); }</style><style>body { background: url('Hey');";
+
+	const input = `${scripts}${styles}`;
+	const { metaRanges } = await convertToTSX(input, { sourcemap: 'external' });
+
+	assert.equal(
+		metaRanges.scripts,
+		[
+			{
+				position: {
+					start: 8,
+					end: 24,
+				},
+				type: 'processed-module',
+				content: "console.log('🦄')",
+				lang: '',
+			},
+			{
+				position: {
+					start: 41,
+					end: 60,
+				},
+				type: 'processed-module',
+				content: "console.log('Hey');",
+				lang: '',
+			},
+		],
+		'expected metaRanges.scripts to match snapshot'
+	);
+	assert.equal(
+		metaRanges.styles,
+		[
+			{
+				position: {
+					start: 76,
+					end: 110,
+				},
+				type: 'tag',
+				content: "body { background: url('🦄.png'); }",
+				lang: 'css',
+			},
+			{
+				position: {
+					start: 125,
+					end: 186,
+				},
+				type: 'tag',
+				content: "body { background: url('Hey');",
+				lang: 'css',
+			},
+		],
+		'expected metaRanges.styles to match snapshot'
+	);
+});
+
 test.run();