From 7a1bdfa02634d29f4020fad527c744aef37ac403 Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Thu, 28 Nov 2024 16:14:41 -0500 Subject: [PATCH 1/3] Account for FrontMatter when calculating sourcepos --- src/parser/mod.rs | 23 +++++ src/tests.rs | 1 + src/tests/front_matter.rs | 181 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 src/tests/front_matter.rs diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 2c233981..69c1faf2 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1141,6 +1141,20 @@ impl<'a, 'o> Parser<'a, 'o> { &self.options.extension.front_matter_delimiter, ) { if let Some((front_matter, rest)) = split_off_front_matter(s, delimiter) { + let lines = front_matter + .as_bytes() + .iter() + .filter(|b| **b == b'\n') + .count(); + + let mut stripped_front_matter = front_matter.to_string(); + strings::remove_trailing_blank_lines(&mut stripped_front_matter); + let stripped_lines = stripped_front_matter + .as_bytes() + .iter() + .filter(|b| **b == b'\n') + .count(); + let node = self.add_child( self.root, NodeValue::FrontMatter(front_matter.to_string()), @@ -1148,6 +1162,15 @@ impl<'a, 'o> Parser<'a, 'o> { ); s = rest; self.finalize(node).unwrap(); + + node.data.borrow_mut().sourcepos = Sourcepos { + start: nodes::LineColumn { line: 1, column: 1 }, + end: nodes::LineColumn { + line: 1 + stripped_lines, + column: delimiter.chars().count(), + }, + }; + self.line_number += lines; } } diff --git a/src/tests.rs b/src/tests.rs index 575310be..9182b24d 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -12,6 +12,7 @@ mod description_lists; mod empty; mod escaped_char_spans; mod footnotes; +mod front_matter; mod fuzz; mod greentext; mod header_ids; diff --git a/src/tests/front_matter.rs b/src/tests/front_matter.rs new file mode 100644 index 00000000..4a2cb49e --- /dev/null +++ b/src/tests/front_matter.rs @@ -0,0 +1,181 @@ +use crate::{format_commonmark, parse_document, Arena, Options}; + +use super::*; + +#[test] +fn round_trip_one_field() { + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("---".to_owned()); + let arena = Arena::new(); + let input = "---\nlayout: post\n---\nText\n"; + let root = parse_document(&arena, input, &options); + let mut buf = Vec::new(); + format_commonmark(&root, &options, &mut buf).unwrap(); + assert_eq!(&String::from_utf8(buf).unwrap(), input); +} + +#[test] +fn round_trip_wide_delimiter() { + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("\u{04fc}".to_owned()); + let arena = Arena::new(); + let input = "\u{04fc}\nlayout: post\n\u{04fc}\nText\n"; + let root = parse_document(&arena, input, &options); + let mut buf = Vec::new(); + format_commonmark(&root, &options, &mut buf).unwrap(); + assert_eq!(&String::from_utf8(buf).unwrap(), input); +} + +#[test] +fn ast_wide_delimiter() { + let input = "\u{04fc}\nlayout: post\n\u{04fc}\nText\n"; + + assert_ast_match_i( + input, + ast!((document (1:1-4:4) [ + (frontmatter (1:1-3:1) []) + (paragraph (4:1-4:4) [ + (text (4:1-4:4) []) + ]) + ])), + |opts| opts.extension.front_matter_delimiter = Some("\u{04fc}".to_owned()), + ); +} + +#[test] +fn ast() { + let input = "q\nlayout: post\nq\nText\n"; + + assert_ast_match_i( + input, + ast!((document (1:1-4:4) [ + (frontmatter (1:1-3:1) []) + (paragraph (4:1-4:4) [ + (text (4:1-4:4) []) + ]) + ])), + |opts| opts.extension.front_matter_delimiter = Some("q".to_owned()), + ); +} + +#[test] +fn ast_blank_line() { + let input = r#"--- +a: b +--- + +hello world +"#; + + assert_ast_match_i( + input, + ast!((document (1:1-5:11) [ + (frontmatter (1:1-3:3) []) + (paragraph (5:1-5:11) [ + (text (5:1-5:11) []) + ]) + ])), + |opts| opts.extension.front_matter_delimiter = Some("---".to_owned()), + ); +} + +#[test] +fn ast_carriage_return() { + let input = "q\r\nlayout: post\r\nq\r\nText\r\n"; + + assert_ast_match_i( + input, + ast!((document (1:1-4:4) [ + (frontmatter (1:1-3:1) []) + (paragraph (4:1-4:4) [ + (text (4:1-4:4) []) + ]) + ])), + |opts| opts.extension.front_matter_delimiter = Some("q".to_owned()), + ); +} + +#[test] +fn trailing_space_open() { + let input = "--- \nlayout: post\n---\nText\n"; + + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("---".to_owned()); + let arena = Arena::new(); + let root = parse_document(&arena, input, &options); + + let found = root + .descendants() + .filter(|n| matches!(n.data.borrow().value, NodeValue::FrontMatter(..))) + .next(); + + assert!(found.is_none(), "no FrontMatter expected"); +} + +#[test] +fn leading_space_open() { + let input = " ---\nlayout: post\n---\nText\n"; + + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("---".to_owned()); + let arena = Arena::new(); + let root = parse_document(&arena, input, &options); + + let found = root + .descendants() + .filter(|n| matches!(n.data.borrow().value, NodeValue::FrontMatter(..))) + .next(); + + assert!(found.is_none(), "no FrontMatter expected"); +} + +#[test] +fn leading_space_close() { + let input = "---\nlayout: post\n ---\nText\n"; + + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("---".to_owned()); + let arena = Arena::new(); + let root = parse_document(&arena, input, &options); + + let found = root + .descendants() + .filter(|n| matches!(n.data.borrow().value, NodeValue::FrontMatter(..))) + .next(); + + assert!(found.is_none(), "no FrontMatter expected"); +} + +#[test] +fn trailing_space_close() { + let input = "---\nlayout: post\n--- \nText\n"; + + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("---".to_owned()); + let arena = Arena::new(); + let root = parse_document(&arena, input, &options); + + let found = root + .descendants() + .filter(|n| matches!(n.data.borrow().value, NodeValue::FrontMatter(..))) + .next(); + + assert!(found.is_none(), "no FrontMatter expected"); +} + +#[test] +fn second_line() { + let input = "\n---\nlayout: post\n ---\nText\n"; + + let mut options = Options::default(); + options.extension.front_matter_delimiter = Some("---".to_owned()); + let arena = Arena::new(); + let root = parse_document(&arena, input, &options); + + let found = root + .descendants() + .filter(|n| matches!(n.data.borrow().value, NodeValue::FrontMatter(..))) + .next(); + + assert!(found.is_none(), "no FrontMatter expected"); +} From 73925f5758dfae653d920c380242607fc48ca1c4 Mon Sep 17 00:00:00 2001 From: Sam Wilson Date: Thu, 28 Nov 2024 16:15:19 -0500 Subject: [PATCH 2/3] Make sourcepos! work in static contexts --- src/tests.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/tests.rs b/src/tests.rs index 9182b24d..e143709f 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -238,10 +238,19 @@ fn asssert_node_eq<'a>(node: &'a AstNode<'a>, location: &[usize], expected: &Nod macro_rules! sourcepos { (($spsl:literal:$spsc:literal-$spel:literal:$spec:literal)) => { - ($spsl, $spsc, $spel, $spec).into() + $crate::nodes::Sourcepos { + start: $crate::nodes::LineColumn { + line: $spsl, + column: $spsc, + }, + end: $crate::nodes::LineColumn { + line: $spel, + column: $spec, + }, + } }; ((XXX)) => { - (0, 1, 0, 1).into() + $crate::tests::sourcepos!((0:1-0:1)) }; } From 17cc1f11399b1e34d8769fd0dda4411d2b257e4a Mon Sep 17 00:00:00 2001 From: Asherah Connor Date: Sat, 30 Nov 2024 07:58:58 +1100 Subject: [PATCH 3/3] Use bytes for sourcepos columns. See discussion at https://github.com/kivikakk/comrak/pull/494#discussion_r1862701264. --- src/parser/mod.rs | 2 +- src/tests/front_matter.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 69c1faf2..56d597a4 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1167,7 +1167,7 @@ impl<'a, 'o> Parser<'a, 'o> { start: nodes::LineColumn { line: 1, column: 1 }, end: nodes::LineColumn { line: 1 + stripped_lines, - column: delimiter.chars().count(), + column: delimiter.len(), }, }; self.line_number += lines; diff --git a/src/tests/front_matter.rs b/src/tests/front_matter.rs index 4a2cb49e..0e6ba755 100644 --- a/src/tests/front_matter.rs +++ b/src/tests/front_matter.rs @@ -33,7 +33,7 @@ fn ast_wide_delimiter() { assert_ast_match_i( input, ast!((document (1:1-4:4) [ - (frontmatter (1:1-3:1) []) + (frontmatter (1:1-3:2) []) (paragraph (4:1-4:4) [ (text (4:1-4:4) []) ])