diff --git a/Cargo.lock b/Cargo.lock index bdb6805b..525a53c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -137,7 +137,7 @@ version = "4.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", @@ -169,6 +169,7 @@ dependencies = [ "regex", "shell-words", "slug", + "strum", "syntect", "toml 0.7.3", "typed-arena", @@ -309,6 +310,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" @@ -724,6 +731,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.87", +] + [[package]] name = "syn" version = "1.0.107" diff --git a/Cargo.toml b/Cargo.toml index 55466560..d1b70d10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,6 +50,7 @@ caseless = "0.2.1" [dev-dependencies] ntest = "0.9" +strum = { version = "0.26.3", features = ["derive"] } toml = "0.7.3" [features] diff --git a/src/nodes.rs b/src/nodes.rs index cd749c1e..9781363d 100644 --- a/src/nodes.rs +++ b/src/nodes.rs @@ -12,6 +12,11 @@ pub use crate::parser::multiline_block_quote::NodeMultilineBlockQuote; /// The core AST node enum. #[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(test, derive(strum::EnumDiscriminants))] +#[cfg_attr( + test, + strum_discriminants(vis(pub(crate)), derive(strum::VariantArray, Hash)) +)] pub enum NodeValue { /// The root of every CommonMark document. Contains **blocks**. Document, @@ -246,7 +251,7 @@ pub struct NodeTable { } /// An inline [code span](https://github.github.com/gfm/#code-spans). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct NodeCode { /// The number of backticks pub num_backticks: usize, @@ -259,7 +264,7 @@ pub struct NodeCode { } /// The details of a link's destination, or an image's source. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct NodeLink { /// The URL for the link destination or image source. pub url: String, @@ -272,7 +277,7 @@ pub struct NodeLink { } /// The details of a wikilink's destination. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct NodeWikiLink { /// The URL for the link destination. pub url: String, diff --git a/src/parser/math.rs b/src/parser/math.rs index b1441e28..4f6cfb57 100644 --- a/src/parser/math.rs +++ b/src/parser/math.rs @@ -1,5 +1,5 @@ /// An inline math span -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] pub struct NodeMath { /// Whether this is dollar math (`$` or `$$`). /// `false` indicates it is code math @@ -8,7 +8,7 @@ pub struct NodeMath { /// Whether this is display math (using `$$`) pub display_math: bool, - /// The literal contents of the math span. + /// The literal contents of the math span. /// As the contents are not interpreted as Markdown at all, /// they are contained within this structure, /// rather than inserted into a child inline of any kind. diff --git a/src/parser/multiline_block_quote.rs b/src/parser/multiline_block_quote.rs index 2a8b5710..e6107eba 100644 --- a/src/parser/multiline_block_quote.rs +++ b/src/parser/multiline_block_quote.rs @@ -1,5 +1,5 @@ /// The metadata of a multiline blockquote. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)] pub struct NodeMultilineBlockQuote { /// The length of the fence. pub fence_length: usize, diff --git a/src/tests.rs b/src/tests.rs index 280bfa52..aafb0a9f 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -24,6 +24,8 @@ mod plugins; mod regressions; mod rewriter; mod shortcodes; +#[path = "tests/sourcepos.rs"] +mod sourcepos_; mod spoiler; mod strikethrough; mod subscript; diff --git a/src/tests/sourcepos.rs b/src/tests/sourcepos.rs new file mode 100644 index 00000000..3119e3a6 --- /dev/null +++ b/src/tests/sourcepos.rs @@ -0,0 +1,512 @@ +use nodes::NodeValueDiscriminants; +use strum::VariantArray; + +use super::*; + +type TestCase = (&'static [Sourcepos], &'static str); + +const DOCUMENT: TestCase = (&[sourcepos!((1:1-1:1))], "a"); + +const FRONT_MATTER: TestCase = ( + &[sourcepos!((1:1-3:3))], + r#"--- +a: b +--- + +hello world +"#, +); + +const BLOCK_QUOTE: TestCase = ( + &[sourcepos!((1:1-3:36))], + r#"> hello world +> this is line 1 +> this is line 2 and some extra text + +hello world"#, +); + +const MULTILINE_BLOCK_QUOTE: TestCase = ( + &[sourcepos!((3:1-7:3))], + r#"Some text + +>>> +hello world +this is line 1 +this is line 2 and some extra text +>>> + +hello world"#, +); + +const LIST: TestCase = ( + &[sourcepos!((1:1-2:38))], + r#"- bullet point one +- bullet point two and some extra text + +hello world +"#, +); + +const ITEM: TestCase = ( + &[sourcepos!((1:1-1:18)), sourcepos!((2:1-2:38))], + r#"- bullet point one +- bullet point two and some extra text + +hello world +"#, +); + +const TASK_ITEM: TestCase = ( + &[sourcepos!((1:1-1:22)), sourcepos!((3:1-3:24))], + r#"- [ ] bullet point one +- bullet point two and some extra text +- [x] bullet point three + +hello world +"#, +); + +const DESCRIPTION_LIST: TestCase = ( + &[sourcepos!((1:1-7:11))], + r#"Term 1 + +: Details 1 + +Term 2 + +: Details 2"#, +); + +const DESCRIPTION_ITEM: TestCase = ( + &[sourcepos!((1:1-3:11)), sourcepos!((5:1-7:11))], + r#"Term 1 + +: Details 1 + +Term 2 + +: Details 2"#, +); + +const DESCRIPTION_TERM: TestCase = ( + &[sourcepos!((1:1-1:6))], + r#"Term 1 + +: Details 1 + +hello world +"#, +); + +const DESCRIPTION_DETAILS: TestCase = ( + &[sourcepos!((3:1-3:11))], + r#"Term 1 + +: Details 1 + +hello world +"#, +); + +const CODE_BLOCK: TestCase = ( + &[sourcepos!((1:1-3:3))], + r#"``` +hello world +``` + +hello world +"#, +); + +const HTML_BLOCK: TestCase = ( + &[sourcepos!((1:1-2:30)), sourcepos!((5:1-5:10))], + r#"
+hello world + +hello world +
+ +hello world +"#, +); + +const HTML_INLINE: TestCase = ( + &[sourcepos!((1:7-3:14))], + r#"hello bar world +"#, +); + +const PARAGRAPH: TestCase = ( + &[sourcepos!((1:1-1:11)), sourcepos!((4:1-4:11))], + r#"hello world + + +hello world +"#, +); + +const HEADING: TestCase = ( + &[sourcepos!((5:1-5:13))], + r#"--- +a: b +--- + +# Hello World + +hello world +"#, +); + +const THEMATIC_BREAK: TestCase = ( + &[sourcepos!((3:1-3:3))], + r#"Hello + +--- + +World"#, +); + +const FOOTNOTE_DEFINITION: TestCase = ( + &[sourcepos!((3:1-3:11))], + r#"Hello[^1] + +[^1]: World +"#, +); + +const FOOTNOTE_REFERENCE: TestCase = ( + &[sourcepos!((1:6-1:9))], + r#"Hello[^1] + +[^1]: World +"#, +); + +const TABLE: TestCase = ( + &[sourcepos!((3:1-5:17))], + r#"stuff before + +| Hello | World | +| ----- | ----- | +| cell1 | cell2 | + +hello world +"#, +); + +const TABLE_ROW: TestCase = ( + &[sourcepos!((3:1-3:17)), sourcepos!((5:1-5:18))], + r#"stuff before + +| Hello | World | +| ----- | ----- | +| cell1 | cell02 | + +hello world +"#, +); + +const TABLE_CELL: TestCase = ( + &[ + sourcepos!((3:2-3:8)), + sourcepos!((3:10-3:16)), + sourcepos!((5:2-5:8)), + sourcepos!((5:10-5:17)), + ], + r#"stuff before + +| Hello | World | +| ----- | ----- | +| cell1 | cell02 | + +hello world +"#, +); + +const TEXT: TestCase = ( + &[ + sourcepos!((1:1-1:12)), + sourcepos!((3:3-3:7)), + sourcepos!((3:11-3:15)), + sourcepos!((5:3-5:7)), + sourcepos!((5:11-5:16)), + sourcepos!((7:1-7:11)), + sourcepos!((9:3-9:13)), + sourcepos!((11:3-11:8)), + sourcepos!((12:3-12:9)), + sourcepos!((12:12-12:15)), + sourcepos!((14:7-14:14)), + ], + r#"stuff before + +| Hello | World | +| ----- | ----- | +| cell1 | cell02 | + +hello world + +> hello world + +- item 1[^1] +- item 2 **bold** + +[^1]: The end. +"#, +); + +const SOFT_BREAK: TestCase = (&[sourcepos!((1:13-1:13))], "stuff before\nstuff after"); +const LINE_BREAK: TestCase = (&[sourcepos!((1:13-1:15))], "stuff before \nstuff after"); + +const CODE: TestCase = (&[sourcepos!((1:7-1:13))], "hello `world`"); + +const EMPH: TestCase = ( + &[sourcepos!((1:7-1:13)), sourcepos!((1:23-2:4))], + "hello *world* between *wo\nrld* after", +); + +const STRONG: TestCase = ( + &[sourcepos!((1:7-1:15)), sourcepos!((1:25-2:5))], + "hello **world** between **wo\nrld** after", +); + +const STRIKETHROUGH: TestCase = ( + &[sourcepos!((1:7-1:15)), sourcepos!((1:25-2:5))], + "hello ~~world~~ between ~~wo\nrld~~ after", +); + +const SUPERSCRIPT: TestCase = ( + &[sourcepos!((1:7-1:13)), sourcepos!((1:23-2:4))], + "hello ^world^ between ^wo\nrld^ after", +); + +const SUBSCRIPT: TestCase = ( + &[sourcepos!((1:7-1:13)), sourcepos!((1:23-2:4))], + "hello ~world~ between ~wo\nrld~ after", +); + +const LINK: TestCase = ( + &[ + sourcepos!((1:7-1:32)), + sourcepos!((2:7-2:32)), + sourcepos!((3:7-3:11)), + sourcepos!((4:7-4:16)), + sourcepos!((5:7-5:29)), + ], + r#"hello world +hello [foo](https://example.com) world +hello [foo] world +hello [bar][bar] world +hello https://example.com/foo world + +[foo]: https://example.com +[bar]: https://example.com"#, +); + +const IMAGE: TestCase = ( + &[sourcepos!((1:7-1:38))], + "hello ![alt text](https://example.com) banana", +); + +const MATH: TestCase = ( + &[ + sourcepos!((3:1-3:7)), + sourcepos!((3:17-3:26)), + sourcepos!((3:36-3:44)), + ], + r#"hello + +$1 + 1$ between $`1 + 23`$ between $$a + b$$ + +banana"#, +); + +const ESCAPED: TestCase = ( + &[ + sourcepos!((1:1-1:2)), + sourcepos!((1:3-1:4)), + sourcepos!((1:5-1:6)), + sourcepos!((1:7-1:8)), + sourcepos!((1:9-1:10)), + sourcepos!((1:11-1:12)), + sourcepos!((1:13-1:14)), + sourcepos!((1:15-1:16)), + sourcepos!((1:17-1:18)), + sourcepos!((1:19-1:20)), + sourcepos!((1:21-1:22)), + sourcepos!((1:23-1:24)), + sourcepos!((1:25-1:26)), + sourcepos!((1:27-1:28)), + sourcepos!((1:29-1:30)), + sourcepos!((1:31-1:32)), + sourcepos!((1:33-1:34)), + sourcepos!((1:35-1:36)), + sourcepos!((1:37-1:38)), + sourcepos!((1:39-1:40)), + sourcepos!((1:41-1:42)), + sourcepos!((1:43-1:44)), + sourcepos!((1:45-1:46)), + sourcepos!((1:47-1:48)), + sourcepos!((1:49-1:50)), + sourcepos!((1:51-1:52)), + sourcepos!((1:53-1:54)), + sourcepos!((1:55-1:56)), + sourcepos!((1:57-1:58)), + sourcepos!((1:59-1:60)), + sourcepos!((1:61-1:62)), + sourcepos!((1:63-1:64)), + ], + r#"\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\`\{\|\}\~\a"#, +); + +const WIKI_LINK: TestCase = ( + &[sourcepos!((1:1-1:9)), sourcepos!((3:1-3:33))], + r#"[[floop]] + +[[http://example.com|some title]] + +after"#, +); + +const UNDERLINE: TestCase = (&[sourcepos!((1:8-1:22))], "before __hello world__ after"); + +const SPOILERED_TEXT: TestCase = ( + &[sourcepos!((2:1-2:11))], + r#"before +||spoiler|| +after"#, +); + +const ESCAPED_TAG: TestCase = ( + &[sourcepos!((2:1-2:8))], + r#"before +||hello| +after"#, +); + +fn node_values() -> HashMap { + use NodeValueDiscriminants::*; + + NodeValueDiscriminants::VARIANTS + .iter() + .filter(|v| { + !matches!( + v, + // Remove buggy variants. + List // end is 3:0 + | Item // end is 3:0 + | TaskItem // end is 4:0 + | DescriptionItem // end is 4:0 + | DescriptionTerm // end is 3:0 + | DescriptionDetails // end is 4:0 + | HtmlInline // end is 1:31 but should be 3:14 + | LineBreak // start is 1:15 but should be 1:13 + | Code // is 1:8-1:12 but should be 1:7-1:13 + | ThematicBreak // end is 4:0 + | Link // inconsistent between link types + | Math // is 3:2-3:6 but should be 3:1-3:7 + ) + }) + .filter_map(|v| { + let text = match v { + Document => DOCUMENT, + FrontMatter => FRONT_MATTER, + BlockQuote => BLOCK_QUOTE, + MultilineBlockQuote => MULTILINE_BLOCK_QUOTE, + List => LIST, + Item => ITEM, + TaskItem => TASK_ITEM, + DescriptionList => DESCRIPTION_LIST, + DescriptionItem => DESCRIPTION_ITEM, + DescriptionTerm => DESCRIPTION_TERM, + DescriptionDetails => DESCRIPTION_DETAILS, + CodeBlock => CODE_BLOCK, + HtmlBlock => HTML_BLOCK, + HtmlInline => HTML_INLINE, + Paragraph => PARAGRAPH, + Heading => HEADING, + ThematicBreak => THEMATIC_BREAK, + FootnoteDefinition => FOOTNOTE_DEFINITION, + FootnoteReference => FOOTNOTE_REFERENCE, + Table => TABLE, + TableRow => TABLE_ROW, + TableCell => TABLE_CELL, + Text => TEXT, + SoftBreak => SOFT_BREAK, + LineBreak => LINE_BREAK, + Code => CODE, + Emph => EMPH, + Strong => STRONG, + Strikethrough => STRIKETHROUGH, + Superscript => SUPERSCRIPT, + Subscript => SUBSCRIPT, + Link => LINK, + Image => IMAGE, + Math => MATH, + Escaped => ESCAPED, + WikiLink => WIKI_LINK, + Underline => UNDERLINE, + SpoileredText => SPOILERED_TEXT, + EscapedTag => ESCAPED_TAG, + }; + Some((*v, text)) + }) + .collect() +} + +#[test] +fn sourcepos() { + // Use a single test instead of one test per node type so that we get a compile error when new + // variants are added to the `NodeValue` enum. + let node_values = node_values(); + + let mut options = Options::default(); + options.render = RenderOptions::builder().escaped_char_spans(true).build(); + + options.extension = ExtensionOptions::builder() + .front_matter_delimiter("---".to_string()) + .description_lists(true) + .footnotes(true) + .table(true) + .tasklist(true) + .strikethrough(true) + .superscript(true) + .subscript(true) + .autolink(true) + .math_code(true) + .math_dollars(true) + .multiline_block_quotes(true) + .wikilinks(WikiLinksMode::UrlFirst) + .underline(true) + .spoiler(true) + .build(); + + for (kind, (expecteds, text)) in node_values { + let arena = Arena::new(); + let root = parse_document(&arena, text, &options); + let asts: Vec<_> = root + .descendants() + .filter(|d| NodeValueDiscriminants::from(&d.data.borrow().value) == kind) + .collect(); + + if asts.len() != expecteds.len() { + panic!( + "expected {} node(s) of kind {:?}, but got {}", + expecteds.len(), + kind, + asts.len() + ); + } + + for (ast, expected) in asts.into_iter().zip(expecteds) { + let actual = ast.data.borrow().sourcepos; + assert_eq!( + *expected, actual, + "{} != {} for {:?}", + expected, actual, kind + ); + } + } +}