diff --git a/CHANGES.rst b/CHANGES.rst index f23b6c96f..c8e357133 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -9,6 +9,7 @@ Unreleased - Use modern packaging metadata with ``pyproject.toml`` instead of ``setup.cfg``. :pr:`1793` - Use ``flit_core`` instead of ``setuptools`` as build backend. +- Preserve comments in ASTs when parsing templates with ``Environment.parse``. :pr:`2037` Version 3.1.5 diff --git a/src/jinja2/lexer.py b/src/jinja2/lexer.py index 6dc94b67d..88f2fc3d6 100644 --- a/src/jinja2/lexer.py +++ b/src/jinja2/lexer.py @@ -146,17 +146,7 @@ f"({'|'.join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x)))})" ) -ignored_tokens = frozenset( - [ - TOKEN_COMMENT_BEGIN, - TOKEN_COMMENT, - TOKEN_COMMENT_END, - TOKEN_WHITESPACE, - TOKEN_LINECOMMENT_BEGIN, - TOKEN_LINECOMMENT_END, - TOKEN_LINECOMMENT, - ] -) +ignored_tokens = frozenset([TOKEN_WHITESPACE]) ignore_if_empty = frozenset( [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT] ) @@ -631,6 +621,10 @@ def wrap( token = TOKEN_BLOCK_BEGIN elif token == TOKEN_LINESTATEMENT_END: token = TOKEN_BLOCK_END + elif token == TOKEN_LINECOMMENT_BEGIN: + token = TOKEN_COMMENT_BEGIN + elif token == TOKEN_LINECOMMENT_END: + token = TOKEN_COMMENT_END # we are not interested in those tokens in the parser elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END): continue diff --git a/src/jinja2/nodes.py b/src/jinja2/nodes.py index 2f93b90ec..9c81008ba 100644 --- a/src/jinja2/nodes.py +++ b/src/jinja2/nodes.py @@ -715,6 +715,13 @@ def as_const(self, eval_ctx: t.Optional[EvalContext] = None) -> t.Any: return self.expr2.as_const(eval_ctx) +class Comment(Stmt): + """A template comment.""" + + fields = ("data",) + data: str + + def args_as_const( node: t.Union["_FilterTestCommon", "Call"], eval_ctx: t.Optional[EvalContext] ) -> t.Tuple[t.List[t.Any], t.Dict[t.Any, t.Any]]: diff --git a/src/jinja2/parser.py b/src/jinja2/parser.py index 817abeccf..9c3a94df8 100644 --- a/src/jinja2/parser.py +++ b/src/jinja2/parser.py @@ -315,10 +315,13 @@ def parse_block(self) -> nodes.Block: # with whitespace data if node.required: for body_node in node.body: - if not isinstance(body_node, nodes.Output) or any( - not isinstance(output_node, nodes.TemplateData) - or not output_node.data.isspace() - for output_node in body_node.nodes + if not isinstance(body_node, (nodes.Output, nodes.Comment)) or ( + isinstance(body_node, nodes.Output) + and any( + not isinstance(output_node, nodes.TemplateData) + or not output_node.data.isspace() + for output_node in body_node.nodes + ) ): self.fail("Required blocks can only contain comments or whitespace") @@ -1025,6 +1028,11 @@ def flush_data() -> None: else: body.append(rv) self.stream.expect("block_end") + elif token.type == "comment_begin": + flush_data() + next(self.stream) + body.append(nodes.Comment(next(self.stream).value)) + self.stream.expect("comment_end") else: raise AssertionError("internal parsing error") diff --git a/tests/test_lexnparse.py b/tests/test_lexnparse.py index c02adad5a..cac32cf71 100644 --- a/tests/test_lexnparse.py +++ b/tests/test_lexnparse.py @@ -314,6 +314,19 @@ def assert_error(code, expected): ) assert_error("{% unknown_tag %}", "Encountered unknown tag 'unknown_tag'.") + def test_comment_preservation(self, env): + ast = env.parse("{# foo #}{{ bar }}") + assert len(ast.body) == 2 + assert isinstance(ast.body[0], nodes.Comment) + assert ast.body[0].data == " foo " + + def test_line_comment_preservation(self, env): + env = Environment(line_comment_prefix="#") + ast = env.parse("# foo\n{{ bar }}") + assert len(ast.body) == 2 + assert isinstance(ast.body[0], nodes.Comment) + assert ast.body[0].data == " foo" + class TestSyntax: def test_call(self, env):