Test coverage for generated lexer

Makefile

@@ -0,0 +1,27 @@
+SHELL=/bin/bash -o pipefail
+
+refresh-codegen: codegen-lexer cleanup-internal-participle-cmd
+
+internal-participle-cmd:
+	@echo Building private build of participle utility...
+	cd cmd/participle && go build -o ../../participle github.com/alecthomas/participle/v2/cmd/participle
+
+internal-generateinternal:
+	@echo Updating internal test files...
+	cd lexer/internal/participlegeninternal && go build -o ../../../participlegeninternal github.com/alecthomas/participle/v2/lexer/internal/participlegeninternal
+	./participlegeninternal
+
+codegen-lexer: internal-participle-cmd internal-generateinternal
+	@echo Regenerating lexers...
+	(./participle gen lexer --name GeneratedA internal < lexer/internal/alexer.json | gofmt > lexer/internal/alexer.go) || rm -f lexer/internal/alexer.go
+	(./participle gen lexer --name GeneratedBasic internal < lexer/internal/basiclexer.json | gofmt > lexer/internal/basiclexer.go) || rm -f lexer/internal/basiclexer.go
+	(./participle gen lexer --name GeneratedHeredoc internal < lexer/internal/heredoclexer.json | gofmt > lexer/internal/heredoclexer.go) || rm -f lexer/internal/heredoclexer.go
+	(./participle gen lexer --name GeneratedHeredocWithWhitespace internal < lexer/internal/heredocwithwhitespacelexer.json | gofmt > lexer/internal/heredocwithwhitespacelexer.go) || rm -f lexer/internal/heredocwithwhitespacelexer.go
+	(./participle gen lexer --name GeneratedInterpolated internal < lexer/internal/interpolatedlexer.json | gofmt > lexer/internal/interpolatedlexer.go) || rm -f lexer/internal/interpolatedlexer.go
+	(./participle gen lexer --name GeneratedInterpolatedWithWhitespace internal < lexer/internal/interpolatedwithwhitespacelexer.json | gofmt > lexer/internal/interpolatedwithwhitespacelexer.go) || rm -f lexer/internal/interpolatedwithwhitespacelexer.go
+	(./participle gen lexer --name GeneratedReference internal < lexer/internal/referencelexer.json | gofmt > lexer/internal/referencelexer.go) || rm -f lexer/internal/referencelexer.go
+
+cleanup-internal-participle-cmd:
+	@echo Cleaning up...
+	rm -f ./participle
+	rm -f ./participlegeninternal

README.md

@@ -419,6 +419,8 @@ lexing. Use `participle/experimental/codegen.GenerateLexer()` to compile a
 This will generally provide around a 10x improvement in lexing performance
 while producing O(1) garbage.
 
+See Makefile for an example how to do lexer codegen.
+
 ## Options
 
 The Parser's behaviour can be configured via [Options](https://pkg.go.dev/github.com/alecthomas/participle/v2#Option).

cmd/participle/gen_lexer_cmd.go

@@ -98,6 +98,7 @@ import (
 )
 
 var _ syntax.Op
+var _ = utf8.DecodeRune
 
 var {{.Name}}Lexer lexer.Definition = lexer{{.Name}}DefinitionImpl{}
 

lexer/internal/alexer.json

@@ -0,0 +1,16 @@
+{
+  "Root": [
+    {
+      "name": "A",
+      "pattern": "a"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    },
+    {
+      "name": "whitespace",
+      "pattern": "\\s+"
+    }
+  ]
+}

lexer/internal/heredoclexer.json

@@ -0,0 +1,47 @@
+{
+  "Common": [
+    {
+      "name": "whitespace",
+      "pattern": "\\s+"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    }
+  ],
+  "Heredoc": [
+    {
+      "name": "End",
+      "pattern": "\\b\\1\\b",
+      "action": {
+        "kind": "pop"
+      }
+    },
+    {
+      "name": "whitespace",
+      "pattern": "\\s+"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    }
+  ],
+  "Root": [
+    {
+      "name": "Heredoc",
+      "pattern": "\u003c\u003c(\\w+\\b)",
+      "action": {
+        "kind": "push",
+        "state": "Heredoc"
+      }
+    },
+    {
+      "name": "whitespace",
+      "pattern": "\\s+"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    }
+  ]
+}

lexer/internal/heredocwithwhitespacelexer.json

@@ -0,0 +1,47 @@
+{
+  "Common": [
+    {
+      "name": "Whitespace",
+      "pattern": "\\s+"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    }
+  ],
+  "Heredoc": [
+    {
+      "name": "End",
+      "pattern": "\\b\\1\\b",
+      "action": {
+        "kind": "pop"
+      }
+    },
+    {
+      "name": "Whitespace",
+      "pattern": "\\s+"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    }
+  ],
+  "Root": [
+    {
+      "name": "Heredoc",
+      "pattern": "\u003c\u003c(\\w+\\b)",
+      "action": {
+        "kind": "push",
+        "state": "Heredoc"
+      }
+    },
+    {
+      "name": "Whitespace",
+      "pattern": "\\s+"
+    },
+    {
+      "name": "Ident",
+      "pattern": "\\w+"
+    }
+  ]
+}