From bf059299a103c784aa70471beb34dd1fb63fbed4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20L=C3=BCpges?= Date: Tue, 5 Nov 2024 10:27:54 +0100 Subject: [PATCH 1/3] Update .gitlab-ci.yml: Disable Mirror --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fabd25b6f7..46583546a2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -483,6 +483,5 @@ githubmirror: - (git commit -m "[Mirror] Replace all internal links and references") || true - git remote add github git@github.com:MontiCore/monticore.git - git push github $CI_COMMIT_BRANCH --force - only: - - dev - - master + rules: + - when: never From 37eebd5c74d602ca201e9f99c797200a2a8c9a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20L=C3=BCpges?= Date: Fri, 8 Nov 2024 13:49:53 +0100 Subject: [PATCH 2/3] #4388 Side-effect free Lexer generation --- .../codegen/parser/ParserGeneratorHelper.java | 20 +++++++++++- .../codegen/parser/antlr/Grammar2Antlr.java | 16 +++++----- .../parser/antlr/Grammar2ParseVisitor.java | 11 ++++--- .../src/main/resources/parser/Lexer.ftl | 2 +- .../java/de/monticore/grammar/LexNamer.java | 32 ++++++++++++++++--- 5 files changed, 62 insertions(+), 19 deletions(-) diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java index e5eae2a8aa..5bf9705175 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java @@ -146,7 +146,7 @@ public String getParserPackage() { /** * @return the name for a lexsymbol that should be used in an Antlr-File */ - public String getLexSymbolName(String constName) { + public String getOrComputeLexSymbolName(String constName) { Log.errorIfNull(constName); if (grammarInfo.getSplitRules().containsKey(constName)) { return grammarInfo.getSplitRules().get(constName); @@ -155,6 +155,24 @@ public String getLexSymbolName(String constName) { } } + /** + * Does not + * @return the name for a lexsymbol that was used in an Antlr-File + */ + public Optional getCachedLexSymbolName(String constName) { + Log.errorIfNull(constName); + if (grammarInfo.getSplitRules().containsKey(constName)) { + return Optional.of(grammarInfo.getSplitRules().get(constName)); + } else { + // Optimally we would call Map#get directly on the LexNamer + // (which is only available with the next release) + if (grammarInfo.getLexNamer().getLexnames().contains(constName)) { + return Optional.of(grammarInfo.getLexNamer().getLexName(grammarSymbol, constName)); + } + return Optional.empty(); + } + } + /** * @return the name for a rule replacing a keyword */ diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java index 8d990df694..829b0c60fc 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/antlr/Grammar2Antlr.java @@ -268,7 +268,7 @@ public void handle(ASTEnumProd ast) { int index = 0; for (ASTConstant c : ast.getConstantList()) { addToCodeSection(sep); - addToCodeSection("\n", "e_" + index++ + "=" + parserHelper.getLexSymbolName(c.getName())); + addToCodeSection("\n", "e_" + index++ + "=" + parserHelper.getOrComputeLexSymbolName(c.getName())); if (embeddedJavaCode) { String temp1 = ""; @@ -319,11 +319,11 @@ public void handle(ASTConstantGroup ast) { if (x.isPresentKeyConstant()) { addToCodeSection(createKeyPredicate(x.getKeyConstant().getStringList(), tmpName + label)); } else if (!grammarInfo.isKeyword(x.getName(), grammarEntry)) { - addToCodeSection(tmpName + label + parserHelper.getLexSymbolName(x.getName())); + addToCodeSection(tmpName + label + parserHelper.getOrComputeLexSymbolName(x.getName())); } else if (grammarInfo.getKeywordRules().contains(x.getName())) { addToCodeSection(tmpName + label + parserHelper.getKeyRuleName(x.getName())); } else { - addToCodeSection(tmpName + label + parserHelper.getLexSymbolName(x.getName())); + addToCodeSection(tmpName + label + parserHelper.getOrComputeLexSymbolName(x.getName())); } del = "|\n"; @@ -464,7 +464,7 @@ public void visit(ASTTerminal ast) { } else if (grammarInfo.isKeyword(ast.getName(), grammarEntry) && grammarInfo.getKeywordRules().contains(ast.getName())) { rulename = parserHelper.getKeyRuleName(ast.getName()); } else { - rulename = parserHelper.getLexSymbolName(ast.getName().intern()); + rulename = parserHelper.getOrComputeLexSymbolName(ast.getName().intern()); } // No actions in predicates @@ -487,7 +487,7 @@ public void visit(ASTTerminal ast) { if (grammarInfo.getKeywordRules().contains(replaceString)) { addToCodeSection(parserHelper.getKeyRuleName(replaceString)); } else { - addToCodeSection(parserHelper.getLexSymbolName(replaceString)); + addToCodeSection(parserHelper.getOrComputeLexSymbolName(replaceString)); } seperator = " | "; } @@ -568,7 +568,7 @@ void handleTerminal(String labelAssignment, String tmpVarName, Map - ${genHelper.getLexSymbolName(lexSymbol)} : '${lexSymbol}'; + ${genHelper.getOrComputeLexSymbolName(lexSymbol)} : '${lexSymbol}'; <#list genHelper.getLexerRulesToGenerate() as lexProd> diff --git a/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java b/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java index 5cb5293f2c..dca444f547 100644 --- a/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java +++ b/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; import de.monticore.grammar.grammar._symboltable.MCGrammarSymbol; @@ -119,14 +120,23 @@ public static String createSimpleGoodName(String x) { } + @Deprecated + public String getLexName(MCGrammarSymbol grammarSymbol, String sym) { + return this.getOrComputeLexName(grammarSymbol, sym); + } + + public Optional getComputedLexName(String sym) { + return Optional.ofNullable(this.usedLex.get(sym)); + } + /** * Returns Human-Readable, antlr conformed name for a lexsymbols nice names for common tokens * (change constructor to add tokenes) LEXi where i is number for unknown ones - * + * * @param sym lexer symbol * @return Human-Readable, antlr conformed name for a lexsymbols */ - public String getLexName(MCGrammarSymbol grammarSymbol, String sym) { + public String getOrComputeLexName(MCGrammarSymbol grammarSymbol, String sym) { if (usedLex.containsKey(sym)) { return usedLex.get(sym); } @@ -139,8 +149,17 @@ public String getLexName(MCGrammarSymbol grammarSymbol, String sym) { Log.debug("Using lexer symbol " + goodName + " for symbol '" + sym + "'", "LexNamer"); return goodName; } - + + @Deprecated public String getConstantName(String sym) { + return this.getOrComputeConstantName(sym); + } + + public Optional getComputedConstantName(String sym) { + return Optional.ofNullable(this.usedConstants.get(sym)); + } + + public String getOrComputeConstantName(String sym) { String s = sym.intern(); if (!usedConstants.containsKey(s)) { @@ -165,8 +184,11 @@ protected String convertKeyword(String key) { return key; } - public Set getLexnames() { - return usedLex.keySet(); + return this.usedLex.keySet(); + } + + public Set getUsedConstants() { + return this.usedConstants.keySet(); } } From dfcfa236b8d9c67cc04fc074365ccc3c29ac0314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20L=C3=BCpges?= Date: Wed, 27 Nov 2024 14:53:06 +0100 Subject: [PATCH 3/3] Improve documentation of the lexer generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Alex Lüpges --- .../de/monticore/codegen/parser/ParserGeneratorHelper.java | 2 +- .../src/main/java/de/monticore/grammar/LexNamer.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java b/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java index 5bf9705175..a5c3018bb2 100644 --- a/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java +++ b/monticore-generator/src/main/java/de/monticore/codegen/parser/ParserGeneratorHelper.java @@ -156,7 +156,7 @@ public String getOrComputeLexSymbolName(String constName) { } /** - * Does not + * Side effect free. * @return the name for a lexsymbol that was used in an Antlr-File */ public Optional getCachedLexSymbolName(String constName) { diff --git a/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java b/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java index dca444f547..69f4c6a5df 100644 --- a/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java +++ b/monticore-grammar/src/main/java/de/monticore/grammar/LexNamer.java @@ -120,6 +120,13 @@ public static String createSimpleGoodName(String x) { } + /** + * Computes a new name for the lexical production or returns the previously + * computed one. + * + * @deprecated Use {@link LexNamer#getComputedLexName(String)} + * or {@link LexNamer#getOrComputeLexName(MCGrammarSymbol, String)} instead + */ @Deprecated public String getLexName(MCGrammarSymbol grammarSymbol, String sym) { return this.getOrComputeLexName(grammarSymbol, sym);