You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@groovy.apache.org by su...@apache.org on 2021/08/21 03:48:21 UTC
[groovy] branch danielsun/tune-parser-performance updated (dc13381
-> f9d19c3)
This is an automated email from the ASF dual-hosted git repository.
sunlan pushed a change to branch danielsun/tune-parser-performance
in repository https://gitbox.apache.org/repos/asf/groovy.git.
discard dc13381 Try to fix the failing build
discard db9686b Tune the performance of parrot parser
new f9d19c3 Tune the performance of parrot parser
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (dc13381)
\
N -- N -- N refs/heads/danielsun/tune-parser-performance (f9d19c3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../groovy/parser/antlr4/Antlr4ParserPlugin.java | 50 ++++++++++------------
.../java/org/codehaus/groovy/ast/ModuleNode.java | 4 +-
2 files changed, 24 insertions(+), 30 deletions(-)
[groovy] 01/01: Tune the performance of parrot parser
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunlan pushed a commit to branch danielsun/tune-parser-performance
in repository https://gitbox.apache.org/repos/asf/groovy.git
commit f9d19c31887bdd39ce4163a4fa55f99fb9d60c04
Author: Daniel Sun <su...@apache.org>
AuthorDate: Sat Aug 21 11:48:01 2021 +0800
Tune the performance of parrot parser
---
.../groovy/parser/antlr4/Antlr4ParserPlugin.java | 55 ++++++++++++++++++++--
.../apache/groovy/parser/antlr4/AstBuilder.java | 37 +++++++++++----
.../java/org/codehaus/groovy/ast/ModuleNode.java | 8 +++-
3 files changed, 85 insertions(+), 15 deletions(-)
diff --git a/src/main/java/org/apache/groovy/parser/antlr4/Antlr4ParserPlugin.java b/src/main/java/org/apache/groovy/parser/antlr4/Antlr4ParserPlugin.java
index 819165e..28e9f06 100644
--- a/src/main/java/org/apache/groovy/parser/antlr4/Antlr4ParserPlugin.java
+++ b/src/main/java/org/apache/groovy/parser/antlr4/Antlr4ParserPlugin.java
@@ -20,6 +20,8 @@ package org.apache.groovy.parser.antlr4;
import org.codehaus.groovy.GroovyBugError;
import org.codehaus.groovy.ast.ModuleNode;
+import org.codehaus.groovy.control.CompilationFailedException;
+import org.codehaus.groovy.control.ErrorCollector;
import org.codehaus.groovy.control.ParserPlugin;
import org.codehaus.groovy.control.SourceUnit;
import org.codehaus.groovy.control.io.StringReaderSource;
@@ -28,6 +30,15 @@ import org.codehaus.groovy.syntax.Reduction;
import java.io.IOException;
import java.io.Reader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
/**
* A parser plugin for the new parser.
@@ -49,12 +60,46 @@ public class Antlr4ParserPlugin implements ParserPlugin {
return null;
}
+
@Override
public ModuleNode buildAST(final SourceUnit sourceUnit, final ClassLoader classLoader, final Reduction cst) {
- AstBuilder builder = new AstBuilder(sourceUnit,
- sourceUnit.getConfiguration().isGroovydocEnabled(),
- sourceUnit.getConfiguration().isRuntimeGroovydocEnabled()
- );
- return builder.buildAST();
+ List<Callable<ModuleNode>> taskList = new ArrayList<>(2);
+ SourceUnit su = new SourceUnit(sourceUnit.getName(), sourceUnit.getSource(), sourceUnit.getConfiguration(),
+ sourceUnit.getClassLoader(), new ErrorCollector(sourceUnit.getConfiguration()));
+ taskList.add(createBuildAstTask(su, AstBuilder.SLL));
+ taskList.add(createBuildAstTask(sourceUnit, AstBuilder.ALL));
+
+ ExecutorService threadPool = Executors.newFixedThreadPool(taskList.size(), r -> {
+ Thread t = new Thread(r);
+ t.setName("parser-thread-" + SEQ.getAndIncrement());
+ return t;
+ });
+
+ try {
+ ModuleNode moduleNode = threadPool.invokeAny(taskList, 10, TimeUnit.SECONDS);
+ moduleNode.setContext(sourceUnit);
+ return moduleNode;
+ } catch (InterruptedException e) {
+ throw new GroovyBugError("Interrupted while parsing", e);
+ } catch (TimeoutException e) {
+ throw new GroovyBugError("Timeout while parsing", e);
+ } catch (ExecutionException e) {
+ throw (CompilationFailedException) e.getCause();
+ } finally {
+ threadPool.shutdown();
+ }
}
+
+ private Callable<ModuleNode> createBuildAstTask(final SourceUnit sourceUnit, final int predictionMode) {
+ return () -> {
+ AstBuilder builder = new AstBuilder(sourceUnit,
+ sourceUnit.getConfiguration().isGroovydocEnabled(),
+ sourceUnit.getConfiguration().isRuntimeGroovydocEnabled(),
+ predictionMode
+ );
+ return builder.buildAST();
+ };
+ }
+
+ private static final AtomicLong SEQ = new AtomicLong(0);
}
diff --git a/src/main/java/org/apache/groovy/parser/antlr4/AstBuilder.java b/src/main/java/org/apache/groovy/parser/antlr4/AstBuilder.java
index 9a7aa5d..256ca32 100644
--- a/src/main/java/org/apache/groovy/parser/antlr4/AstBuilder.java
+++ b/src/main/java/org/apache/groovy/parser/antlr4/AstBuilder.java
@@ -370,7 +370,15 @@ import static org.codehaus.groovy.runtime.DefaultGroovyMethods.last;
* Builds the AST from the parse tree generated by Antlr4.
*/
public class AstBuilder extends GroovyParserBaseVisitor<Object> {
+ public static final int SLL = 0;
+ public static final int ALL = 1;
+ public static final int AUTO = -1;
+
public AstBuilder(final SourceUnit sourceUnit, final boolean groovydocEnabled, final boolean runtimeGroovydocEnabled) {
+ this(sourceUnit, groovydocEnabled, runtimeGroovydocEnabled, AUTO);
+ }
+
+ public AstBuilder(final SourceUnit sourceUnit, final boolean groovydocEnabled, final boolean runtimeGroovydocEnabled, final int predictionMode) {
this.sourceUnit = sourceUnit;
this.moduleNode = new ModuleNode(sourceUnit);
CharStream charStream = createCharStream(sourceUnit);
@@ -381,6 +389,8 @@ public class AstBuilder extends GroovyParserBaseVisitor<Object> {
this.groovydocManager = new GroovydocManager(groovydocEnabled, runtimeGroovydocEnabled);
this.tryWithResourcesASTTransformation = new TryWithResourcesASTTransformation(this);
+
+ this.predictionMode = predictionMode;
}
private CharStream createCharStream(final SourceUnit sourceUnit) {
@@ -404,14 +414,24 @@ public class AstBuilder extends GroovyParserBaseVisitor<Object> {
// parsing have to wait util clearing is complete.
AtnManager.READ_LOCK.lock();
try {
- result = buildCST(PredictionMode.SLL);
- } catch (Throwable t) {
- // if some syntax error occurred in the lexer, no need to retry the powerful LL mode
- if (t instanceof GroovySyntaxError && GroovySyntaxError.LEXER == ((GroovySyntaxError) t).getSource()) {
- throw t;
- }
+ if (SLL == predictionMode) {
+ result = buildCST(PredictionMode.SLL);
+ } else if (ALL == predictionMode) {
+ result = buildCST(PredictionMode.LL);
+ } else if (AUTO == predictionMode) {
+ try {
+ result = buildCST(PredictionMode.SLL);
+ } catch (Throwable t) {
+ // if some syntax error occurred in the lexer, no need to retry the powerful LL mode
+ if (t instanceof GroovySyntaxError && GroovySyntaxError.LEXER == ((GroovySyntaxError) t).getSource()) {
+ throw t;
+ }
- result = buildCST(PredictionMode.LL);
+ result = buildCST(PredictionMode.LL);
+ }
+ } else {
+ throw new GroovyBugError("Unknown prediction mode: " + predictionMode);
+ }
} finally {
AtnManager.READ_LOCK.unlock();
}
@@ -424,11 +444,11 @@ public class AstBuilder extends GroovyParserBaseVisitor<Object> {
private GroovyParserRuleContext buildCST(final PredictionMode predictionMode) {
parser.getInterpreter().setPredictionMode(predictionMode);
+ parser.getInputStream().seek(0);
if (PredictionMode.SLL.equals(predictionMode)) {
this.removeErrorListeners();
} else {
- parser.getInputStream().seek(0);
this.addErrorListeners();
}
@@ -5020,6 +5040,7 @@ public class AstBuilder extends GroovyParserBaseVisitor<Object> {
private final GroovyLangParser parser;
private final GroovydocManager groovydocManager;
private final TryWithResourcesASTTransformation tryWithResourcesASTTransformation;
+ private final int predictionMode;
private final List<ClassNode> classNodeList = new LinkedList<>();
private final Deque<ClassNode> classNodeStack = new ArrayDeque<>();
diff --git a/src/main/java/org/codehaus/groovy/ast/ModuleNode.java b/src/main/java/org/codehaus/groovy/ast/ModuleNode.java
index ba0dbb1..61c624e 100644
--- a/src/main/java/org/codehaus/groovy/ast/ModuleNode.java
+++ b/src/main/java/org/codehaus/groovy/ast/ModuleNode.java
@@ -38,6 +38,8 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import static org.codehaus.groovy.ast.ClassHelper.isObjectType;
+import static org.codehaus.groovy.ast.ClassHelper.isPrimitiveVoid;
import static org.codehaus.groovy.ast.tools.GeneralUtils.args;
import static org.codehaus.groovy.ast.tools.GeneralUtils.callX;
import static org.codehaus.groovy.ast.tools.GeneralUtils.classX;
@@ -46,8 +48,6 @@ import static org.codehaus.groovy.ast.tools.GeneralUtils.param;
import static org.codehaus.groovy.ast.tools.GeneralUtils.params;
import static org.codehaus.groovy.ast.tools.GeneralUtils.stmt;
import static org.codehaus.groovy.ast.tools.GeneralUtils.varX;
-import static org.codehaus.groovy.ast.ClassHelper.isObjectType;
-import static org.codehaus.groovy.ast.ClassHelper.isPrimitiveVoid;
import static org.objectweb.asm.Opcodes.ACC_ABSTRACT;
import static org.objectweb.asm.Opcodes.ACC_FINAL;
import static org.objectweb.asm.Opcodes.ACC_INTERFACE;
@@ -282,6 +282,10 @@ public class ModuleNode extends ASTNode {
return context;
}
+ public void setContext(SourceUnit context) {
+ this.context = context;
+ }
+
private boolean isPackageInfo() {
return context != null && context.getName() != null && context.getName().endsWith("package-info.groovy");
}