You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@groovy.apache.org by su...@apache.org on 2020/06/13 07:55:22 UTC
[groovy] branch GROOVY-9589 created (now 54cd891)
This is an automated email from the ASF dual-hosted git repository.
sunlan pushed a change to branch GROOVY-9589
in repository https://gitbox.apache.org/repos/asf/groovy.git.
at 54cd891 GROOVY-9589: Parse source codes in parallel
This branch includes the following new commits:
new 54cd891 GROOVY-9589: Parse source codes in parallel
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[groovy] 01/01: GROOVY-9589: Parse source codes in parallel
Posted by su...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sunlan pushed a commit to branch GROOVY-9589
in repository https://gitbox.apache.org/repos/asf/groovy.git
commit 54cd891ec5c6fb7a5d642add7e07f5771d5a7e5b
Author: Daniel Sun <su...@apache.org>
AuthorDate: Sat Jun 13 15:52:18 2020 +0800
GROOVY-9589: Parse source codes in parallel
---
.../codehaus/groovy/control/CompilationUnit.java | 85 ++++++++++++++++++----
.../groovy/control/CompilerConfiguration.java | 5 ++
2 files changed, 75 insertions(+), 15 deletions(-)
diff --git a/src/main/java/org/codehaus/groovy/control/CompilationUnit.java b/src/main/java/org/codehaus/groovy/control/CompilationUnit.java
index 24d52e1..2a0e3ef 100644
--- a/src/main/java/org/codehaus/groovy/control/CompilationUnit.java
+++ b/src/main/java/org/codehaus/groovy/control/CompilationUnit.java
@@ -64,6 +64,7 @@ import java.net.URL;
import java.security.CodeSource;
import java.util.ArrayList;
import java.util.Deque;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -73,6 +74,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import static java.util.stream.Collectors.toList;
import static org.codehaus.groovy.ast.tools.GeneralUtils.classX;
@@ -90,18 +93,25 @@ import static org.codehaus.groovy.transform.stc.StaticTypesMarker.SWITCH_CONDITI
*/
public class CompilationUnit extends ProcessingUnit {
- /** The overall AST for this CompilationUnit. */
+ /**
+ * The overall AST for this CompilationUnit.
+ */
protected CompileUnit ast; // TODO: Switch to private and access through getAST().
- /** The source units from which this unit is built. */
+ /**
+ * The source units from which this unit is built.
+ */
protected Map<String, SourceUnit> sources = new LinkedHashMap<>();
protected Queue<SourceUnit> queuedSources = new LinkedList<>();
- /** The classes generated during classgen. */
+ /**
+ * The classes generated during classgen.
+ */
private List<GroovyClass> generatedClasses = new ArrayList<>();
private Deque<PhaseOperation>[] phaseOperations;
private Deque<PhaseOperation>[] newPhaseOperations;
+
{
final int n = Phases.ALL + 1;
phaseOperations = new Deque[n];
@@ -112,20 +122,30 @@ public class CompilationUnit extends ProcessingUnit {
}
}
- /** Controls behavior of {@link #classgen()} and other routines. */
+ /**
+ * Controls behavior of {@link #classgen()} and other routines.
+ */
protected boolean debug;
- /** True after the first {@link #configure(CompilerConfiguration)} operation. */
+ /**
+ * True after the first {@link #configure(CompilerConfiguration)} operation.
+ */
protected boolean configured;
- /** A callback for use during {@link #classgen()} */
+ /**
+ * A callback for use during {@link #classgen()}
+ */
protected ClassgenCallback classgenCallback;
- /** A callback for use during {@link #compile()} */
+ /**
+ * A callback for use during {@link #compile()}
+ */
protected ProgressCallback progressCallback;
protected ClassNodeResolver classNodeResolver = new ClassNodeResolver();
protected ResolveVisitor resolveVisitor = new ResolveVisitor(this);
- /** The AST transformations state data. */
+ /**
+ * The AST transformations state data.
+ */
protected ASTTransformationsContext astTransformationsContext;
private Set<javax.tools.JavaFileObject> javaCompilationUnitSet = new HashSet<>();
@@ -188,11 +208,12 @@ public class CompilationUnit extends ProcessingUnit {
addPhaseOperation(SourceUnit::parse, Phases.PARSING);
addPhaseOperation(source -> {
- source.convert();
+ ModuleNode moduleNode =
+ astCache.computeIfAbsent(source, sourceUnitConvertFunction());
// add module to compile unit
- getAST().addModule(source.getAST());
+ getAST().addModule(moduleNode);
Optional.ofNullable(getProgressCallback())
- .ifPresent(callback -> callback.call(source, getPhase()));
+ .ifPresent(callback -> callback.call(source, getPhase()));
}, Phases.CONVERSION);
addPhaseOperation((final SourceUnit source, final GeneratorContext context, final ClassNode classNode) -> {
@@ -521,15 +542,18 @@ public class CompilationUnit extends ProcessingUnit {
public Iterator<SourceUnit> iterator() {
return new Iterator<SourceUnit>() {
private Iterator<String> nameIterator = sources.keySet().iterator();
+
@Override
public boolean hasNext() {
return nameIterator.hasNext();
}
+
@Override
public SourceUnit next() {
String name = nameIterator.next();
return sources.get(name);
}
+
@Override
public void remove() {
throw new UnsupportedOperationException();
@@ -626,12 +650,16 @@ public class CompilationUnit extends ProcessingUnit {
if (dequeued()) continue;
}
+ if (phase == Phases.CONVERSION) {
+ initAstCache(sources);
+ }
+
processPhaseOperations(phase);
// Grab processing may have brought in new AST transforms into various phases, process them as well
processNewPhaseOperations(phase);
Optional.ofNullable(getProgressCallback())
- .ifPresent(callback -> callback.call(this, phase));
+ .ifPresent(callback -> callback.call(this, phase));
completePhase();
mark();
@@ -647,6 +675,31 @@ public class CompilationUnit extends ProcessingUnit {
getErrorCollector().failIfErrors();
}
+ private final Map<SourceUnit, ModuleNode> astCache = new HashMap<>();
+ private void initAstCache(Map<String, SourceUnit> sources) {
+ Boolean parallelParseEnabled = configuration.getOptimizationOptions().get(CompilerConfiguration.PARALLEL_PARSE);
+ if (null != parallelParseEnabled && !parallelParseEnabled) {
+ return;
+ }
+
+ astCache.clear();
+ Map<SourceUnit, ModuleNode> astMap =
+ sources.values().parallelStream()
+ .collect(Collectors.toMap(
+ Function.identity(),
+ sourceUnitConvertFunction(),
+ (existing, replacement) -> existing
+ ));
+ astCache.putAll(astMap);
+ }
+
+ private static Function<SourceUnit, ModuleNode> sourceUnitConvertFunction() {
+ return s -> {
+ s.convert();
+ return s.getAST();
+ };
+ }
+
private void processPhaseOperations(final int phase) {
for (PhaseOperation op : phaseOperations[phase]) {
op.doPhaseOperation(this);
@@ -784,7 +837,7 @@ public class CompilationUnit extends ProcessingUnit {
// Handle any callback that's been set
//
Optional.ofNullable(getClassgenCallback())
- .ifPresent(callback -> callback.call(classVisitor, classNode));
+ .ifPresent(callback -> callback.call(classVisitor, classNode));
//
// Recurse for inner classes
@@ -819,6 +872,7 @@ public class CompilationUnit extends ProcessingUnit {
ClassNodeResolver.LookupResult lookupResult = getClassNodeResolver().resolveName(name, CompilationUnit.this);
return lookupResult == null ? null : lookupResult.getClassNode();
}
+
private ClassNode getCommonSuperClassNode(ClassNode c, ClassNode d) {
// adapted from ClassWriter code
if (c.isDerivedFrom(d)) return d;
@@ -830,11 +884,12 @@ public class CompilationUnit extends ProcessingUnit {
if (c == null) return ClassHelper.OBJECT_TYPE;
return c;
}
+
@Override
protected String getCommonSuperClass(String arg1, String arg2) {
ClassNode a = getClassNode(arg1.replace('/', '.'));
ClassNode b = getClassNode(arg2.replace('/', '.'));
- return getCommonSuperClassNode(a,b).getName().replace('.','/');
+ return getCommonSuperClassNode(a, b).getName().replace('.', '/');
}
};
}
@@ -1017,7 +1072,7 @@ public class CompilationUnit extends ProcessingUnit {
private List<ClassNode> getPrimaryClassNodes(final boolean sort) {
List<ClassNode> unsorted = getAST().getModules().stream()
- .flatMap(module -> module.getClasses().stream()).collect(toList());
+ .flatMap(module -> module.getClasses().stream()).collect(toList());
if (!sort) return unsorted;
diff --git a/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java b/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java
index 468da4b..a1cf154 100644
--- a/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java
+++ b/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java
@@ -60,6 +60,10 @@ public class CompilerConfiguration {
/** Joint Compilation Option for enabling generating stubs in memory. */
public static final String MEM_STUB = "memStub";
+ /** Optimization Option for enabling attaching groovydoc as AST node metadata. */
+ public static final String PARALLEL_PARSE = "parallelParse";
+
+
/** This (<code>"1.4"</code>) is the value for targetBytecode to compile for a JDK 1.4. */
public static final String JDK4 = "1.4";
/** This (<code>"1.5"</code>) is the value for targetBytecode to compile for a JDK 1.5. */
@@ -439,6 +443,7 @@ public class CompilerConfiguration {
handleOptimizationOption(optimizationOptions, INVOKEDYNAMIC, "groovy.target.indy", "true");
handleOptimizationOption(optimizationOptions, GROOVYDOC, "groovy.attach.groovydoc");
handleOptimizationOption(optimizationOptions, RUNTIME_GROOVYDOC, "groovy.attach.runtime.groovydoc");
+ handleOptimizationOption(optimizationOptions, PARALLEL_PARSE, "groovy.parallel.parse", "true");
}
private void handleOptimizationOption(final Map<String, Boolean> options, final String optionName, final String sysOptionName) {