You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@groovy.apache.org by su...@apache.org on 2020/06/15 23:36:11 UTC
[groovy] branch master updated: GROOVY-9589: Parse source codes in
parallel(closes #1275)
This is an automated email from the ASF dual-hosted git repository.
sunlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/groovy.git
The following commit(s) were added to refs/heads/master by this push:
new 64f722d GROOVY-9589: Parse source codes in parallel(closes #1275)
64f722d is described below
commit 64f722d2303886a79407469e0afd5e06d58a9ebb
Author: Daniel Sun <su...@apache.org>
AuthorDate: Sun Jun 14 09:30:38 2020 +0800
GROOVY-9589: Parse source codes in parallel(closes #1275)
---
.../codehaus/groovy/control/CompilationUnit.java | 23 ++++++++
.../groovy/control/CompilerConfiguration.java | 4 ++
.../org/codehaus/groovy/control/SourceUnit.java | 19 ++++--
src/test/groovy/bugs/Groovy9589.groovy | 68 ++++++++++++++++++++++
.../apache/groovy/parser/antlr4/TestUtils.groovy | 12 ++++
5 files changed, 121 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/codehaus/groovy/control/CompilationUnit.java b/src/main/java/org/codehaus/groovy/control/CompilationUnit.java
index 24d52e1..a4d2019 100644
--- a/src/main/java/org/codehaus/groovy/control/CompilationUnit.java
+++ b/src/main/java/org/codehaus/groovy/control/CompilationUnit.java
@@ -63,6 +63,7 @@ import java.io.InputStream;
import java.net.URL;
import java.security.CodeSource;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
@@ -73,6 +74,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.Queue;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static java.util.stream.Collectors.toList;
import static org.codehaus.groovy.ast.tools.GeneralUtils.classX;
@@ -626,6 +629,10 @@ public class CompilationUnit extends ProcessingUnit {
if (dequeued()) continue;
}
+ if (phase == Phases.CONVERSION) {
+ buildASTs();
+ }
+
processPhaseOperations(phase);
// Grab processing may have brought in new AST transforms into various phases, process them as well
processNewPhaseOperations(phase);
@@ -647,6 +654,22 @@ public class CompilationUnit extends ProcessingUnit {
getErrorCollector().failIfErrors();
}
+ private void buildASTs() {
+ Boolean parallelParseEnabled = configuration.getOptimizationOptions().get(CompilerConfiguration.PARALLEL_PARSE);
+ if (null != parallelParseEnabled && !parallelParseEnabled) {
+ return;
+ }
+
+ Collection<SourceUnit> sourceUnits = sources.values();
+ Stream<SourceUnit> sourceUnitStream =
+ sourceUnits.size() < 2
+ ? sourceUnits.stream() // no need to build AST with parallel stream when we just have one/no source unit
+ : sourceUnits.parallelStream();
+
+ // DON'T replace `collect(Collectors.counting())` with `count()` here, otherwise peek will NOT be triggered
+ sourceUnitStream.peek(SourceUnit::buildAST).collect(Collectors.counting());
+ }
+
private void processPhaseOperations(final int phase) {
for (PhaseOperation op : phaseOperations[phase]) {
op.doPhaseOperation(this);
diff --git a/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java b/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java
index e65e6db..df8f454 100644
--- a/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java
+++ b/src/main/java/org/codehaus/groovy/control/CompilerConfiguration.java
@@ -57,6 +57,9 @@ public class CompilerConfiguration {
/** Optimization Option for enabling attaching {@link groovy.lang.Groovydoc} annotation. */
public static final String RUNTIME_GROOVYDOC = "runtimeGroovydoc";
+ /** Optimization Option for enabling parallel parsing. */
+ public static final String PARALLEL_PARSE = "parallelParse";
+
/** Joint Compilation Option for enabling generating stubs in memory. */
public static final String MEM_STUB = "memStub";
@@ -439,6 +442,7 @@ public class CompilerConfiguration {
handleOptimizationOption(optimizationOptions, INVOKEDYNAMIC, "groovy.target.indy", "true");
handleOptimizationOption(optimizationOptions, GROOVYDOC, "groovy.attach.groovydoc");
handleOptimizationOption(optimizationOptions, RUNTIME_GROOVYDOC, "groovy.attach.runtime.groovydoc");
+ handleOptimizationOption(optimizationOptions, PARALLEL_PARSE, "groovy.parallel.parse", "true");
}
private void handleOptimizationOption(final Map<String, Boolean> options, final String optionName, final String sysOptionName) {
diff --git a/src/main/java/org/codehaus/groovy/control/SourceUnit.java b/src/main/java/org/codehaus/groovy/control/SourceUnit.java
index b25cee0..c23653b 100644
--- a/src/main/java/org/codehaus/groovy/control/SourceUnit.java
+++ b/src/main/java/org/codehaus/groovy/control/SourceUnit.java
@@ -240,6 +240,19 @@ public class SourceUnit extends ProcessingUnit {
//
// Build the AST
+ buildAST();
+
+ String property = (String) AccessController.doPrivileged((PrivilegedAction) () -> System.getProperty("groovy.ast"));
+
+ if ("xml".equals(property)) {
+ XStreamUtils.serialize(name, ast);
+ }
+ }
+
+ public ModuleNode buildAST() {
+ if (null != this.ast) {
+ return this.ast;
+ }
try {
this.ast = parserPlugin.buildAST(this, this.classLoader, this.cst);
@@ -252,11 +265,7 @@ public class SourceUnit extends ProcessingUnit {
getErrorCollector().addError(new SyntaxErrorMessage(e, this));
}
- String property = (String) AccessController.doPrivileged((PrivilegedAction) () -> System.getProperty("groovy.ast"));
-
- if ("xml".equals(property)) {
- XStreamUtils.serialize(name, ast);
- }
+ return this.ast;
}
//---------------------------------------------------------------------------
diff --git a/src/test/groovy/bugs/Groovy9589.groovy b/src/test/groovy/bugs/Groovy9589.groovy
new file mode 100644
index 0000000..c1a903c
--- /dev/null
+++ b/src/test/groovy/bugs/Groovy9589.groovy
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package groovy.bugs
+
+import org.apache.groovy.parser.antlr4.TestUtils
+import org.codehaus.groovy.ast.ModuleNode
+import org.codehaus.groovy.control.CompilationUnit
+import org.codehaus.groovy.control.CompilerConfiguration
+import org.codehaus.groovy.control.Phases
+import org.junit.Test
+
+final class Groovy9589 {
+ @Test
+ void testParallelParse() {
+ final cnt = 2
+ def sources = (1..cnt).inject([:]) { r, e -> r["hello${e}.groovy"] = source(e); r }
+ def ast1 = parse(sources, false)
+ def ast2 = parse(sources, true)
+
+ assert cnt == ast1.modules.size()
+ assert ast1.modules.size() == ast2.modules.size()
+
+ out:
+ for (ModuleNode m1 in ast1.modules) {
+ for (ModuleNode m2 in ast2.modules) {
+ if (TestUtils.compareAST(m1, m2)) continue out
+ }
+
+ assert false
+ }
+ }
+
+ private parse(Map sources, boolean parallelParse) {
+ new CompilationUnit(new CompilerConfiguration(optimizationOptions: [parallelParse: parallelParse])).tap {
+ sources.each { k, v ->
+ addSource k, v
+ }
+ compile Phases.CONVERSION
+ }.ast
+ }
+
+
+ private static String source(int index) {
+ """
+class Hello${index} {
+ def m() {
+ println 'Hello'
+ }
+}
+ """
+ }
+}
diff --git a/subprojects/parser-antlr4/src/test/groovy/org/apache/groovy/parser/antlr4/TestUtils.groovy b/subprojects/parser-antlr4/src/test/groovy/org/apache/groovy/parser/antlr4/TestUtils.groovy
index 6dc881d..ccb8233 100644
--- a/subprojects/parser-antlr4/src/test/groovy/org/apache/groovy/parser/antlr4/TestUtils.groovy
+++ b/subprojects/parser-antlr4/src/test/groovy/org/apache/groovy/parser/antlr4/TestUtils.groovy
@@ -171,6 +171,18 @@ final class TestUtils {
assert genSrc(ast1) == genSrc(ast2)
}
+ static boolean compareAST(ModuleNode ast1, ModuleNode ast2) {
+ boolean r1 = ast1 != null && ast2 != null
+ boolean r2
+ ASTComparatorCategory.apply(ASTComparatorCategory.DEFAULT_CONFIGURATION) {
+ r2 = ast1 == ast2
+ }
+
+ boolean r3 = genSrc(ast1) == genSrc(ast2)
+
+ return r1 && r2 && r3
+ }
+
static String genSrc(ModuleNode ast) {
return new AstDumper(ast).gen()
}