You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/09/06 02:17:31 UTC
[incubator-wayang] branch WAYANG-37 created (now f72e543)
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a change to branch WAYANG-37
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git.
at f72e543 [WAYANG-37] update Apache Spark version from 2.4.0 to 3.1.2
This branch includes the following new commits:
new f72e543 [WAYANG-37] update Apache Spark version from 2.4.0 to 3.1.2
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-wayang] 01/01: [WAYANG-37] update Apache Spark version
from 2.4.0 to 3.1.2
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-37
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit f72e543aab2fd582b6cb7ea48a73e16dbb22148b
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Mon Sep 6 04:16:47 2021 +0200
[WAYANG-37] update Apache Spark version from 2.4.0 to 3.1.2
Signed-off-by: bertty <be...@gmail.com>
---
wayang-api/wayang-api-python/pom.xml | 4 ++--
wayang-api/wayang-api-scala-java/pom.xml | 4 ++--
wayang-commons/pom.xml | 2 +-
wayang-platforms/wayang-spark/pom.xml | 14 +++++++++++++-
.../java/org/apache/wayang/spark/channels/RddChannel.java | 8 +++++---
.../src/main/resources/wayang-spark-defaults.properties | 1 +
.../wayang/spark/operators/SparkOperatorTestBase.java | 3 ++-
wayang-plugins/wayang-iejoin/pom.xml | 3 ++-
wayang-profiler/pom.xml | 2 +-
wayang-tests-integration/pom.xml | 2 +-
10 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/wayang-api/wayang-api-python/pom.xml b/wayang-api/wayang-api-python/pom.xml
index 03588cd..5fa5dc8 100644
--- a/wayang-api/wayang-api-python/pom.xml
+++ b/wayang-api/wayang-api-python/pom.xml
@@ -36,6 +36,6 @@
<properties>
<java-module-name>org.apache.wayang.api</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
-</project>
\ No newline at end of file
+</project>
diff --git a/wayang-api/wayang-api-scala-java/pom.xml b/wayang-api/wayang-api-scala-java/pom.xml
index b7e9670..a1cbb5e 100644
--- a/wayang-api/wayang-api-scala-java/pom.xml
+++ b/wayang-api/wayang-api-scala-java/pom.xml
@@ -34,7 +34,7 @@
<properties>
<java-module-name>org.apache.wayang.api</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
<dependencyManagement>
@@ -68,7 +68,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.3.2</version>
+ <version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml
index 5bb233b..b5c111b 100644
--- a/wayang-commons/pom.xml
+++ b/wayang-commons/pom.xml
@@ -176,7 +176,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.3.2</version>
+ <version>3.12.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
diff --git a/wayang-platforms/wayang-spark/pom.xml b/wayang-platforms/wayang-spark/pom.xml
index 7f8a976..1a0d786 100644
--- a/wayang-platforms/wayang-spark/pom.xml
+++ b/wayang-platforms/wayang-spark/pom.xml
@@ -36,7 +36,7 @@
<properties>
<java-module-name>org.apache.wayang.platform.spark</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
@@ -86,6 +86,18 @@
<version>2.8</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.7.7</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>2.7.7</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java
index 36f1877..7f4f360 100644
--- a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java
+++ b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java
@@ -18,8 +18,9 @@
package org.apache.wayang.spark.channels;
-import org.apache.spark.Accumulator;
+import org.apache.spark.util.AccumulatorV2;
import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.util.LongAccumulator;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.OptimizationContext;
import org.apache.wayang.core.plan.executionplan.Channel;
@@ -75,7 +76,7 @@ public class RddChannel extends Channel {
private JavaRDD<?> rdd;
- private Accumulator<Integer> accumulator;
+ private LongAccumulator accumulator;
public Instance(SparkExecutor executor,
OptimizationContext.OperatorContext producerOperatorContext,
@@ -85,7 +86,8 @@ public class RddChannel extends Channel {
public void accept(JavaRDD<?> rdd, SparkExecutor sparkExecutor) throws WayangException {
if (this.isMarkedForInstrumentation() && !this.isRddCached()) {
- final Accumulator<Integer> accumulator = sparkExecutor.sc.accumulator(0);
+ final LongAccumulator accumulator = sparkExecutor.sc.sc().longAccumulator();
+ accumulator.setValue(0);
this.rdd = rdd.filter(dataQuantum -> {
accumulator.add(1);
return true;
diff --git a/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties b/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties
index 4c52ea9..7b4feb3 100644
--- a/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties
+++ b/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties
@@ -18,6 +18,7 @@
spark.master = local[1]
spark.app.name = Wayang App
spark.ui.showConsoleProgress = false
+spark.driver.allowMultipleContexts=true
# spark.driver.memory = 1g
wayang.spark.cpu.mhz = 2700
diff --git a/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java b/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java
index a1ae810..7265ccc 100644
--- a/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java
+++ b/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java
@@ -51,7 +51,8 @@ public class SparkOperatorTestBase {
@Before
public void setUp() {
this.configuration = new Configuration();
- this.sparkExecutor = (SparkExecutor) SparkPlatform.getInstance().getExecutorFactory().create(this.mockJob());
+ if(sparkExecutor == null)
+ this.sparkExecutor = (SparkExecutor) SparkPlatform.getInstance().getExecutorFactory().create(this.mockJob());
}
Job mockJob() {
diff --git a/wayang-plugins/wayang-iejoin/pom.xml b/wayang-plugins/wayang-iejoin/pom.xml
index d81104d..8bfb314 100644
--- a/wayang-plugins/wayang-iejoin/pom.xml
+++ b/wayang-plugins/wayang-iejoin/pom.xml
@@ -24,7 +24,8 @@
<properties>
<java-module-name>org.apache.wayang.extensions.iejoin</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
<dependencies>
diff --git a/wayang-profiler/pom.xml b/wayang-profiler/pom.xml
index 6d9a1b4..0d9d15e 100644
--- a/wayang-profiler/pom.xml
+++ b/wayang-profiler/pom.xml
@@ -34,7 +34,7 @@
<properties>
<java-module-name>org.apache.wayang.profiler</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
<dependencyManagement>
diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml
index 231c8dd..71a0957 100644
--- a/wayang-tests-integration/pom.xml
+++ b/wayang-tests-integration/pom.xml
@@ -35,7 +35,7 @@
<properties>
<java-module-name>org.apache.wayang.test.integration</java-module-name>
<graphchi.version>0.2.2</graphchi.version>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
<flink.version>1.7.1</flink.version>
<giraph.version>1.2.0-hadoop2</giraph.version>
</properties>