You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2021/09/06 02:17:32 UTC
[incubator-wayang] 01/01: [WAYANG-37] update Apache Spark version
from 2.4.0 to 3.1.2
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch WAYANG-37
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit f72e543aab2fd582b6cb7ea48a73e16dbb22148b
Author: Bertty Contreras-Rojas <be...@scalytics.io>
AuthorDate: Mon Sep 6 04:16:47 2021 +0200
[WAYANG-37] update Apache Spark version from 2.4.0 to 3.1.2
Signed-off-by: bertty <be...@gmail.com>
---
wayang-api/wayang-api-python/pom.xml | 4 ++--
wayang-api/wayang-api-scala-java/pom.xml | 4 ++--
wayang-commons/pom.xml | 2 +-
wayang-platforms/wayang-spark/pom.xml | 14 +++++++++++++-
.../java/org/apache/wayang/spark/channels/RddChannel.java | 8 +++++---
.../src/main/resources/wayang-spark-defaults.properties | 1 +
.../wayang/spark/operators/SparkOperatorTestBase.java | 3 ++-
wayang-plugins/wayang-iejoin/pom.xml | 3 ++-
wayang-profiler/pom.xml | 2 +-
wayang-tests-integration/pom.xml | 2 +-
10 files changed, 30 insertions(+), 13 deletions(-)
diff --git a/wayang-api/wayang-api-python/pom.xml b/wayang-api/wayang-api-python/pom.xml
index 03588cd..5fa5dc8 100644
--- a/wayang-api/wayang-api-python/pom.xml
+++ b/wayang-api/wayang-api-python/pom.xml
@@ -36,6 +36,6 @@
<properties>
<java-module-name>org.apache.wayang.api</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
-</project>
\ No newline at end of file
+</project>
diff --git a/wayang-api/wayang-api-scala-java/pom.xml b/wayang-api/wayang-api-scala-java/pom.xml
index b7e9670..a1cbb5e 100644
--- a/wayang-api/wayang-api-scala-java/pom.xml
+++ b/wayang-api/wayang-api-scala-java/pom.xml
@@ -34,7 +34,7 @@
<properties>
<java-module-name>org.apache.wayang.api</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
<dependencyManagement>
@@ -68,7 +68,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.3.2</version>
+ <version>3.12.0</version>
</dependency>
<dependency>
<groupId>org.apache.wayang</groupId>
diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml
index 5bb233b..b5c111b 100644
--- a/wayang-commons/pom.xml
+++ b/wayang-commons/pom.xml
@@ -176,7 +176,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
- <version>3.3.2</version>
+ <version>3.12.0</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
diff --git a/wayang-platforms/wayang-spark/pom.xml b/wayang-platforms/wayang-spark/pom.xml
index 7f8a976..1a0d786 100644
--- a/wayang-platforms/wayang-spark/pom.xml
+++ b/wayang-platforms/wayang-spark/pom.xml
@@ -36,7 +36,7 @@
<properties>
<java-module-name>org.apache.wayang.platform.spark</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
@@ -86,6 +86,18 @@
<version>2.8</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>2.7.7</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>2.7.7</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java
index 36f1877..7f4f360 100644
--- a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java
+++ b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/RddChannel.java
@@ -18,8 +18,9 @@
package org.apache.wayang.spark.channels;
-import org.apache.spark.Accumulator;
+import org.apache.spark.util.AccumulatorV2;
import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.util.LongAccumulator;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.OptimizationContext;
import org.apache.wayang.core.plan.executionplan.Channel;
@@ -75,7 +76,7 @@ public class RddChannel extends Channel {
private JavaRDD<?> rdd;
- private Accumulator<Integer> accumulator;
+ private LongAccumulator accumulator;
public Instance(SparkExecutor executor,
OptimizationContext.OperatorContext producerOperatorContext,
@@ -85,7 +86,8 @@ public class RddChannel extends Channel {
public void accept(JavaRDD<?> rdd, SparkExecutor sparkExecutor) throws WayangException {
if (this.isMarkedForInstrumentation() && !this.isRddCached()) {
- final Accumulator<Integer> accumulator = sparkExecutor.sc.accumulator(0);
+ final LongAccumulator accumulator = sparkExecutor.sc.sc().longAccumulator();
+ accumulator.setValue(0);
this.rdd = rdd.filter(dataQuantum -> {
accumulator.add(1);
return true;
diff --git a/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties b/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties
index 4c52ea9..7b4feb3 100644
--- a/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties
+++ b/wayang-platforms/wayang-spark/src/main/resources/wayang-spark-defaults.properties
@@ -18,6 +18,7 @@
spark.master = local[1]
spark.app.name = Wayang App
spark.ui.showConsoleProgress = false
+spark.driver.allowMultipleContexts=true
# spark.driver.memory = 1g
wayang.spark.cpu.mhz = 2700
diff --git a/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java b/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java
index a1ae810..7265ccc 100644
--- a/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java
+++ b/wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkOperatorTestBase.java
@@ -51,7 +51,8 @@ public class SparkOperatorTestBase {
@Before
public void setUp() {
this.configuration = new Configuration();
- this.sparkExecutor = (SparkExecutor) SparkPlatform.getInstance().getExecutorFactory().create(this.mockJob());
+ if(sparkExecutor == null)
+ this.sparkExecutor = (SparkExecutor) SparkPlatform.getInstance().getExecutorFactory().create(this.mockJob());
}
Job mockJob() {
diff --git a/wayang-plugins/wayang-iejoin/pom.xml b/wayang-plugins/wayang-iejoin/pom.xml
index d81104d..8bfb314 100644
--- a/wayang-plugins/wayang-iejoin/pom.xml
+++ b/wayang-plugins/wayang-iejoin/pom.xml
@@ -24,7 +24,8 @@
<properties>
<java-module-name>org.apache.wayang.extensions.iejoin</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
+ <scala.mayor.version>2.12</scala.mayor.version>
</properties>
<dependencies>
diff --git a/wayang-profiler/pom.xml b/wayang-profiler/pom.xml
index 6d9a1b4..0d9d15e 100644
--- a/wayang-profiler/pom.xml
+++ b/wayang-profiler/pom.xml
@@ -34,7 +34,7 @@
<properties>
<java-module-name>org.apache.wayang.profiler</java-module-name>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
</properties>
<dependencyManagement>
diff --git a/wayang-tests-integration/pom.xml b/wayang-tests-integration/pom.xml
index 231c8dd..71a0957 100644
--- a/wayang-tests-integration/pom.xml
+++ b/wayang-tests-integration/pom.xml
@@ -35,7 +35,7 @@
<properties>
<java-module-name>org.apache.wayang.test.integration</java-module-name>
<graphchi.version>0.2.2</graphchi.version>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.1.2</spark.version>
<flink.version>1.7.1</flink.version>
<giraph.version>1.2.0-hadoop2</giraph.version>
</properties>