You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by am...@apache.org on 2016/04/11 09:07:29 UTC
[1/2] incubator-beam git commit: [BEAM-43] Upgrade to Spark 1.6.1
Repository: incubator-beam
Updated Branches:
refs/heads/master d42a07086 -> 1cef64261
[BEAM-43] Upgrade to Spark 1.6.1
Replace deprecated Function<T, Void> with VoidFunction<T>
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/d77a7902
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/d77a7902
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/d77a7902
Branch: refs/heads/master
Commit: d77a7902f9c64781001c60ac7e13e19754fb80ff
Parents: d42a070
Author: Sela <an...@paypal.com>
Authored: Sun Apr 10 20:34:17 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Mon Apr 11 10:03:35 2016 +0300
----------------------------------------------------------------------
runners/spark/pom.xml | 2 +-
.../translation/streaming/StreamingEvaluationContext.java | 7 +++----
.../translation/streaming/StreamingTransformTranslator.java | 6 +++---
3 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d77a7902/runners/spark/pom.xml
----------------------------------------------------------------------
diff --git a/runners/spark/pom.xml b/runners/spark/pom.xml
index 7fa8392..edcac6b 100644
--- a/runners/spark/pom.xml
+++ b/runners/spark/pom.xml
@@ -36,7 +36,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.7</java.version>
- <spark.version>1.5.2</spark.version>
+ <spark.version>1.6.1</spark.version>
<beam.version>0.1.0-incubating-SNAPSHOT</beam.version>
</properties>
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d77a7902/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
index 0e87355..b35c47c 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
@@ -40,7 +40,7 @@ import org.apache.beam.runners.spark.translation.SparkRuntimeContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaRDDLike;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaDStreamLike;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
@@ -150,12 +150,11 @@ public class StreamingEvaluationContext extends EvaluationContext {
}
private static <T> void computeOutput(DStreamHolder<T> streamHolder) {
- streamHolder.getDStream().foreachRDD(new Function<JavaRDD<WindowedValue<T>>, Void>() {
+ streamHolder.getDStream().foreachRDD(new VoidFunction<JavaRDD<WindowedValue<T>>>() {
@Override
- public Void call(JavaRDD<WindowedValue<T>> rdd) throws Exception {
+ public void call(JavaRDD<WindowedValue<T>> rdd) throws Exception {
rdd.rdd().cache();
rdd.count();
- return null;
}
}); // force a DStream action
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/d77a7902/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
index fdab667..711358e 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingTransformTranslator.java
@@ -63,6 +63,7 @@ import org.apache.beam.runners.spark.translation.WindowingHelpers;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
+import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
@@ -274,7 +275,7 @@ public final class StreamingTransformTranslator {
* @param <PT> PTransform type
*/
private static final class RDDOutputOperator<PT extends PTransform<?, ?>>
- implements Function<JavaRDD<WindowedValue<Object>>, Void> {
+ implements VoidFunction<JavaRDD<WindowedValue<Object>>> {
private final StreamingEvaluationContext context;
private final AppliedPTransform<?, ?, ?> appliedPTransform;
@@ -292,13 +293,12 @@ public final class StreamingTransformTranslator {
@Override
@SuppressWarnings("unchecked")
- public Void call(JavaRDD<WindowedValue<Object>> rdd) throws Exception {
+ public void call(JavaRDD<WindowedValue<Object>> rdd) throws Exception {
AppliedPTransform<?, ?, ?> existingAPT = context.getCurrentTransform();
context.setCurrentTransform(appliedPTransform);
context.setInputRDD(transform, rdd);
rddEvaluator.evaluate(transform, context);
context.setCurrentTransform(existingAPT);
- return null;
}
}
[2/2] incubator-beam git commit: [BEAM-43] Upgrade to Spark 1.6
Posted by am...@apache.org.
[BEAM-43] Upgrade to Spark 1.6
This closes #159
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1cef6426
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1cef6426
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1cef6426
Branch: refs/heads/master
Commit: 1cef642610631e606092f895b6efbf30c84a2e4a
Parents: d42a070 d77a790
Author: Sela <an...@paypal.com>
Authored: Mon Apr 11 10:06:24 2016 +0300
Committer: Sela <an...@paypal.com>
Committed: Mon Apr 11 10:06:24 2016 +0300
----------------------------------------------------------------------
runners/spark/pom.xml | 2 +-
.../translation/streaming/StreamingEvaluationContext.java | 7 +++----
.../translation/streaming/StreamingTransformTranslator.java | 6 +++---
3 files changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------