You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ke...@apache.org on 2016/10/18 20:29:24 UTC
[1/3] incubator-beam git commit: [BEAM-595] avoid throwing Exception
in waitUntilFinish in SparkRunner
Repository: incubator-beam
Updated Branches:
refs/heads/master 7f562cc10 -> a18c27488
[BEAM-595] avoid throwing Exception in waitUntilFinish in SparkRunner
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c92f9861
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c92f9861
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c92f9861
Branch: refs/heads/master
Commit: c92f9861bad7f3352294cc2eb062f1e48be18445
Parents: b8e6eea
Author: Pei He <pe...@google.com>
Authored: Thu Oct 13 14:14:15 2016 -0700
Committer: Pei He <pe...@google.com>
Committed: Mon Oct 17 11:43:04 2016 -0700
----------------------------------------------------------------------
.../beam/runners/spark/translation/EvaluationContext.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c92f9861/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 317c7be..83ef3c5 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -309,7 +309,9 @@ public class EvaluationContext implements EvaluationResult {
@Override
public State waitUntilFinish(Duration duration)
throws IOException, InterruptedException {
- throw new UnsupportedOperationException(
- "Spark runner EvaluationContext does not support waitUntilFinish.");
+ // This is no-op, since Spark runner is blocking.
+ // It needs to be updated once SparkRunner supports non-blocking execution:
+ // https://issues.apache.org/jira/browse/BEAM-595
+ return State.DONE;
}
}
[2/3] incubator-beam git commit: Keep throws in
StreamingEvaluationContext cancel() and waitUntilFinish
Posted by ke...@apache.org.
Keep throws in StreamingEvaluationContext cancel() and waitUntilFinish
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b37526ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b37526ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b37526ea
Branch: refs/heads/master
Commit: b37526ea53f82f294eaed8e258c357a46ff01cd6
Parents: c92f986
Author: Pei He <pe...@google.com>
Authored: Mon Oct 17 11:41:41 2016 -0700
Committer: Pei He <pe...@google.com>
Committed: Tue Oct 18 10:57:49 2016 -0700
----------------------------------------------------------------------
.../spark/translation/EvaluationContext.java | 2 +-
.../streaming/StreamingEvaluationContext.java | 22 ++++++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b37526ea/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
index 83ef3c5..c1c65dd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/EvaluationContext.java
@@ -309,7 +309,7 @@ public class EvaluationContext implements EvaluationResult {
@Override
public State waitUntilFinish(Duration duration)
throws IOException, InterruptedException {
- // This is no-op, since Spark runner is blocking.
+ // This is no-op, since Spark runner in batch is blocking.
// It needs to be updated once SparkRunner supports non-blocking execution:
// https://issues.apache.org/jira/browse/BEAM-595
return State.DONE;
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/b37526ea/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
index 0b32dfd..2652f2b 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/streaming/StreamingEvaluationContext.java
@@ -20,6 +20,7 @@ package org.apache.beam.runners.spark.translation.streaming;
import com.google.common.collect.Iterables;
+import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.Map;
@@ -45,6 +46,7 @@ import org.apache.spark.api.java.function.VoidFunction;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaDStreamLike;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
+import org.joda.time.Duration;
/**
@@ -198,6 +200,26 @@ public class StreamingEvaluationContext extends EvaluationContext {
return state;
}
+ @Override
+ public State cancel() throws IOException {
+ throw new UnsupportedOperationException(
+ "Spark runner StreamingEvaluationContext does not support cancel.");
+ }
+
+ @Override
+ public State waitUntilFinish()
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException(
+ "Spark runner StreamingEvaluationContext does not support waitUntilFinish.");
+ }
+
+ @Override
+ public State waitUntilFinish(Duration duration)
+ throws IOException, InterruptedException {
+ throw new UnsupportedOperationException(
+ "Spark runner StreamingEvaluationContext does not support waitUntilFinish.");
+ }
+
//---------------- override in order to expose in package
@Override
protected <InputT extends PInput> InputT getInput(PTransform<InputT, ?> transform) {
[3/3] incubator-beam git commit: This closes #1096
Posted by ke...@apache.org.
This closes #1096
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a18c2748
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a18c2748
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a18c2748
Branch: refs/heads/master
Commit: a18c27488666c02ace2731fb4a040ca903e43b7b
Parents: 7f562cc b37526e
Author: Kenneth Knowles <kl...@google.com>
Authored: Tue Oct 18 13:20:11 2016 -0700
Committer: Kenneth Knowles <kl...@google.com>
Committed: Tue Oct 18 13:20:11 2016 -0700
----------------------------------------------------------------------
.../spark/translation/EvaluationContext.java | 6 ++++--
.../streaming/StreamingEvaluationContext.java | 22 ++++++++++++++++++++
2 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------