You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2016/07/26 02:17:56 UTC
[1/2] incubator-beam git commit: Remove overrides of isStreaming()
and getAppName() in SparkPipelineOptions
Repository: incubator-beam
Updated Branches:
refs/heads/master 7809f6bd2 -> 267136fb6
Remove overrides of isStreaming() and getAppName() in SparkPipelineOptions
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f52a656b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f52a656b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f52a656b
Branch: refs/heads/master
Commit: f52a656babc5de1dd8f9dc77bffc28c1726407a2
Parents: 7809f6b
Author: Pei He <pe...@google.com>
Authored: Fri Jul 22 16:49:34 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Jul 25 19:16:26 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/runners/spark/SparkPipelineOptions.java | 9 ---------
.../beam/runners/spark/SparkStreamingPipelineOptions.java | 9 ---------
.../spark/translation/streaming/FlattenStreamingTest.java | 2 +-
.../spark/translation/streaming/KafkaStreamingTest.java | 2 +-
.../translation/streaming/SimpleStreamingWordCountTest.java | 2 +-
5 files changed, 3 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 091382e..4bb2a57 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -32,14 +32,5 @@ public interface SparkPipelineOptions extends PipelineOptions, StreamingOptions,
@Description("The url of the spark master to connect to, (e.g. spark://host:port, local[4]).")
@Default.String("local[1]")
String getSparkMaster();
-
void setSparkMaster(String master);
-
- @Override
- @Default.Boolean(false)
- boolean isStreaming();
-
- @Override
- @Default.String("spark dataflow pipeline job")
- String getAppName();
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
index c789447..5944acd 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkStreamingPipelineOptions.java
@@ -28,14 +28,5 @@ public interface SparkStreamingPipelineOptions extends SparkPipelineOptions {
+ "execution is stopped")
@Default.Long(-1)
Long getTimeout();
-
void setTimeout(Long batchInterval);
-
- @Override
- @Default.Boolean(true)
- boolean isStreaming();
-
- @Override
- @Default.String("spark streaming dataflow pipeline job")
- String getAppName();
}
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
index 160f21d..ed7e9b7 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/FlattenStreamingTest.java
@@ -59,8 +59,8 @@ public class FlattenStreamingTest {
public void testRun() throws Exception {
SparkStreamingPipelineOptions options =
PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
- options.setAppName(this.getClass().getSimpleName());
options.setRunner(SparkRunner.class);
+ options.setStreaming(true);
options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
Pipeline p = Pipeline.create(options);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
index 5578e35..904b448 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/KafkaStreamingTest.java
@@ -91,8 +91,8 @@ public class KafkaStreamingTest {
// test read from Kafka
SparkStreamingPipelineOptions options =
PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
- options.setAppName(this.getClass().getSimpleName());
options.setRunner(SparkRunner.class);
+ options.setStreaming(true);
options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
Pipeline p = Pipeline.create(options);
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f52a656b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
----------------------------------------------------------------------
diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
index 75ab274..4fa03f6 100644
--- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/streaming/SimpleStreamingWordCountTest.java
@@ -56,8 +56,8 @@ public class SimpleStreamingWordCountTest implements Serializable {
public void testRun() throws Exception {
SparkStreamingPipelineOptions options =
PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
- options.setAppName(this.getClass().getSimpleName());
options.setRunner(SparkRunner.class);
+ options.setStreaming(true);
options.setTimeout(TEST_TIMEOUT_MSEC); // run for one interval
Pipeline p = Pipeline.create(options);
[2/2] incubator-beam git commit: Remove overrides of isStreaming()
and getAppName() in SparkPipelineOptions
Posted by lc...@apache.org.
Remove overrides of isStreaming() and getAppName() in SparkPipelineOptions
This closes #720
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/267136fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/267136fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/267136fb
Branch: refs/heads/master
Commit: 267136fb609d474b418e5064a52bc1b52753a31e
Parents: 7809f6b f52a656
Author: Luke Cwik <lc...@google.com>
Authored: Mon Jul 25 19:17:39 2016 -0700
Committer: Luke Cwik <lc...@google.com>
Committed: Mon Jul 25 19:17:39 2016 -0700
----------------------------------------------------------------------
.../org/apache/beam/runners/spark/SparkPipelineOptions.java | 9 ---------
.../beam/runners/spark/SparkStreamingPipelineOptions.java | 9 ---------
.../spark/translation/streaming/FlattenStreamingTest.java | 2 +-
.../spark/translation/streaming/KafkaStreamingTest.java | 2 +-
.../translation/streaming/SimpleStreamingWordCountTest.java | 2 +-
5 files changed, 3 insertions(+), 21 deletions(-)
----------------------------------------------------------------------