You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/09/26 18:48:53 UTC
git commit: SPARK-3639 | Removed settings master in examples
Repository: spark
Updated Branches:
refs/heads/master 1aa549ba9 -> d16e161d7
SPARK-3639 | Removed settings master in examples
This patch removes setting of master as local in Kinesis examples so that users can set it using submit-job.
Author: aniketbhatnagar <an...@gmail.com>
Closes #2536 from aniketbhatnagar/Kinesis-Examples-Master-Unset and squashes the following commits:
c9723ac [aniketbhatnagar] Merge remote-tracking branch 'origin/Kinesis-Examples-Master-Unset' into Kinesis-Examples-Master-Unset
fec8ead [aniketbhatnagar] SPARK-3639 | Removed settings master in examples
31cdc59 [aniketbhatnagar] SPARK-3639 | Removed settings master in examples
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d16e161d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d16e161d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d16e161d
Branch: refs/heads/master
Commit: d16e161d744b27291fd2ee7e3578917ee14d83f9
Parents: 1aa549b
Author: aniketbhatnagar <an...@gmail.com>
Authored: Fri Sep 26 09:47:58 2014 -0700
Committer: Andrew Or <an...@gmail.com>
Committed: Fri Sep 26 09:48:46 2014 -0700
----------------------------------------------------------------------
.../examples/streaming/JavaKinesisWordCountASL.java | 9 ++++-----
.../spark/examples/streaming/KinesisWordCountASL.scala | 13 +++++--------
2 files changed, 9 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d16e161d/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java b/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
index aa917d0..b0bff27 100644
--- a/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
+++ b/extras/kinesis-asl/src/main/java/org/apache/spark/examples/streaming/JavaKinesisWordCountASL.java
@@ -71,6 +71,9 @@ import com.google.common.collect.Lists;
* org.apache.spark.examples.streaming.JavaKinesisWordCountASL mySparkStream \
* https://kinesis.us-east-1.amazonaws.com
*
+ * Note that number of workers/threads should be 1 more than the number of receivers.
+ * This leaves one thread available for actually processing the data.
+ *
* There is a companion helper class called KinesisWordCountProducerASL which puts dummy data
* onto the Kinesis stream.
* Usage instructions for KinesisWordCountProducerASL are provided in the class definition.
@@ -114,12 +117,8 @@ public final class JavaKinesisWordCountASL { // needs to be public for access fr
/* In this example, we're going to create 1 Kinesis Worker/Receiver/DStream for each shard */
int numStreams = numShards;
- /* Must add 1 more thread than the number of receivers or the output won't show properly from the driver */
- int numSparkThreads = numStreams + 1;
-
/* Setup the Spark config. */
- SparkConf sparkConfig = new SparkConf().setAppName("KinesisWordCount").setMaster(
- "local[" + numSparkThreads + "]");
+ SparkConf sparkConfig = new SparkConf().setAppName("KinesisWordCount");
/* Kinesis checkpoint interval. Same as batchInterval for this example. */
Duration checkpointInterval = batchInterval;
http://git-wip-us.apache.org/repos/asf/spark/blob/d16e161d/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala b/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
index fffd90d..32da085 100644
--- a/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
+++ b/extras/kinesis-asl/src/main/scala/org/apache/spark/examples/streaming/KinesisWordCountASL.scala
@@ -65,6 +65,10 @@ import org.apache.log4j.Level
* org.apache.spark.examples.streaming.KinesisWordCountASL mySparkStream \
* https://kinesis.us-east-1.amazonaws.com
*
+ *
+ * Note that number of workers/threads should be 1 more than the number of receivers.
+ * This leaves one thread available for actually processing the data.
+ *
* There is a companion helper class below called KinesisWordCountProducerASL which puts
* dummy data onto the Kinesis stream.
* Usage instructions for KinesisWordCountProducerASL are provided in that class definition.
@@ -97,17 +101,10 @@ private object KinesisWordCountASL extends Logging {
/* In this example, we're going to create 1 Kinesis Worker/Receiver/DStream for each shard. */
val numStreams = numShards
- /*
- * numSparkThreads should be 1 more thread than the number of receivers.
- * This leaves one thread available for actually processing the data.
- */
- val numSparkThreads = numStreams + 1
-
/* Setup the and SparkConfig and StreamingContext */
/* Spark Streaming batch interval */
- val batchInterval = Milliseconds(2000)
+ val batchInterval = Milliseconds(2000)
val sparkConfig = new SparkConf().setAppName("KinesisWordCount")
- .setMaster(s"local[$numSparkThreads]")
val ssc = new StreamingContext(sparkConfig, batchInterval)
/* Kinesis checkpoint interval. Same as batchInterval for this example. */
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org