You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/21 07:26:28 UTC
[1/2] git commit: Merge pull request #483 from pwendell/gitignore
Updated Branches:
refs/heads/branch-0.9 50b88ffcc -> 410ba06ff
Merge pull request #483 from pwendell/gitignore
Restricting /lib to top level directory in .gitignore
This patch was proposed by Sean Mackrory.
(cherry picked from commit 7373ffb5e794d3163d3f8d1801836c891e0d6cca)
Signed-off-by: Patrick Wendell <pw...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/f1379479
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/f1379479
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/f1379479
Branch: refs/heads/branch-0.9
Commit: f1379479998a40d8774b014459e58a90c82b2feb
Parents: 50b88ff
Author: Reynold Xin <rx...@apache.org>
Authored: Mon Jan 20 21:44:29 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Jan 20 22:24:07 2014 -0800
----------------------------------------------------------------------
.gitignore | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/f1379479/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 39635d7..3d17899 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,4 +44,4 @@ derby.log
dist/
spark-*-bin.tar.gz
unit-tests.log
-lib/
+/lib/
[2/2] git commit: Merge pull request #482 from
tdas/streaming-example-fix
Posted by pw...@apache.org.
Merge pull request #482 from tdas/streaming-example-fix
Added StreamingContext.awaitTermination to streaming examples
StreamingContext.start() currently starts a non-daemon thread which prevents termination of a Spark Streaming program even if main function has exited. Since the expected behavior of a streaming program is to run until explicitly killed, this was sort of fine when spark streaming applications are launched from the command line. However, when launched in Yarn-standalone mode, this did not work as the driver effectively got terminated when the main function exits. So SparkStreaming examples did not work on Yarn.
This addition to the examples ensures that the examples work on Yarn and also ensures that everyone learns that StreamingContext.awaitTermination() being necessary for SparkStreaming programs to wait.
The true bug-fix of making sure all threads by Spark Streaming are daemon threads is left for post-0.9.
(cherry picked from commit 0367981d47761cdccd8a44fc6fe803079979c5e3)
Signed-off-by: Patrick Wendell <pw...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/410ba06f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/410ba06f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/410ba06f
Branch: refs/heads/branch-0.9
Commit: 410ba06ff0d7c7bfd31621f6d7d95d7eab00cb1a
Parents: f137947
Author: Patrick Wendell <pw...@gmail.com>
Authored: Mon Jan 20 22:25:50 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Mon Jan 20 22:26:14 2014 -0800
----------------------------------------------------------------------
.../org/apache/spark/streaming/examples/JavaFlumeEventCount.java | 1 +
.../org/apache/spark/streaming/examples/JavaKafkaWordCount.java | 1 +
.../org/apache/spark/streaming/examples/JavaNetworkWordCount.java | 1 +
.../java/org/apache/spark/streaming/examples/JavaQueueStream.java | 1 +
.../scala/org/apache/spark/streaming/examples/ActorWordCount.scala | 1 +
.../scala/org/apache/spark/streaming/examples/FlumeEventCount.scala | 1 +
.../scala/org/apache/spark/streaming/examples/HdfsWordCount.scala | 1 +
.../scala/org/apache/spark/streaming/examples/KafkaWordCount.scala | 1 +
.../scala/org/apache/spark/streaming/examples/MQTTWordCount.scala | 1 +
.../org/apache/spark/streaming/examples/NetworkWordCount.scala | 1 +
.../scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala | 1 +
.../spark/streaming/examples/RecoverableNetworkWordCount.scala | 1 +
.../apache/spark/streaming/examples/StatefulNetworkWordCount.scala | 1 +
.../org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala | 1 +
.../org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala | 1 +
.../org/apache/spark/streaming/examples/TwitterPopularTags.scala | 1 +
.../scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala | 1 +
17 files changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java b/examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java
index 7b5a243..f061001 100644
--- a/examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java
+++ b/examples/src/main/java/org/apache/spark/streaming/examples/JavaFlumeEventCount.java
@@ -70,5 +70,6 @@ public final class JavaFlumeEventCount {
}).print();
ssc.start();
+ ssc.awaitTermination();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java b/examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java
index 04f62ee..2ffd351 100644
--- a/examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java
+++ b/examples/src/main/java/org/apache/spark/streaming/examples/JavaKafkaWordCount.java
@@ -104,5 +104,6 @@ public final class JavaKafkaWordCount {
wordCounts.print();
jssc.start();
+ jssc.awaitTermination();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/java/org/apache/spark/streaming/examples/JavaNetworkWordCount.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/spark/streaming/examples/JavaNetworkWordCount.java b/examples/src/main/java/org/apache/spark/streaming/examples/JavaNetworkWordCount.java
index 349d826..7777c98 100644
--- a/examples/src/main/java/org/apache/spark/streaming/examples/JavaNetworkWordCount.java
+++ b/examples/src/main/java/org/apache/spark/streaming/examples/JavaNetworkWordCount.java
@@ -84,5 +84,6 @@ public final class JavaNetworkWordCount {
wordCounts.print();
ssc.start();
+ ssc.awaitTermination();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/java/org/apache/spark/streaming/examples/JavaQueueStream.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/spark/streaming/examples/JavaQueueStream.java b/examples/src/main/java/org/apache/spark/streaming/examples/JavaQueueStream.java
index e2d55f1..26c4462 100644
--- a/examples/src/main/java/org/apache/spark/streaming/examples/JavaQueueStream.java
+++ b/examples/src/main/java/org/apache/spark/streaming/examples/JavaQueueStream.java
@@ -80,5 +80,6 @@ public final class JavaQueueStream {
reducedStream.print();
ssc.start();
+ ssc.awaitTermination();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala
index 5a4aa7f..a588881 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/ActorWordCount.scala
@@ -171,5 +171,6 @@ object ActorWordCount {
lines.flatMap(_.split("\\s+")).map(x => (x, 1)).reduceByKey(_ + _).print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/FlumeEventCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/FlumeEventCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/FlumeEventCount.scala
index a59be78..11c3aaa 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/FlumeEventCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/FlumeEventCount.scala
@@ -60,5 +60,6 @@ object FlumeEventCount {
stream.count().map(cnt => "Received " + cnt + " flume events." ).print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/HdfsWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/HdfsWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/HdfsWordCount.scala
index 704b315..954bcc9 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/HdfsWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/HdfsWordCount.scala
@@ -50,6 +50,7 @@ object HdfsWordCount {
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala
index 4a3d81c..d9cb732 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/KafkaWordCount.scala
@@ -61,6 +61,7 @@ object KafkaWordCount {
wordCounts.print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/MQTTWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/MQTTWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/MQTTWordCount.scala
index 78b49fd..eb61caf 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/MQTTWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/MQTTWordCount.scala
@@ -101,5 +101,6 @@ object MQTTWordCount {
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/NetworkWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/NetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/NetworkWordCount.scala
index 0226475..5656d48 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/NetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/NetworkWordCount.scala
@@ -54,5 +54,6 @@ object NetworkWordCount {
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala
index 99b79c3..cdd7547 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/RawNetworkGrep.scala
@@ -61,5 +61,6 @@ object RawNetworkGrep {
union.filter(_.contains("the")).count().foreachRDD(r =>
println("Grep count: " + r.collect().mkString))
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/RecoverableNetworkWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/RecoverableNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/RecoverableNetworkWordCount.scala
index 8c5d0bd..aa82bf3 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/RecoverableNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/RecoverableNetworkWordCount.scala
@@ -114,5 +114,6 @@ object RecoverableNetworkWordCount {
createContext(master, ip, port, outputPath)
})
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/StatefulNetworkWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/StatefulNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/StatefulNetworkWordCount.scala
index 1183eba..88f1cef 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/StatefulNetworkWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/StatefulNetworkWordCount.scala
@@ -65,5 +65,6 @@ object StatefulNetworkWordCount {
val stateDstream = wordDstream.updateStateByKey[Int](updateFunc)
stateDstream.print()
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala
index 483c4d3..bbd4494 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdCMS.scala
@@ -110,5 +110,6 @@ object TwitterAlgebirdCMS {
})
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala
index 94c2bf2..a0094d4 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterAlgebirdHLL.scala
@@ -87,5 +87,6 @@ object TwitterAlgebirdHLL {
})
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterPopularTags.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterPopularTags.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterPopularTags.scala
index 8a70d4a..896d010 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterPopularTags.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/TwitterPopularTags.scala
@@ -69,5 +69,6 @@ object TwitterPopularTags {
})
ssc.start()
+ ssc.awaitTermination()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/410ba06f/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala b/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
index 12d2a10..85b4ce5 100644
--- a/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
+++ b/examples/src/main/scala/org/apache/spark/streaming/examples/ZeroMQWordCount.scala
@@ -91,5 +91,6 @@ object ZeroMQWordCount {
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
+ ssc.awaitTermination()
}
}