You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/05/29 00:51:26 UTC
git commit: Spark 1916
Repository: spark
Updated Branches:
refs/heads/branch-1.0 032493e12 -> 0b769b73f
Spark 1916
The changes could be ported back to 0.9 as well.
Changing in.read to in.readFully to read the whole input stream rather than the first 1020 bytes.
This should ok considering that Flume caps the body size to 32K by default.
Author: David Lemieux <da...@radialpoint.com>
Closes #865 from lemieud/SPARK-1916 and squashes the following commits:
a265673 [David Lemieux] Updated SparkFlumeEvent to read the whole stream rather than the first X bytes.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0b769b73
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0b769b73
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0b769b73
Branch: refs/heads/branch-1.0
Commit: 0b769b73fb7ae314325857138a2d3138ed157908
Parents: 032493e
Author: David Lemieux <da...@radialpoint.com>
Authored: Wed May 28 15:50:35 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed May 28 15:50:35 2014 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0b769b73/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
----------------------------------------------------------------------
diff --git a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
index df7605f..5be33f1 100644
--- a/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
+++ b/external/flume/src/main/scala/org/apache/spark/streaming/flume/FlumeInputDStream.scala
@@ -63,7 +63,7 @@ class SparkFlumeEvent() extends Externalizable {
def readExternal(in: ObjectInput) {
val bodyLength = in.readInt()
val bodyBuff = new Array[Byte](bodyLength)
- in.read(bodyBuff)
+ in.readFully(bodyBuff)
val numHeaders = in.readInt()
val headers = new java.util.HashMap[CharSequence, CharSequence]