You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2014/08/29 19:13:13 UTC
[1/8] git commit: fix issue on KafkaOffsetMetric
Repository: incubator-storm
Updated Branches:
refs/heads/master dab609695 -> 002aae294
fix issue on KafkaOffsetMetric
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/96150dfe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/96150dfe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/96150dfe
Branch: refs/heads/master
Commit: 96150dfe6901e0bf89e75c160e01575a711be304
Parents: 85c9cd5
Author: fxjwind <fx...@hotmail.com>
Authored: Tue Jul 22 11:58:32 2014 +0800
Committer: fxjwind <fx...@hotmail.com>
Committed: Tue Jul 22 11:58:32 2014 +0800
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/96150dfe/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
index 25d3709..9e61789 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
@@ -115,7 +115,8 @@ public class KafkaUtils {
}
long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime());
- if (latestTimeOffset == 0 || earliestTimeOffset == 0) {
+ //if (latestTimeOffset == 0 || earliestTimeOffset == 0) {
+ if (latestTimeOffset == 0) {
LOG.warn("No data found in Kafka Partition " + partition.getId());
return null;
}
[8/8] git commit: Add STORM-476, STORM-424 to changelog
Posted by pt...@apache.org.
Add STORM-476, STORM-424 to changelog
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/002aae29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/002aae29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/002aae29
Branch: refs/heads/master
Commit: 002aae294a7070cdc76ea025eb4d8a89e766527b
Parents: 5e63311
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Aug 29 13:10:30 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Aug 29 13:10:30 2014 -0400
----------------------------------------------------------------------
CHANGELOG.md | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/002aae29/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 66465b6..6ce7627 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,6 @@
## 0.9.3-incubating
+ * STORM-476: external/storm-kafka: avoid NPE on null message payloads
+ * STORM-424: fix error message related to kafka offsets
* STORM-454: correct documentation in STORM-UI-REST-API.md
* STORM-474: Reformat UI HTML code
* STORM-447: shade/relocate packages of dependencies that are common causes of dependency conflicts
[2/8] git commit: remove comment
Posted by pt...@apache.org.
remove comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/5e423647
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/5e423647
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/5e423647
Branch: refs/heads/master
Commit: 5e423647f0f238652b4ad7203f83f527881a8f32
Parents: 96150df
Author: fxjwind <fx...@hotmail.com>
Authored: Fri Jul 25 18:16:42 2014 +0800
Committer: fxjwind <fx...@hotmail.com>
Committed: Fri Jul 25 18:16:42 2014 +0800
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/5e423647/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
index 9e61789..6881c57 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
@@ -115,7 +115,6 @@ public class KafkaUtils {
}
long latestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.LatestTime());
long earliestTimeOffset = getOffset(consumer, _topic, partition.partition, kafka.api.OffsetRequest.EarliestTime());
- //if (latestTimeOffset == 0 || earliestTimeOffset == 0) {
if (latestTimeOffset == 0) {
LOG.warn("No data found in Kafka Partition " + partition.getId());
return null;
[5/8] git commit: Merge branch 'master' of
github.com:rick-kilgore/incubator-storm into temp
Posted by pt...@apache.org.
Merge branch 'master' of github.com:rick-kilgore/incubator-storm into temp
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/529c6db1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/529c6db1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/529c6db1
Branch: refs/heads/master
Commit: 529c6db1ac5b648cf636f93c8221d52be685e8ad
Parents: dab6096 4da836f
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Aug 29 12:40:07 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Aug 29 12:40:07 2014 -0400
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[3/8] git commit: don't discard unfinished messages that are not old
Posted by pt...@apache.org.
don't discard unfinished messages that are not old
the ack() message in PartitionManager appears to have a bug that will
cause it to discard messages that have not been acknowledged even though
they have not fallen behind by maxOffsetBehind.
This can happen if there is at least one message that _is_ behind by
maxOffsetBehind. The code correctly identifies that one or more
messages in _pending are old, but then potentially discards some that
are not.
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/4da836f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/4da836f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/4da836f2
Branch: refs/heads/master
Commit: 4da836f251fad76eb9cd3c0ed6a7bb60289ebf42
Parents: d4ebb6d
Author: Rick Kilgore <ri...@hbo.com>
Authored: Wed Aug 20 17:57:47 2014 -0700
Committer: Rick Kilgore <ri...@hbo.com>
Committed: Wed Aug 20 17:57:47 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/4da836f2/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 2c32a2c..9b48678 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -188,10 +188,9 @@ public class PartitionManager {
public void ack(Long offset) {
if (!_pending.isEmpty() && _pending.first() < offset - _spoutConfig.maxOffsetBehind) {
// Too many things pending!
- _pending.headSet(offset).clear();
- } else {
- _pending.remove(offset);
+ _pending.headSet(offset - _spoutConfig.maxOffsetBehind).clear();
}
+ _pending.remove(offset);
numberAcked++;
}
[6/8] git commit: Merge branch 'fixing_stormkafka' of
github.com:fxjwind/incubator-storm into temp
Posted by pt...@apache.org.
Merge branch 'fixing_stormkafka' of github.com:fxjwind/incubator-storm into temp
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/5f61c08b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/5f61c08b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/5f61c08b
Branch: refs/heads/master
Commit: 5f61c08bb4326995ebdca83d823352fb65f17aa3
Parents: 529c6db 5e42364
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Aug 29 12:53:11 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Aug 29 12:53:11 2014 -0400
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[4/8] git commit: external/storm-kafka: avoid NPE on null message
payloads
Posted by pt...@apache.org.
external/storm-kafka: avoid NPE on null message payloads
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/01f7a02b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/01f7a02b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/01f7a02b
Branch: refs/heads/master
Commit: 01f7a02bd0971eb40322d4ab63fd8bc948192544
Parents: 9760867
Author: Scott Bessler <sc...@relateiq.com>
Authored: Mon Aug 25 21:22:43 2014 -0700
Committer: Scott Bessler <sc...@relateiq.com>
Committed: Mon Aug 25 21:22:43 2014 -0700
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/01f7a02b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
index 25d3709..4b9d113 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
@@ -202,6 +202,9 @@ public class KafkaUtils {
public static Iterable<List<Object>> generateTuples(KafkaConfig kafkaConfig, Message msg) {
Iterable<List<Object>> tups;
ByteBuffer payload = msg.payload();
+ if (payload == null) {
+ return null;
+ }
ByteBuffer key = msg.key();
if (key != null && kafkaConfig.scheme instanceof KeyValueSchemeAsMultiScheme) {
tups = ((KeyValueSchemeAsMultiScheme) kafkaConfig.scheme).deserializeKeyAndValue(Utils.toByteArray(key), Utils.toByteArray(payload));
[7/8] git commit: Merge branch 'master' of
github.com:relateiq/incubator-storm into temp
Posted by pt...@apache.org.
Merge branch 'master' of github.com:relateiq/incubator-storm into temp
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/5e633112
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/5e633112
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/5e633112
Branch: refs/heads/master
Commit: 5e6331126b7177c4223133a8c13a47d6ce5668df
Parents: 5f61c08 01f7a02
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Fri Aug 29 12:59:04 2014 -0400
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Fri Aug 29 12:59:04 2014 -0400
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/5e633112/external/storm-kafka/src/jvm/storm/kafka/KafkaUtils.java
----------------------------------------------------------------------