You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2015/08/25 05:05:19 UTC
[01/11] storm git commit: Added Kafka Topic as Stream ID for emitted
tuples
Repository: storm
Updated Branches:
refs/heads/master 949da3853 -> 528958c27
Added Kafka Topic as Stream ID for emitted tuples
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3ba9aba3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3ba9aba3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3ba9aba3
Branch: refs/heads/master
Commit: 3ba9aba3983ac14fe6b25e74e05aa24cb1e9e44e
Parents: 544e55c
Author: rohan_agarwal <ro...@gmail.com>
Authored: Fri Aug 7 17:03:22 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Fri Aug 7 17:03:22 2015 +0530
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/3ba9aba3/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index ce18677..e72cf3c 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -138,7 +138,7 @@ public class PartitionManager {
Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
if (tups != null) {
for (List<Object> tup : tups) {
- collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
+ collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
}
break;
} else {
[02/11] storm git commit: restored
Posted by ka...@apache.org.
restored
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/80c82e30
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/80c82e30
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/80c82e30
Branch: refs/heads/master
Commit: 80c82e30dbd58b030cb72ed9ab13738eaf6dc106
Parents: 3ba9aba
Author: rohan_agarwal <ro...@gmail.com>
Authored: Mon Aug 10 10:36:13 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Mon Aug 10 10:36:13 2015 +0530
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/80c82e30/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index e72cf3c..ce18677 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -138,7 +138,7 @@ public class PartitionManager {
Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
if (tups != null) {
for (List<Object> tup : tups) {
- collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
+ collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
}
break;
} else {
[10/11] storm git commit: Merge branch 'master' of
https://github.com/rohanag12/storm into STORM-854
Posted by ka...@apache.org.
Merge branch 'master' of https://github.com/rohanag12/storm into STORM-854
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d66a1b7b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d66a1b7b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d66a1b7b
Branch: refs/heads/master
Commit: d66a1b7b10d33de864f942a600f4d42bf0a43452
Parents: 949da38 6491c4d
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Aug 25 12:03:17 2015 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Aug 25 12:03:17 2015 +0900
----------------------------------------------------------------------
external/storm-kafka/README.md | 3 +++
external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java | 6 +++++-
.../storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 10 ++++++++--
external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java | 3 +++
4 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[03/11] storm git commit: Added a topicAsStreamId flag to SpoutConfig
Posted by ka...@apache.org.
Added a topicAsStreamId flag to SpoutConfig
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bfa3095e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bfa3095e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bfa3095e
Branch: refs/heads/master
Commit: bfa3095ef871e0fdd76edef5d9ab1daf098b88b8
Parents: 80c82e3
Author: rohan_agarwal <ro...@gmail.com>
Authored: Mon Aug 10 14:35:26 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Mon Aug 10 14:35:26 2015 +0530
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 6 +++++-
external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java | 3 +++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bfa3095e/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index ce18677..77087ec 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -138,7 +138,11 @@ public class PartitionManager {
Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
if (tups != null) {
for (List<Object> tup : tups) {
- collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
+ if(_spoutConfig.topicAsStreamId) {
+ collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
+ } else {
+ collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
+ }
}
break;
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/bfa3095e/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
index 61d0b35..27edd7a 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/SpoutConfig.java
@@ -27,6 +27,9 @@ public class SpoutConfig extends KafkaConfig implements Serializable {
public String zkRoot = null;
public String id = null;
+ // if set to true, spout will set Kafka topic as the emitted Stream ID
+ public boolean topicAsStreamId = false;
+
// setting for how often to save the current kafka offset to ZooKeeper
public long stateUpdateIntervalMs = 2000;
[08/11] storm git commit: Merge remote-tracking branch
'upstream/master'
Posted by ka...@apache.org.
Merge remote-tracking branch 'upstream/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/044ed00c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/044ed00c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/044ed00c
Branch: refs/heads/master
Commit: 044ed00c38a26c5a18b806439aeade5fc2e64035
Parents: e2b08eb 9c94188
Author: rohan_agarwal <ro...@gmail.com>
Authored: Wed Aug 19 10:10:20 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Wed Aug 19 10:10:20 2015 +0530
----------------------------------------------------------------------
CHANGELOG.md | 2 ++
README.markdown | 1 +
SECURITY.md | 2 +-
bin/storm-config.cmd | 18 +++++------
bin/storm.py | 22 ++++++-------
conf/defaults.yaml | 2 ++
docs/documentation/FAQ.md | 7 ++--
.../flux-core/src/test/resources/log4j2.xml | 34 ++++++++++++++++++++
.../flux-core/src/test/resources/logback.xml | 30 -----------------
.../src/clj/backtype/storm/daemon/logviewer.clj | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 9 +++---
storm-core/src/jvm/backtype/storm/Config.java | 14 ++++++++
12 files changed, 84 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
[11/11] storm git commit: add STORM-854 to CHANGELOG.md
Posted by ka...@apache.org.
add STORM-854 to CHANGELOG.md
* also add Rohan Agarwal to contributor list
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/528958c2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/528958c2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/528958c2
Branch: refs/heads/master
Commit: 528958c27f71da4538f9bca115e3b8eda55445de
Parents: d66a1b7
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Aug 25 12:04:50 2015 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Aug 25 12:04:50 2015 +0900
----------------------------------------------------------------------
CHANGELOG.md | 1 +
README.markdown | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/528958c2/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6be3d69..6a6dc8f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-854: [Storm-Kafka] KafkaSpout can set the topic name as the output streamid
* STORM-990: Refactored TimeCacheMap to extend RotatingMap
* STORM-829: Hadoop dependency confusion
* STORM-166: Nimbus HA
http://git-wip-us.apache.org/repos/asf/storm/blob/528958c2/README.markdown
----------------------------------------------------------------------
diff --git a/README.markdown b/README.markdown
index 85974ef..57ce0d5 100644
--- a/README.markdown
+++ b/README.markdown
@@ -217,6 +217,7 @@ under the License.
* Yvonne Ironberg ([@YvonneIronberg](https://github.com/YvonneIronberg))
* Boyang Jerry Peng ([@jerrypeng](https://github.com/jerrypeng))
* Li Wang ([@wangli1426](https://github.com/wangli1426))
+* Rohan Agarwal ([@rohanag12](https://github.com/rohanag12))
## Acknowledgements
[06/11] storm git commit: Added config info to storm-kafka README
Posted by ka...@apache.org.
Added config info to storm-kafka README
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f8f162c9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f8f162c9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f8f162c9
Branch: refs/heads/master
Commit: f8f162c95ac8deaa26a6fe97e39bfd234c542136
Parents: 4262a19
Author: rohan_agarwal <ro...@gmail.com>
Authored: Mon Aug 17 10:47:40 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Mon Aug 17 10:47:40 2015 +0530
----------------------------------------------------------------------
external/storm-kafka/README.md | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f8f162c9/external/storm-kafka/README.md
----------------------------------------------------------------------
diff --git a/external/storm-kafka/README.md b/external/storm-kafka/README.md
index 1e9cc12..3fe46e9 100644
--- a/external/storm-kafka/README.md
+++ b/external/storm-kafka/README.md
@@ -70,6 +70,9 @@ In addition to these parameters, SpoutConfig contains the following fields that
public long retryInitialDelayMs = 0;
public double retryDelayMultiplier = 1.0;
public long retryDelayMaxMs = 60 * 1000;
+
+ // if set to true, spout will set Kafka topic as the emitted Stream ID
+ public boolean topicAsStreamId = false;
```
Core KafkaSpout only accepts an instance of SpoutConfig.
[07/11] storm git commit: Merge remote-tracking branch
'upstream/master'
Posted by ka...@apache.org.
Merge remote-tracking branch 'upstream/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e2b08ebc
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e2b08ebc
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e2b08ebc
Branch: refs/heads/master
Commit: e2b08ebc3545e5ca50340cb81a41f90c9fc81511
Parents: f8f162c 6f37493
Author: rohan_agarwal <ro...@gmail.com>
Authored: Tue Aug 18 11:30:26 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Tue Aug 18 11:30:26 2015 +0530
----------------------------------------------------------------------
CHANGELOG.md | 1 +
TODO | 178 --------------
examples/storm-starter/pom.xml | 29 +++
.../starter/trident/TridentKafkaWordCount.java | 230 +++++++++++++++++++
external/storm-kafka/CHANGELOG.md | 13 --
pom.xml | 2 +-
6 files changed, 261 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
[04/11] storm git commit: Declared stream in declareOutputFields if
topicAsStreamId flag is true
Posted by ka...@apache.org.
Declared stream in declareOutputFields if topicAsStreamId flag is true
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/84a282c4
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/84a282c4
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/84a282c4
Branch: refs/heads/master
Commit: 84a282c48ac54ca2573464e4935cd652dd1f84a3
Parents: bfa3095
Author: rohan_agarwal <ro...@gmail.com>
Authored: Mon Aug 10 17:29:47 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Mon Aug 10 17:29:47 2015 +0530
----------------------------------------------------------------------
external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java | 6 +++++-
.../storm-kafka/src/jvm/storm/kafka/PartitionManager.java | 10 ++++++----
2 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/84a282c4/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java b/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java
index f3bc3ea..3260ad1 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java
@@ -182,7 +182,11 @@ public class KafkaSpout extends BaseRichSpout {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(_spoutConfig.scheme.getOutputFields());
+ if (_spoutConfig.topicAsStreamId) {
+ declarer.declareStream(_spoutConfig.topic, _spoutConfig.scheme.getOutputFields());
+ } else {
+ declarer.declare(_spoutConfig.scheme.getOutputFields());
+ }
}
private void commit() {
http://git-wip-us.apache.org/repos/asf/storm/blob/84a282c4/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 77087ec..052d525 100644
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@ -137,10 +137,12 @@ public class PartitionManager {
}
Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
if (tups != null) {
- for (List<Object> tup : tups) {
- if(_spoutConfig.topicAsStreamId) {
- collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
- } else {
+ if(_spoutConfig.topicAsStreamId) {
+ for (List<Object> tup : tups) {
+ collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
+ }
+ } else {
+ for (List<Object> tup : tups) {
collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
}
}
[05/11] storm git commit: Merge remote-tracking branch
'upstream/master'
Posted by ka...@apache.org.
Merge remote-tracking branch 'upstream/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4262a194
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4262a194
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4262a194
Branch: refs/heads/master
Commit: 4262a194a41ab871bac8d9380d9506913601d0fa
Parents: 84a282c 87fc298
Author: rohan_agarwal <ro...@gmail.com>
Authored: Mon Aug 17 10:38:02 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Mon Aug 17 10:38:02 2015 +0530
----------------------------------------------------------------------
CHANGELOG.md | 9 +
README.markdown | 3 +
conf/defaults.yaml | 1 +
dev-tools/travis/travis-script.sh | 2 +-
external/storm-hdfs/README.md | 9 +
external/storm-hdfs/pom.xml | 11 +
.../apache/storm/hdfs/trident/HdfsState.java | 392 +++++++++++++++----
.../trident/rotation/FileRotationPolicy.java | 14 +
.../rotation/FileSizeRotationPolicy.java | 13 +
.../hdfs/trident/rotation/NoRotationPolicy.java | 10 +
.../trident/rotation/TimedRotationPolicy.java | 31 +-
.../storm/hdfs/trident/HdfsStateTest.java | 206 ++++++++++
external/storm-hive/README.md | 17 +-
external/storm-hive/pom.xml | 24 ++
.../org/apache/storm/hive/bolt/HiveBolt.java | 39 +-
.../apache/storm/hive/common/HiveOptions.java | 11 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 12 +-
external/storm-jdbc/README.md | 6 +-
.../storm/jdbc/trident/state/JdbcState.java | 2 +-
.../UserPersistanceTridentTopology.java | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 4 +-
storm-core/src/jvm/backtype/storm/Config.java | 8 +-
.../jvm/backtype/storm/ConfigValidation.java | 13 +-
.../security/auth/SaslTransportPlugin.java | 6 +
.../kerberos/KerberosSaslTransportPlugin.java | 5 +-
.../security/auth/kerberos/NoOpTTrasport.java | 40 ++
.../src/jvm/backtype/storm/task/ShellBolt.java | 8 +-
.../test/clj/backtype/storm/config_test.clj | 4 +-
28 files changed, 778 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
[09/11] storm git commit: Merge remote-tracking branch
'upstream/master'
Posted by ka...@apache.org.
Merge remote-tracking branch 'upstream/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6491c4dd
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6491c4dd
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6491c4dd
Branch: refs/heads/master
Commit: 6491c4dd961b284fa6026f3f4df7accc1c374be2
Parents: 044ed00 cc27608
Author: rohan_agarwal <ro...@gmail.com>
Authored: Thu Aug 20 19:31:02 2015 +0530
Committer: rohan_agarwal <ro...@gmail.com>
Committed: Thu Aug 20 19:31:02 2015 +0530
----------------------------------------------------------------------
CHANGELOG.md | 1 +
README.markdown | 1 +
storm-core/src/clj/backtype/storm/ui/core.clj | 6 ++--
.../templates/topology-page-template.html | 12 ++++++-
storm-core/src/ui/public/topology.html | 35 +++++++++++++++++++-
5 files changed, 51 insertions(+), 4 deletions(-)
----------------------------------------------------------------------