You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/09/21 19:40:54 UTC
[09/24] storm git commit: Merging upstream/master
Merging upstream/master
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7940d0c5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7940d0c5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7940d0c5
Branch: refs/heads/master
Commit: 7940d0c57c07c6b168617a0daaf182895cec8028
Parents: 306ec57 544e55c
Author: Arun Mahadevan <ai...@hortonworks.com>
Authored: Mon Aug 10 15:08:10 2015 +0530
Committer: Arun Mahadevan <ai...@hortonworks.com>
Committed: Mon Aug 10 15:08:10 2015 +0530
----------------------------------------------------------------------
CHANGELOG.md | 16 ++-
DEVELOPER.md | 7 +-
README.markdown | 1 +
external/storm-elasticsearch/README.md | 72 ++++++++++
external/storm-elasticsearch/pom.xml | 95 +++++++++++++
.../elasticsearch/bolt/AbstractEsBolt.java | 81 +++++++++++
.../storm/elasticsearch/bolt/EsIndexBolt.java | 68 ++++++++++
.../elasticsearch/bolt/EsPercolateBolt.java | 78 +++++++++++
.../storm/elasticsearch/common/EsConfig.java | 54 ++++++++
.../storm/elasticsearch/trident/EsState.java | 117 ++++++++++++++++
.../elasticsearch/trident/EsStateFactory.java | 50 +++++++
.../storm/elasticsearch/trident/EsUpdater.java | 31 +++++
.../elasticsearch/bolt/AbstractEsBoltTest.java | 81 +++++++++++
.../elasticsearch/bolt/EsIndexBoltTest.java | 68 ++++++++++
.../elasticsearch/bolt/EsIndexTopology.java | 120 +++++++++++++++++
.../elasticsearch/bolt/EsPercolateBoltTest.java | 65 +++++++++
.../storm/elasticsearch/common/EsConstants.java | 22 +++
.../storm/elasticsearch/common/EsTestUtil.java | 70 ++++++++++
.../trident/TridentEsTopology.java | 135 +++++++++++++++++++
external/storm-hive/README.md | 1 +
.../org/apache/storm/hive/bolt/HiveBolt.java | 44 ++++--
.../apache/storm/hive/common/HiveWriter.java | 5 +-
.../apache/storm/hive/bolt/TestHiveBolt.java | 100 ++++++++++++--
.../src/jvm/storm/kafka/ZkCoordinator.java | 2 +-
external/storm-redis/README.md | 1 +
log4j2/cluster.xml | 6 +-
log4j2/worker.xml | 6 +-
pom.xml | 1 +
storm-core/src/clj/backtype/storm/cluster.clj | 2 +-
.../src/clj/backtype/storm/daemon/task.clj | 1 -
storm-core/src/clj/backtype/storm/tuple.clj | 6 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 26 ++--
storm-core/src/jvm/backtype/storm/Config.java | 2 +-
.../jvm/backtype/storm/ConfigValidation.java | 15 +--
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 6 +-
.../storm/grouping/PartialKeyGrouping.java | 27 +++-
.../storm/testing/TestWordBytesCounter.java | 27 ++++
.../backtype/storm/testing/TestWordCounter.java | 6 +-
.../src/jvm/storm/trident/TridentTopology.java | 13 +-
.../test/clj/backtype/storm/config_test.clj | 19 ++-
.../test/clj/backtype/storm/grouping_test.clj | 43 +++---
.../test/clj/backtype/storm/nimbus_test.clj | 30 +++++
storm-dist/binary/src/main/assembly/binary.xml | 14 ++
43 files changed, 1532 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7940d0c5/storm-core/src/clj/backtype/storm/cluster.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7940d0c5/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7940d0c5/storm-core/test/clj/backtype/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/nimbus_test.clj
index a8a3acd,00fb6d6..10d135c
--- a/storm-core/test/clj/backtype/storm/nimbus_test.clj
+++ b/storm-core/test/clj/backtype/storm/nimbus_test.clj
@@@ -1182,10 -1181,31 +1182,40 @@@
(submit-local-topology-with-opts nimbus "test" bad-config topology
(SubmitOptions.))))))))
+ (deftest test-stateless-with-scheduled-topology-to-be-killed
+ ; tests regression of STORM-856
+ (with-inprocess-zookeeper zk-port
+ (with-local-tmp [nimbus-dir]
+ (letlocals
+ (bind conf (merge (read-storm-config)
+ {STORM-ZOOKEEPER-SERVERS ["localhost"]
+ STORM-CLUSTER-MODE "local"
+ STORM-ZOOKEEPER-PORT zk-port
+ STORM-LOCAL-DIR nimbus-dir}))
+ (bind cluster-state (cluster/mk-storm-cluster-state conf))
+ (bind nimbus (nimbus/service-handler conf (nimbus/standalone-nimbus)))
+ (bind topology (thrift/mk-topology
+ {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 3)}
+ {}))
+ (submit-local-topology nimbus "t1" {TOPOLOGY-MESSAGE-TIMEOUT-SECS 30} topology)
+ ; make transition for topology t1 to be killed -> nimbus applies this event to cluster state
+ (.killTopology nimbus "t1")
+ ; shutdown nimbus immediately to achieve nimbus doesn't handle event right now
+ (.shutdown nimbus)
+
+ ; in startup of nimbus it reads cluster state and take proper actions
+ ; in this case nimbus registers topology transition event to scheduler again
+ ; before applying STORM-856 nimbus was killed with NPE
+ (bind nimbus (nimbus/service-handler conf (nimbus/standalone-nimbus)))
+ (.shutdown nimbus)
+ (.disconnect cluster-state)
- ))))
++ ))))
++
+(deftest test-debug-on
+ (with-local-cluster [cluster]
+ (let [nimbus (:nimbus cluster)]
+ (stubbing [converter/clojurify-storm-base (backtype.storm.daemon.common.StormBase. "storm-name" 100
+ {:type :active} 1 nil nil nil nil nil)
+ cluster/maybe-deserialize nil]
+ (.debug nimbus "test" true)))))
++