You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/08/25 20:21:45 UTC
[1/9] git commit: STORM-132 sort supervisor by free slot in desending
order in EvenScheduler to schedule more evenly between supervisor
Repository: incubator-storm
Updated Branches:
refs/heads/master 5ddf308a8 -> 157576945
STORM-132 sort supervisor by free slot in desending order
in EvenScheduler to schedule more evenly between supervisor
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/6b275d95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/6b275d95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/6b275d95
Branch: refs/heads/master
Commit: 6b275d95fbdbc8374a215ecb2551f0fca3438d81
Parents: 9fe14b4
Author: Kang Xiao <kx...@gmail.com>
Authored: Tue Feb 18 23:23:50 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Tue Feb 18 23:23:50 2014 +0800
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/6b275d95/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
index 28b9202..828606d 100644
--- a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
+++ b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
@@ -22,7 +22,12 @@
:implements [backtype.storm.scheduler.IScheduler]))
(defn sort-slots [all-slots]
- (let [split-up (vals (group-by first all-slots))]
+ (let [split-up
+ (map second
+ (reverse
+ (sort
+ (for [[host ports] (group-by first all-slots)]
+ [(count ports) ports]))))]
(apply interleave-all split-up)
))
[8/9] git commit: Merge branch 'STORM-132' of
https://github.com/xiaokang/incubator-storm into STORM-132
Posted by bo...@apache.org.
Merge branch 'STORM-132' of https://github.com/xiaokang/incubator-storm into STORM-132
STORM-132: sort supervisor by free slot in desending order
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/6393d31b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/6393d31b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/6393d31b
Branch: refs/heads/master
Commit: 6393d31b0be99f93213405543c8af8d6e8cd6f63
Parents: 5ddf308 a2484f3
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 25 13:15:57 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 25 13:15:57 2014 -0500
----------------------------------------------------------------------
.../backtype/storm/scheduler/EvenScheduler.clj | 2 +-
.../test/clj/backtype/storm/scheduler_test.clj | 21 ++++++++++++++++++++
2 files changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[5/9] git commit: modify EvenScheduler.clj sort-slots and add a
test-sort-slots testcase
Posted by bo...@apache.org.
modify EvenScheduler.clj sort-slots and add a test-sort-slots testcase
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/3a20181b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/3a20181b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/3a20181b
Branch: refs/heads/master
Commit: 3a20181b034c18adb0dc9c881059ca884603eba5
Parents: fad1f65
Author: Kang Xiao <kx...@gmail.com>
Authored: Fri Aug 8 02:20:53 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Fri Aug 8 02:20:53 2014 +0800
----------------------------------------------------------------------
.../backtype/storm/scheduler/EvenScheduler.clj | 2 +-
.../test/clj/backtype/storm/scheduler_test.clj | 25 ++++++++++++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/3a20181b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
index 3bedca3..e377c13 100644
--- a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
+++ b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
@@ -22,7 +22,7 @@
:implements [backtype.storm.scheduler.IScheduler]))
(defn sort-slots [all-slots]
- (let [split-up (sort-by count (vals (group-by first all-slots)))]
+ (let [split-up (sort #(> (count %1) (count %2)) (vals (group-by first all-slots)))]
(apply interleave-all split-up)
))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/3a20181b/storm-core/test/clj/backtype/storm/scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/scheduler_test.clj b/storm-core/test/clj/backtype/storm/scheduler_test.clj
index f8cc1ab..a11e6b1 100644
--- a/storm-core/test/clj/backtype/storm/scheduler_test.clj
+++ b/storm-core/test/clj/backtype/storm/scheduler_test.clj
@@ -16,6 +16,7 @@
(ns backtype.storm.scheduler-test
(:use [clojure test])
(:use [backtype.storm bootstrap config testing])
+ (:use [backtype.storm.scheduler EvenScheduler])
(:require [backtype.storm.daemon [nimbus :as nimbus]])
(:import [backtype.storm.generated StormTopology])
(:import [backtype.storm.scheduler Cluster SupervisorDetails WorkerSlot ExecutorDetails
@@ -259,3 +260,27 @@
(is (= false (.isSlotOccupied cluster (WorkerSlot. "supervisor1" (int 3)))))
(is (= false (.isSlotOccupied cluster (WorkerSlot. "supervisor1" (int 5)))))
))
+
+(deftest test-sort-slots
+ ;; test supervisor2 has more free slots
+ (is (= '(["supervisor2" 6700] ["supervisor1" 6700]
+ ["supervisor2" 6701] ["supervisor1" 6701]
+ ["supervisor2" 6702])
+ (sort-slots [["supervisor1" 6700] ["supervisor1" 6701]
+ ["supervisor2" 6700] ["supervisor2" 6701] ["supervisor2" 6702]
+ ])))
+ ;; test supervisor3 has more free slots
+ (is (= '(["supervisor3" 6700] ["supervisor2" 6700] ["supervisor1" 6700]
+ ["supervisor3" 6701] ["supervisor2" 6701] ["supervisor1" 6701]
+ ["supervisor3" 6702] ["supervisor2" 6702]
+ ["supervisor3" 6703])
+ (sort-slots [["supervisor1" 6700] ["supervisor1" 6701]
+ ["supervisor2" 6700] ["supervisor2" 6701] ["supervisor2" 6702]
+ ["supervisor3" 6700] ["supervisor3" 6701] ["supervisor3" 6702] ["supervisor3" 6703]
+ ])))
+ ;; test supervisor1 and supervisor2 has the same free slot
+ (is (= '(["supervisor1" 6700] ["supervisor2" 6700]
+ ["supervisor1" 6701] ["supervisor2" 6701])
+ (sort-slots [["supervisor1" 6700] ["supervisor1" 6701] ["supervisor2" 6700] ["supervisor2" 6701]])))
+ )
+
[4/9] git commit: Merge branch 'master' of
https://github.com/apache/incubator-storm into STORM-132
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/apache/incubator-storm into STORM-132
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/fad1f653
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/fad1f653
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/fad1f653
Branch: refs/heads/master
Commit: fad1f65369925771265d0ba322d41823683a932d
Parents: a41a0ac 2ebdbfa
Author: Kang Xiao <kx...@gmail.com>
Authored: Fri Aug 8 02:18:58 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Fri Aug 8 02:18:58 2014 +0800
----------------------------------------------------------------------
CHANGELOG.md | 1 +
storm-core/src/clj/backtype/storm/util.clj | 2 +-
storm-core/src/jvm/backtype/storm/task/ShellBolt.java | 3 +++
3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[9/9] git commit: Added STORM-132 to Changelog
Posted by bo...@apache.org.
Added STORM-132 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/15757694
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/15757694
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/15757694
Branch: refs/heads/master
Commit: 15757694512319540d385257af3af6d19c8b1aaa
Parents: 6393d31
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 25 13:16:47 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 25 13:16:47 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/15757694/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8f769d8..2962c47 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -58,6 +58,7 @@
* STORM-361: Add JSON-P support to Storm UI API
* STORM-373: Provide Additional String substitutions for *.worker.childopts
* STORM-274: Add support for command remoteconfvalue in storm.cmd
+ * STORM-132: sort supervisor by free slot in desending order
## 0.9.2-incubating
* STORM-66: send taskid on initial handshake
[3/9] git commit: Merge branch 'master' of
https://github.com/apache/incubator-storm into STORM-132
Posted by bo...@apache.org.
Merge branch 'master' of https://github.com/apache/incubator-storm into STORM-132
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/a41a0ac4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/a41a0ac4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/a41a0ac4
Branch: refs/heads/master
Commit: a41a0ac4e7240338ec4825dc23b10e5e18db1efd
Parents: f7502f4 b1c5e7e
Author: Kang Xiao <kx...@gmail.com>
Authored: Thu Aug 7 23:52:31 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Thu Aug 7 23:52:31 2014 +0800
----------------------------------------------------------------------
.gitignore | 4 +-
BYLAWS.md | 96 ++
CHANGELOG.md | 107 ++
DEVELOPER.md | 343 +++++
LICENSE | 154 +-
NOTICE | 9 +-
README.markdown | 59 +-
SECURITY.md | 74 +
STORM-UI-REST-API.md | 565 +++++++
bin/storm | 134 +-
bin/storm-config.cmd | 27 +-
bin/storm.cmd | 59 +-
conf/defaults.yaml | 16 +-
conf/storm.yaml.example | 2 +-
conf/storm_env.ini | 30 +
dev-tools/github/__init__.py | 109 ++
dev-tools/jira-github-join.py | 80 +
dev-tools/jira/__init__.py | 232 +++
examples/storm-starter/README.markdown | 155 ++
.../multilang/resources/splitsentence.py | 24 +
.../multilang/resources/splitsentence.rb | 26 +
.../storm-starter/multilang/resources/storm.py | 221 +++
.../storm-starter/multilang/resources/storm.rb | 200 +++
examples/storm-starter/pom.xml | 165 ++
.../src/clj/storm/starter/clj/word_count.clj | 95 ++
.../jvm/storm/starter/BasicDRPCTopology.java | 78 +
.../jvm/storm/starter/ExclamationTopology.java | 87 ++
.../src/jvm/storm/starter/ManualDRPC.java | 68 +
.../jvm/storm/starter/PrintSampleStream.java | 58 +
.../src/jvm/storm/starter/ReachTopology.java | 196 +++
.../src/jvm/storm/starter/RollingTopWords.java | 132 ++
.../jvm/storm/starter/SingleJoinExample.java | 64 +
.../storm/starter/TransactionalGlobalCount.java | 173 +++
.../jvm/storm/starter/TransactionalWords.java | 246 +++
.../jvm/storm/starter/WordCountTopology.java | 107 ++
.../storm/starter/bolt/AbstractRankerBolt.java | 110 ++
.../starter/bolt/IntermediateRankingsBolt.java | 58 +
.../src/jvm/storm/starter/bolt/PrinterBolt.java | 37 +
.../storm/starter/bolt/RollingCountBolt.java | 142 ++
.../jvm/storm/starter/bolt/SingleJoinBolt.java | 114 ++
.../storm/starter/bolt/TotalRankingsBolt.java | 59 +
.../starter/spout/RandomSentenceSpout.java | 64 +
.../storm/starter/spout/TwitterSampleSpout.java | 164 ++
.../tools/NthLastModifiedTimeTracker.java | 70 +
.../src/jvm/storm/starter/tools/Rankable.java | 32 +
.../starter/tools/RankableObjectWithFields.java | 148 ++
.../src/jvm/storm/starter/tools/Rankings.java | 156 ++
.../starter/tools/SlidingWindowCounter.java | 119 ++
.../storm/starter/tools/SlotBasedCounter.java | 118 ++
.../jvm/storm/starter/trident/TridentReach.java | 156 ++
.../storm/starter/trident/TridentWordCount.java | 85 +
.../src/jvm/storm/starter/util/StormRunner.java | 47 +
.../jvm/storm/starter/util/TupleHelpers.java | 33 +
.../bolt/IntermediateRankingsBoltTest.java | 146 ++
.../starter/bolt/RollingCountBoltTest.java | 113 ++
.../starter/bolt/TotalRankingsBoltTest.java | 147 ++
.../storm/starter/tools/MockTupleHelpers.java | 40 +
.../tools/NthLastModifiedTimeTrackerTest.java | 125 ++
.../tools/RankableObjectWithFieldsTest.java | 252 +++
.../jvm/storm/starter/tools/RankingsTest.java | 368 +++++
.../starter/tools/SlidingWindowCounterTest.java | 106 ++
.../starter/tools/SlotBasedCounterTest.java | 181 +++
external/storm-kafka/CHANGELOG.md | 13 +
external/storm-kafka/README.md | 49 +
external/storm-kafka/pom.xml | 119 ++
.../storm-kafka/src/jvm/storm/kafka/Broker.java | 86 ++
.../src/jvm/storm/kafka/BrokerHosts.java | 25 +
.../jvm/storm/kafka/DynamicBrokersReader.java | 147 ++
.../kafka/DynamicPartitionConnections.java | 94 ++
.../jvm/storm/kafka/FailedFetchException.java | 29 +
.../src/jvm/storm/kafka/KafkaConfig.java | 52 +
.../src/jvm/storm/kafka/KafkaError.java | 43 +
.../src/jvm/storm/kafka/KafkaSpout.java | 195 +++
.../src/jvm/storm/kafka/KafkaUtils.java | 243 +++
.../src/jvm/storm/kafka/KeyValueScheme.java | 28 +
.../kafka/KeyValueSchemeAsMultiScheme.java | 37 +
.../src/jvm/storm/kafka/Partition.java | 69 +
.../jvm/storm/kafka/PartitionCoordinator.java | 28 +
.../src/jvm/storm/kafka/PartitionManager.java | 265 ++++
.../src/jvm/storm/kafka/SpoutConfig.java | 36 +
.../src/jvm/storm/kafka/StaticCoordinator.java | 51 +
.../src/jvm/storm/kafka/StaticHosts.java | 38 +
.../storm/kafka/StaticPartitionConnections.java | 52 +
.../jvm/storm/kafka/StringKeyValueScheme.java | 37 +
.../src/jvm/storm/kafka/StringScheme.java | 46 +
.../src/jvm/storm/kafka/ZkCoordinator.java | 113 ++
.../src/jvm/storm/kafka/ZkHosts.java | 36 +
.../src/jvm/storm/kafka/ZkState.java | 116 ++
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 89 ++
.../jvm/storm/kafka/trident/Coordinator.java | 50 +
.../storm/kafka/trident/DefaultCoordinator.java | 31 +
.../trident/GlobalPartitionInformation.java | 98 ++
.../storm/kafka/trident/IBatchCoordinator.java | 26 +
.../jvm/storm/kafka/trident/IBrokerReader.java | 25 +
.../src/jvm/storm/kafka/trident/MaxMetric.java | 40 +
.../kafka/trident/OpaqueTridentKafkaSpout.java | 59 +
.../storm/kafka/trident/StaticBrokerReader.java | 36 +
.../trident/TransactionalTridentKafkaSpout.java | 58 +
.../storm/kafka/trident/TridentKafkaConfig.java | 37 +
.../kafka/trident/TridentKafkaEmitter.java | 268 ++++
.../jvm/storm/kafka/trident/ZkBrokerReader.java | 70 +
.../storm/kafka/DynamicBrokersReaderTest.java | 172 +++
.../src/test/storm/kafka/KafkaErrorTest.java | 56 +
.../src/test/storm/kafka/KafkaTestBroker.java | 92 ++
.../src/test/storm/kafka/KafkaUtilsTest.java | 242 +++
.../storm/kafka/StringKeyValueSchemeTest.java | 55 +
.../src/test/storm/kafka/TestUtils.java | 37 +
.../src/test/storm/kafka/ZkCoordinatorTest.java | 147 ++
.../test/storm/kafka/bolt/KafkaBoltTest.java | 188 +++
logback/cluster.xml | 2 +-
pom.xml | 121 +-
.../maven-shade-clojure-transformer/pom.xml | 2 +-
storm-core/pom.xml | 60 +-
.../src/clj/backtype/storm/LocalCluster.clj | 76 +-
storm-core/src/clj/backtype/storm/LocalDRPC.clj | 5 +-
storm-core/src/clj/backtype/storm/bootstrap.clj | 9 +-
storm-core/src/clj/backtype/storm/clojure.clj | 10 +-
storm-core/src/clj/backtype/storm/cluster.clj | 460 +++---
.../src/clj/backtype/storm/command/monitor.clj | 37 +
.../clj/backtype/storm/command/rebalance.clj | 7 +-
storm-core/src/clj/backtype/storm/config.clj | 145 +-
.../src/clj/backtype/storm/daemon/common.clj | 8 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 139 +-
.../src/clj/backtype/storm/daemon/executor.clj | 46 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 182 ++-
.../src/clj/backtype/storm/daemon/nimbus.clj | 19 +-
.../clj/backtype/storm/daemon/supervisor.clj | 157 +-
.../src/clj/backtype/storm/daemon/worker.clj | 101 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 89 +-
storm-core/src/clj/backtype/storm/event.clj | 49 +-
storm-core/src/clj/backtype/storm/log.clj | 22 +-
.../src/clj/backtype/storm/messaging/loader.clj | 81 +-
.../src/clj/backtype/storm/messaging/local.clj | 20 +-
.../clj/backtype/storm/process_simulator.clj | 27 +-
storm-core/src/clj/backtype/storm/stats.clj | 289 ++--
storm-core/src/clj/backtype/storm/testing.clj | 545 +++----
storm-core/src/clj/backtype/storm/testing4j.clj | 58 +-
storm-core/src/clj/backtype/storm/thrift.clj | 255 +--
storm-core/src/clj/backtype/storm/timer.clj | 109 +-
storm-core/src/clj/backtype/storm/tuple.clj | 7 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 1453 ++++++++----------
.../src/clj/backtype/storm/ui/helpers.clj | 13 +-
storm-core/src/clj/backtype/storm/util.clj | 780 +++++-----
storm-core/src/clj/backtype/storm/zookeeper.clj | 191 ++-
.../src/dev/resources/tester_bolt_metrics.py | 35 +
.../src/dev/resources/tester_spout_metrics.py | 51 +
storm-core/src/jvm/backtype/storm/Config.java | 182 ++-
.../jvm/backtype/storm/ConfigValidation.java | 87 ++
.../src/jvm/backtype/storm/StormSubmitter.java | 176 ++-
.../jvm/backtype/storm/generated/ErrorInfo.java | 188 ++-
.../backtype/storm/messaging/IConnection.java | 15 +-
.../backtype/storm/messaging/netty/Client.java | 338 ++--
.../backtype/storm/messaging/netty/Context.java | 57 +-
.../storm/messaging/netty/ControlMessage.java | 6 +-
.../storm/messaging/netty/MessageBatch.java | 52 +-
.../storm/messaging/netty/MessageDecoder.java | 108 +-
.../netty/NettyRenameThreadFactory.java | 52 +
.../backtype/storm/messaging/netty/Server.java | 147 +-
.../netty/StormClientErrorHandler.java | 41 +
.../messaging/netty/StormClientHandler.java | 121 --
.../netty/StormClientPipelineFactory.java | 2 +-
.../messaging/netty/StormServerHandler.java | 40 +-
.../metric/api/rpc/AssignableShellMetric.java | 30 +
.../metric/api/rpc/CombinedShellMetric.java | 31 +
.../storm/metric/api/rpc/CountShellMetric.java | 38 +
.../storm/metric/api/rpc/IShellMetric.java | 31 +
.../metric/api/rpc/ReducedShellMetric.java | 32 +
.../jvm/backtype/storm/multilang/BoltMsg.java | 80 +
.../backtype/storm/multilang/ISerializer.java | 82 +
.../storm/multilang/JsonSerializer.java | 194 +++
.../storm/multilang/NoOutputException.java | 40 +
.../jvm/backtype/storm/multilang/ShellMsg.java | 168 ++
.../jvm/backtype/storm/multilang/SpoutMsg.java | 51 +
.../src/jvm/backtype/storm/spout/ISpout.java | 2 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 155 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 180 ++-
.../backtype/storm/task/TopologyContext.java | 50 +-
.../storm/testing/PythonShellMetricsBolt.java | 32 +
.../storm/testing/PythonShellMetricsSpout.java | 35 +
.../storm/testing/TestEventLogSpout.java | 139 ++
.../storm/testing/TestEventOrderCheckBolt.java | 76 +
.../transactional/state/TransactionalState.java | 2 +-
.../storm/ui/InvalidRequestException.java | 20 +
.../backtype/storm/utils/DisruptorQueue.java | 121 +-
.../src/jvm/backtype/storm/utils/Monitor.java | 249 +++
.../jvm/backtype/storm/utils/ShellProcess.java | 195 ++-
.../src/jvm/backtype/storm/utils/Time.java | 18 +-
.../backtype/storm/utils/TransferDrainer.java | 113 ++
.../src/jvm/backtype/storm/utils/Utils.java | 120 +-
storm-core/src/jvm/storm/trident/Stream.java | 5 +-
.../trident/spout/RichSpoutBatchTriggerer.java | 1 +
.../jvm/storm/trident/state/map/OpaqueMap.java | 6 +-
.../trident/state/map/RemovableMapState.java | 25 +
.../storm/trident/testing/MemoryMapState.java | 27 +-
.../topology/MasterBatchCoordinator.java | 4 +
.../topology/state/TransactionalState.java | 2 +-
.../jvm/storm/trident/util/TridentUtils.java | 14 +-
storm-core/src/multilang/py/storm.py | 30 +-
storm-core/src/multilang/rb/storm.rb | 24 +-
storm-core/src/py/storm/ttypes.py | 28 +-
storm-core/src/storm.thrift | 2 +
storm-core/src/ui/public/component.html | 125 ++
storm-core/src/ui/public/css/style.css | 1 +
storm-core/src/ui/public/index.html | 89 ++
storm-core/src/ui/public/js/arbor-graphics.js | 51 +
storm-core/src/ui/public/js/arbor-tween.js | 86 ++
storm-core/src/ui/public/js/arbor.js | 67 +
storm-core/src/ui/public/js/jquery.mustache.js | 592 +++++++
.../src/ui/public/js/jquery.tablesorter.min.js | 9 +-
storm-core/src/ui/public/js/moment.min.js | 6 +
storm-core/src/ui/public/js/purl.js | 267 ++++
storm-core/src/ui/public/js/script.js | 60 +-
storm-core/src/ui/public/js/visualization.js | 421 +++++
.../templates/component-page-template.html | 173 +++
.../public/templates/index-page-template.html | 78 +
.../public/templates/json-error-template.html | 20 +
.../templates/topology-page-template.html | 152 ++
storm-core/src/ui/public/topology.html | 119 ++
.../test/clj/backtype/storm/cluster_test.clj | 11 +-
.../test/clj/backtype/storm/config_test.clj | 61 +-
.../test/clj/backtype/storm/drpc_test.clj | 27 +-
.../clj/backtype/storm/integration_test.clj | 9 +-
.../storm/messaging/netty_unit_test.clj | 46 +-
.../test/clj/backtype/storm/messaging_test.clj | 35 +-
.../test/clj/backtype/storm/metrics_test.clj | 218 ++-
.../test/clj/backtype/storm/multilang_test.clj | 6 +-
.../test/clj/backtype/storm/nimbus_test.clj | 16 +-
.../test/clj/backtype/storm/supervisor_test.clj | 184 ++-
.../test/clj/backtype/storm/utils_test.clj | 13 +-
.../test/clj/storm/trident/integration_test.clj | 2 +-
.../test/clj/storm/trident/state_test.clj | 33 +-
.../storm/utils/DisruptorQueueTest.java | 152 ++
storm-dist/binary/LICENSE | 154 +-
storm-dist/binary/NOTICE | 11 +-
storm-dist/binary/pom.xml | 42 +-
storm-dist/binary/src/main/assembly/binary.xml | 41 +
storm-dist/source/pom.xml | 40 +-
237 files changed, 20900 insertions(+), 3549 deletions(-)
----------------------------------------------------------------------
[2/9] git commit: STORM-132 simplify code using sort-by function
Posted by bo...@apache.org.
STORM-132 simplify code using sort-by function
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/f7502f41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/f7502f41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/f7502f41
Branch: refs/heads/master
Commit: f7502f41a7bdfc67a432714efa27b0c952b7bb0e
Parents: 6b275d9
Author: Kang Xiao <kx...@gmail.com>
Authored: Sun Jul 20 00:19:21 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Sun Jul 20 00:19:21 2014 +0800
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/f7502f41/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
index 828606d..3bedca3 100644
--- a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
+++ b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
@@ -22,12 +22,7 @@
:implements [backtype.storm.scheduler.IScheduler]))
(defn sort-slots [all-slots]
- (let [split-up
- (map second
- (reverse
- (sort
- (for [[host ports] (group-by first all-slots)]
- [(count ports) ports]))))]
+ (let [split-up (sort-by count (vals (group-by first all-slots)))]
(apply interleave-all split-up)
))
[7/9] git commit: remove unnessery test and change port numbers order
Posted by bo...@apache.org.
remove unnessery test and change port numbers order
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/a2484f3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/a2484f3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/a2484f3e
Branch: refs/heads/master
Commit: a2484f3e7659841d986439b73c79116f8ba0d70b
Parents: d6a7114
Author: Kang Xiao <kx...@gmail.com>
Authored: Sat Aug 16 00:12:53 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Sat Aug 16 00:12:53 2014 +0800
----------------------------------------------------------------------
storm-core/test/clj/backtype/storm/scheduler_test.clj | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/a2484f3e/storm-core/test/clj/backtype/storm/scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/scheduler_test.clj b/storm-core/test/clj/backtype/storm/scheduler_test.clj
index 9744ac6..856ddeb 100644
--- a/storm-core/test/clj/backtype/storm/scheduler_test.clj
+++ b/storm-core/test/clj/backtype/storm/scheduler_test.clj
@@ -262,10 +262,6 @@
))
(deftest test-sort-slots
- ;; test supervisor1 and supervisor2 has the same free slot
- (is (= '(["supervisor1" 6700] ["supervisor2" 6700]
- ["supervisor1" 6701] ["supervisor2" 6701])
- (sort-slots [["supervisor1" 6700] ["supervisor1" 6701] ["supervisor2" 6700] ["supervisor2" 6701]])))
;; test supervisor2 has more free slots
(is (= '(["supervisor2" 6700] ["supervisor1" 6700]
["supervisor2" 6701] ["supervisor1" 6701]
@@ -275,12 +271,12 @@
])))
;; test supervisor3 has more free slots
(is (= '(["supervisor3" 6700] ["supervisor2" 6700] ["supervisor1" 6700]
- ["supervisor3" 6701] ["supervisor2" 6701] ["supervisor1" 6701]
+ ["supervisor3" 6703] ["supervisor2" 6701] ["supervisor1" 6701]
["supervisor3" 6702] ["supervisor2" 6702]
- ["supervisor3" 6703])
+ ["supervisor3" 6701])
(sort-slots [["supervisor1" 6700] ["supervisor1" 6701]
["supervisor2" 6700] ["supervisor2" 6701] ["supervisor2" 6702]
- ["supervisor3" 6700] ["supervisor3" 6701] ["supervisor3" 6702] ["supervisor3" 6703]
+ ["supervisor3" 6700] ["supervisor3" 6703] ["supervisor3" 6702] ["supervisor3" 6701]
])))
)
[6/9] git commit: 1. fix test case passed on original code;
2. use count-by
Posted by bo...@apache.org.
1. fix test case passed on original code; 2. use count-by
Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/d6a71142
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/d6a71142
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/d6a71142
Branch: refs/heads/master
Commit: d6a71142eec6f669f335312f418134960143fdbc
Parents: 3a20181
Author: Kang Xiao <kx...@gmail.com>
Authored: Thu Aug 14 00:34:06 2014 +0800
Committer: Kang Xiao <kx...@gmail.com>
Committed: Thu Aug 14 00:34:06 2014 +0800
----------------------------------------------------------------------
.../src/clj/backtype/storm/scheduler/EvenScheduler.clj | 2 +-
storm-core/test/clj/backtype/storm/scheduler_test.clj | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/d6a71142/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
index e377c13..25ba03b 100644
--- a/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
+++ b/storm-core/src/clj/backtype/storm/scheduler/EvenScheduler.clj
@@ -22,7 +22,7 @@
:implements [backtype.storm.scheduler.IScheduler]))
(defn sort-slots [all-slots]
- (let [split-up (sort #(> (count %1) (count %2)) (vals (group-by first all-slots)))]
+ (let [split-up (sort-by count > (vals (group-by first all-slots)))]
(apply interleave-all split-up)
))
http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/d6a71142/storm-core/test/clj/backtype/storm/scheduler_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/backtype/storm/scheduler_test.clj b/storm-core/test/clj/backtype/storm/scheduler_test.clj
index a11e6b1..9744ac6 100644
--- a/storm-core/test/clj/backtype/storm/scheduler_test.clj
+++ b/storm-core/test/clj/backtype/storm/scheduler_test.clj
@@ -262,6 +262,10 @@
))
(deftest test-sort-slots
+ ;; test supervisor1 and supervisor2 has the same free slot
+ (is (= '(["supervisor1" 6700] ["supervisor2" 6700]
+ ["supervisor1" 6701] ["supervisor2" 6701])
+ (sort-slots [["supervisor1" 6700] ["supervisor1" 6701] ["supervisor2" 6700] ["supervisor2" 6701]])))
;; test supervisor2 has more free slots
(is (= '(["supervisor2" 6700] ["supervisor1" 6700]
["supervisor2" 6701] ["supervisor1" 6701]
@@ -278,9 +282,5 @@
["supervisor2" 6700] ["supervisor2" 6701] ["supervisor2" 6702]
["supervisor3" 6700] ["supervisor3" 6701] ["supervisor3" 6702] ["supervisor3" 6703]
])))
- ;; test supervisor1 and supervisor2 has the same free slot
- (is (= '(["supervisor1" 6700] ["supervisor2" 6700]
- ["supervisor1" 6701] ["supervisor2" 6701])
- (sort-slots [["supervisor1" 6700] ["supervisor1" 6701] ["supervisor2" 6700] ["supervisor2" 6701]])))
)