You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/11/11 22:24:11 UTC
[06/10] storm git commit: Merge remote-tracking branch
'upstream/master' into STORM-1155
Merge remote-tracking branch 'upstream/master' into STORM-1155
Conflicts:
storm-core/src/clj/backtype/storm/daemon/supervisor.clj
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b3521c57
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b3521c57
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b3521c57
Branch: refs/heads/master
Commit: b3521c577bef6654dcf90269c4639064c3ae60eb
Parents: be9799e 316b23e
Author: Thomas Graves <tg...@decadefade.corp.ne1.yahoo.com>
Authored: Thu Nov 5 19:08:45 2015 +0000
Committer: Thomas Graves <tg...@decadefade.corp.ne1.yahoo.com>
Committed: Thu Nov 5 19:08:45 2015 +0000
----------------------------------------------------------------------
CHANGELOG.md | 5 +
DISCLAIMER | 10 -
STORM-UI-REST-API.md | 735 -------
bin/flight.bash | 139 ++
bin/storm.py | 14 +-
conf/defaults.yaml | 3 +
docs/DYNAMIC_LOG_LEVEL_SETTINGS.md | 41 -
docs/documentation/Documentation.md | 3 +
.../Message-passing-implementation.md | 34 +-
.../documentation/dynamic-log-level-settings.md | 41 +
docs/documentation/dynamic-worker-profiling.md | 29 +
.../images/dynamic_log_level_settings_1.png | Bin 0 -> 93689 bytes
.../images/dynamic_log_level_settings_2.png | Bin 0 -> 78785 bytes
.../images/dynamic_profiling_debugging_1.png | Bin 0 -> 93635 bytes
.../images/dynamic_profiling_debugging_2.png | Bin 0 -> 138120 bytes
.../images/dynamic_profiling_debugging_3.png | Bin 0 -> 96974 bytes
docs/documentation/ui-rest-api.md | 930 ++++++++
docs/images/dynamic_log_level_settings_1.png | Bin 93689 -> 0 bytes
docs/images/dynamic_log_level_settings_2.png | Bin 78785 -> 0 bytes
external/storm-kafka/README.md | 11 +
.../jvm/storm/kafka/DynamicBrokersReader.java | 97 +-
.../kafka/DynamicPartitionConnections.java | 20 +-
.../src/jvm/storm/kafka/KafkaSpout.java | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 85 +-
.../src/jvm/storm/kafka/Partition.java | 26 +-
.../src/jvm/storm/kafka/PartitionManager.java | 18 +-
.../src/jvm/storm/kafka/StaticCoordinator.java | 11 +-
.../storm/kafka/StringMultiSchemeWithTopic.java | 57 +
.../src/jvm/storm/kafka/ZkCoordinator.java | 2 +-
.../jvm/storm/kafka/trident/Coordinator.java | 7 +-
.../trident/GlobalPartitionInformation.java | 26 +-
.../jvm/storm/kafka/trident/IBrokerReader.java | 7 +-
.../kafka/trident/OpaqueTridentKafkaSpout.java | 5 +-
.../storm/kafka/trident/StaticBrokerReader.java | 23 +-
.../kafka/trident/TridentKafkaEmitter.java | 36 +-
.../jvm/storm/kafka/trident/ZkBrokerReader.java | 20 +-
.../storm/kafka/DynamicBrokersReaderTest.java | 114 +-
.../src/test/storm/kafka/KafkaUtilsTest.java | 58 +-
.../src/test/storm/kafka/TestUtils.java | 13 +-
.../src/test/storm/kafka/ZkCoordinatorTest.java | 8 +-
.../test/storm/kafka/bolt/KafkaBoltTest.java | 4 +-
storm-core/src/clj/backtype/storm/cluster.clj | 57 +-
storm-core/src/clj/backtype/storm/config.clj | 4 +
storm-core/src/clj/backtype/storm/converter.clj | 19 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 45 +
.../src/clj/backtype/storm/daemon/nimbus.clj | 35 +-
.../clj/backtype/storm/daemon/supervisor.clj | 155 +-
.../src/clj/backtype/storm/daemon/worker.clj | 4 +-
storm-core/src/clj/backtype/storm/stats.clj | 9 +
storm-core/src/clj/backtype/storm/ui/core.clj | 166 +-
storm-core/src/clj/backtype/storm/util.clj | 10 +-
storm-core/src/jvm/backtype/storm/Config.java | 21 +
.../src/jvm/backtype/storm/StormSubmitter.java | 20 +-
.../storm/coordination/BatchBoltExecutor.java | 4 +-
.../storm/coordination/CoordinatedBolt.java | 14 +-
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 10 +-
.../src/jvm/backtype/storm/drpc/JoinResult.java | 8 +-
.../jvm/backtype/storm/generated/Nimbus.java | 1977 +++++++++++++++++-
.../backtype/storm/generated/ProfileAction.java | 74 +
.../storm/generated/ProfileRequest.java | 631 ++++++
.../security/auth/ShellBasedGroupsMapping.java | 10 +-
.../auth/authorizer/DRPCAuthorizerBase.java | 2 +-
.../auth/authorizer/SimpleACLAuthorizer.java | 8 +
.../authorizer/SimpleWhitelistAuthorizer.java | 7 +-
.../storm/security/auth/kerberos/AutoTGT.java | 10 +-
.../serialization/BlowfishTupleSerializer.java | 6 +-
.../jvm/backtype/storm/spout/ShellSpout.java | 4 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 42 +-
.../storm/testing/MemoryTransactionalSpout.java | 9 +-
.../testing/OpaqueMemoryTransactionalSpout.java | 8 +-
.../storm/testing/TupleCaptureBolt.java | 4 +-
.../TransactionalSpoutBatchExecutor.java | 4 +-
.../src/jvm/backtype/storm/tuple/TupleImpl.java | 6 +-
.../jvm/backtype/storm/utils/LocalState.java | 22 +-
.../jvm/backtype/storm/utils/ShellProcess.java | 6 +-
.../jvm/backtype/storm/utils/ShellUtils.java | 2 +-
.../src/jvm/backtype/storm/utils/Time.java | 4 +-
.../storm/validation/ConfigValidation.java | 83 +-
.../trident/spout/TridentSpoutExecutor.java | 2 +-
.../trident/topology/TridentBoltExecutor.java | 6 +-
.../storm/trident/tuple/TridentTupleView.java | 12 +-
.../src/native/worker-launcher/impl/main.c | 10 +
.../worker-launcher/impl/worker-launcher.c | 47 +
.../worker-launcher/impl/worker-launcher.h | 2 +
storm-core/src/py/storm/Nimbus-remote | 14 +
storm-core/src/py/storm/Nimbus.py | 396 ++++
storm-core/src/py/storm/ttypes.py | 122 ++
storm-core/src/storm.thrift | 20 +
storm-core/src/ui/public/component.html | 167 +-
.../templates/component-page-template.html | 53 +
90 files changed, 5819 insertions(+), 1166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/bin/storm.py
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/conf/defaults.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/storm-core/src/clj/backtype/storm/config.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 64cbfd6,c86b73f..118e2c2
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -26,9 -26,8 +26,9 @@@
(:use [backtype.storm config util log timer local-state])
(:import [backtype.storm.utils VersionInfo])
(:import [backtype.storm Config])
- (:import [backtype.storm.generated WorkerResources])
+ (:import [backtype.storm.generated WorkerResources ProfileAction])
(:use [backtype.storm.daemon common])
+ (:require [backtype.storm.command [healthcheck :as healthcheck]])
(:require [backtype.storm.daemon [worker :as worker]]
[backtype.storm [process-simulator :as psim] [cluster :as cluster] [event :as event]]
[clojure.set :as set])
@@@ -547,16 -674,11 +675,21 @@@
0
(conf SUPERVISOR-MONITOR-FREQUENCY-SECS)
(fn [] (.add processes-event-manager sync-processes)))
+ (schedule-recurring (:event-timer supervisor)
+ (* 60 5)
+ (* 60 5)
+ (fn [] (let [health-code (healthcheck/health-check conf)
+ ids (my-worker-ids conf)]
+ (if (not (= health-code 0))
+ (do
+ (doseq [id ids]
+ (shutdown-worker supervisor id))
+ (throw (RuntimeException. "Supervisor failed health check. Exiting."))))))))
+ ;; Launch a thread that Runs profiler commands . Starts with 30 seconds delay, every 30 seconds
+ (schedule-recurring (:event-timer supervisor)
+ 30
+ 30
+ (fn [] (.add event-manager run-profiler-actions-fn))))
(log-message "Starting supervisor with id " (:supervisor-id supervisor) " at host " (:my-hostname supervisor))
(reify
Shutdownable
http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------