You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2015/10/03 06:46:50 UTC
[2/4] storm git commit: Merge remote-tracking branch 'apache/master'
into storm-993-ui-uptime-seconds
Merge remote-tracking branch 'apache/master' into storm-993-ui-uptime-seconds
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/0a2ed633
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/0a2ed633
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/0a2ed633
Branch: refs/heads/master
Commit: 0a2ed63310f9ad9214ef7493134232b5bb6da80c
Parents: de0e146 8dd9e6e
Author: Derek Dagit <de...@yahoo-inc.com>
Authored: Sun Sep 27 20:27:07 2015 -0500
Committer: Derek Dagit <de...@yahoo-inc.com>
Committed: Sun Sep 27 20:27:07 2015 -0500
----------------------------------------------------------------------
.gitignore | 2 +-
CHANGELOG.md | 51 +-
DEVELOPER.md | 21 +
README.markdown | 18 +
SECURITY.md | 244 ++--
STORM-UI-REST-API.md | 47 +-
TODO | 178 ---
bin/storm-config.cmd | 18 +-
bin/storm.py | 38 +-
conf/defaults.yaml | 15 +-
docs/about/multi-language.md | 2 +-
docs/documentation/FAQ.md | 7 +-
.../documentation/Setting-up-a-Storm-cluster.md | 10 +-
docs/documentation/Tutorial.md | 2 +-
.../nimbus_ha_leader_election_and_failover.png | Bin 0 -> 154316 bytes
.../images/nimbus_ha_topology_submission.png | Bin 0 -> 134180 bytes
docs/documentation/nimbus-ha-design.md | 217 +++
examples/storm-starter/pom.xml | 29 +
.../starter/trident/TridentKafkaWordCount.java | 230 +++
.../flux-core/src/test/resources/log4j2.xml | 34 +
.../flux-core/src/test/resources/logback.xml | 30 -
external/storm-elasticsearch/README.md | 91 +-
external/storm-elasticsearch/pom.xml | 16 +-
.../elasticsearch/ElasticsearchGetRequest.java | 36 +
.../elasticsearch/EsLookupResultOutput.java | 43 +
.../elasticsearch/bolt/AbstractEsBolt.java | 56 +-
.../storm/elasticsearch/bolt/EsIndexBolt.java | 25 +-
.../storm/elasticsearch/bolt/EsLookupBolt.java | 81 ++
.../elasticsearch/bolt/EsPercolateBolt.java | 26 +-
.../common/DefaultEsTupleMapper.java | 42 +
.../storm/elasticsearch/common/EsConfig.java | 64 +-
.../elasticsearch/common/EsTupleMapper.java | 55 +
.../common/StormElasticSearchClient.java | 48 +
.../common/TransportAddresses.java | 72 +
.../storm/elasticsearch/trident/EsState.java | 54 +-
.../elasticsearch/trident/EsStateFactory.java | 27 +-
.../storm/elasticsearch/trident/EsUpdater.java | 6 +-
.../bolt/AbstractEsBoltIntegrationTest.java | 91 ++
.../elasticsearch/bolt/AbstractEsBoltTest.java | 80 +-
.../elasticsearch/bolt/EsIndexBoltTest.java | 43 +-
.../elasticsearch/bolt/EsIndexTopology.java | 8 +-
.../bolt/EsLookupBoltIntegrationTest.java | 137 ++
.../elasticsearch/bolt/EsLookupBoltTest.java | 125 ++
.../elasticsearch/bolt/EsPercolateBoltTest.java | 35 +-
.../elasticsearch/common/EsConfigTest.java | 71 +
.../storm/elasticsearch/common/EsTestUtil.java | 5 +
.../common/TransportAddressesTest.java | 81 ++
.../trident/EsStateFactoryTest.java | 32 +
.../trident/TridentEsTopology.java | 10 +-
.../trident/OpaqueTridentEventHubSpout.java | 2 +-
.../TransactionalTridentEventHubSpout.java | 2 +-
external/storm-hbase/pom.xml | 2 +-
.../org/apache/storm/hbase/common/Utils.java | 10 +-
external/storm-hdfs/pom.xml | 4 +-
.../ha/codedistributor/HDFSCodeDistributor.java | 101 ++
.../storm/hdfs/trident/FixedBatchSpout.java | 2 +-
.../storm/hive/trident/TridentHiveTopology.java | 2 +-
external/storm-jdbc/README.md | 6 +-
external/storm-kafka/CHANGELOG.md | 13 -
external/storm-kafka/README.md | 49 +-
.../ExponentialBackoffMsgRetryManager.java | 4 +-
.../src/jvm/storm/kafka/KafkaSpout.java | 17 +-
.../src/jvm/storm/kafka/PartitionManager.java | 10 +-
.../src/jvm/storm/kafka/SpoutConfig.java | 3 +
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 17 +-
.../kafka/trident/OpaqueTridentKafkaSpout.java | 4 +-
.../trident/TransactionalTridentKafkaSpout.java | 4 +-
.../src/test/storm/kafka/KafkaUtilsTest.java | 1 +
.../test/storm/kafka/bolt/KafkaBoltTest.java | 32 +
external/storm-solr/README.md | 201 +++
external/storm-solr/pom.xml | 98 ++
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 136 ++
.../storm/solr/config/CountBasedCommit.java | 59 +
.../storm/solr/config/SolrCommitStrategy.java | 30 +
.../apache/storm/solr/config/SolrConfig.java | 42 +
.../storm/solr/mapper/SolrFieldsMapper.java | 182 +++
.../storm/solr/mapper/SolrJsonMapper.java | 160 +++
.../apache/storm/solr/mapper/SolrMapper.java | 32 +
.../storm/solr/mapper/SolrMapperException.java | 24 +
.../org/apache/storm/solr/schema/CopyField.java | 50 +
.../org/apache/storm/solr/schema/Field.java | 50 +
.../org/apache/storm/solr/schema/FieldType.java | 63 +
.../org/apache/storm/solr/schema/Schema.java | 116 ++
.../storm/solr/schema/SolrFieldTypeFinder.java | 182 +++
.../schema/builder/RestJsonSchemaBuilder.java | 69 +
.../solr/schema/builder/SchemaBuilder.java | 27 +
.../apache/storm/solr/trident/SolrState.java | 67 +
.../storm/solr/trident/SolrStateFactory.java | 44 +
.../apache/storm/solr/trident/SolrUpdater.java | 33 +
.../storm/solr/spout/SolrFieldsSpout.java | 76 +
.../apache/storm/solr/spout/SolrJsonSpout.java | 120 ++
.../storm/solr/topology/SolrFieldsTopology.java | 56 +
.../storm/solr/topology/SolrJsonTopology.java | 48 +
.../storm/solr/topology/SolrTopology.java | 82 ++
.../solr/trident/SolrFieldsTridentTopology.java | 45 +
.../solr/trident/SolrJsonTridentTopology.java | 45 +
.../org/apache/storm/solr/util/TestUtil.java | 30 +
pom.xml | 19 +-
storm-core/pom.xml | 90 +-
storm-core/src/clj/backtype/storm/cluster.clj | 150 +-
.../backtype/storm/command/shell_submission.clj | 9 +-
storm-core/src/clj/backtype/storm/config.clj | 28 +-
storm-core/src/clj/backtype/storm/converter.clj | 23 +-
.../backtype/storm/daemon/builtin_metrics.clj | 25 +-
.../src/clj/backtype/storm/daemon/common.clj | 47 +-
.../src/clj/backtype/storm/daemon/executor.clj | 178 ++-
.../src/clj/backtype/storm/daemon/logviewer.clj | 3 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 281 +++-
.../clj/backtype/storm/daemon/supervisor.clj | 65 +-
.../src/clj/backtype/storm/daemon/worker.clj | 117 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 19 +-
storm-core/src/clj/backtype/storm/thrift.clj | 23 +-
storm-core/src/clj/backtype/storm/timer.clj | 7 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 205 ++-
storm-core/src/clj/backtype/storm/util.clj | 18 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 108 +-
storm-core/src/jvm/backtype/storm/Config.java | 115 +-
.../storm/codedistributor/ICodeDistributor.java | 56 +
.../LocalFileSystemCodeDistributor.java | 106 ++
.../coordination/BatchSubtopologyBuilder.java | 6 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 9 +-
.../backtype/storm/generated/Assignment.java | 194 +--
.../jvm/backtype/storm/generated/BoltStats.java | 342 ++---
.../storm/generated/ClusterSummary.java | 292 ++--
.../storm/generated/ClusterWorkerHeartbeat.java | 55 +-
.../backtype/storm/generated/Credentials.java | 46 +-
.../backtype/storm/generated/DebugOptions.java | 506 +++++++
.../backtype/storm/generated/ExecutorStats.java | 162 +--
.../storm/generated/LSApprovedWorkers.java | 46 +-
.../generated/LSSupervisorAssignments.java | 50 +-
.../storm/generated/LSWorkerHeartbeat.java | 38 +-
.../storm/generated/LocalAssignment.java | 38 +-
.../storm/generated/LocalStateData.java | 50 +-
.../jvm/backtype/storm/generated/Nimbus.java | 1305 +++++++++++++++++-
.../backtype/storm/generated/NimbusSummary.java | 796 +++++++++++
.../jvm/backtype/storm/generated/NodeInfo.java | 34 +-
.../storm/generated/RebalanceOptions.java | 46 +-
.../backtype/storm/generated/SpoutStats.java | 226 +--
.../jvm/backtype/storm/generated/StormBase.java | 225 ++-
.../storm/generated/SupervisorInfo.java | 110 +-
.../backtype/storm/generated/TopologyInfo.java | 398 +++++-
.../storm/generated/TopologySummary.java | 107 +-
.../backtype/storm/messaging/netty/Client.java | 39 +-
.../backtype/storm/metric/EventLoggerBolt.java | 58 +
.../storm/metric/FileBasedEventLogger.java | 105 ++
.../jvm/backtype/storm/metric/IEventLogger.java | 59 +
.../jvm/backtype/storm/metric/SystemBolt.java | 15 +
.../backtype/storm/nimbus/ILeaderElector.java | 60 +
.../jvm/backtype/storm/nimbus/NimbusInfo.java | 93 ++
.../security/auth/SaslTransportPlugin.java | 8 +-
.../security/auth/ShellBasedGroupsMapping.java | 2 +-
.../authorizer/DRPCSimpleACLAuthorizer.java | 2 +-
.../authorizer/ImpersonationAuthorizer.java | 2 +-
.../kerberos/KerberosSaslTransportPlugin.java | 5 +-
.../security/auth/kerberos/NoOpTTrasport.java | 40 +
.../testing/OpaqueMemoryTransactionalSpout.java | 2 +-
.../topology/BaseConfigurationDeclarer.java | 2 +-
.../ComponentConfigurationDeclarer.java | 2 +-
.../storm/topology/TopologyBuilder.java | 34 +-
.../TransactionalTopologyBuilder.java | 12 +-
.../src/jvm/backtype/storm/tuple/Fields.java | 21 +-
.../src/jvm/backtype/storm/tuple/ITuple.java | 126 +-
.../utils/DisruptorBackpressureCallback.java | 27 +
.../backtype/storm/utils/DisruptorQueue.java | 226 ++-
.../jvm/backtype/storm/utils/NimbusClient.java | 78 +-
.../jvm/backtype/storm/utils/RateTracker.java | 119 ++
.../jvm/backtype/storm/utils/RotatingMap.java | 12 +-
.../jvm/backtype/storm/utils/TimeCacheMap.java | 60 +-
.../src/jvm/backtype/storm/utils/Utils.java | 32 +-
.../storm/utils/WorkerBackpressureCallback.java | 26 +
.../storm/utils/WorkerBackpressureThread.java | 59 +
.../storm/trident/planner/SubtopologyBolt.java | 2 +-
.../storm/trident/spout/BatchSpoutExecutor.java | 2 +-
.../jvm/storm/trident/spout/IBatchSpout.java | 2 +-
.../spout/IOpaquePartitionedTridentSpout.java | 2 +-
.../trident/spout/IPartitionedTridentSpout.java | 2 +-
.../jvm/storm/trident/spout/ITridentSpout.java | 6 +-
.../OpaquePartitionedTridentSpoutExecutor.java | 20 +-
.../spout/PartitionedTridentSpoutExecutor.java | 26 +-
.../trident/spout/RichSpoutBatchExecutor.java | 8 +-
.../trident/spout/RichSpoutBatchTriggerer.java | 14 +-
.../trident/spout/TridentSpoutCoordinator.java | 6 +-
.../trident/spout/TridentSpoutExecutor.java | 10 +-
.../storm/trident/testing/FeederBatchSpout.java | 4 +-
.../testing/FeederCommitterBatchSpout.java | 2 +-
.../storm/trident/testing/FixedBatchSpout.java | 2 +-
.../trident/topology/TridentBoltExecutor.java | 17 +-
.../topology/TridentTopologyBuilder.java | 22 +-
storm-core/src/py/storm/Nimbus-remote | 7 +
storm-core/src/py/storm/Nimbus.py | 263 ++++
storm-core/src/py/storm/ttypes.py | 990 ++++++++-----
storm-core/src/storm.thrift | 26 +-
storm-core/src/ui/public/component.html | 8 +
storm-core/src/ui/public/index.html | 21 +
storm-core/src/ui/public/js/script.js | 78 +-
.../templates/component-page-template.html | 13 +
.../public/templates/index-page-template.html | 59 +-
.../templates/topology-page-template.html | 28 +-
storm-core/src/ui/public/topology.html | 38 +-
.../test/clj/backtype/storm/cluster_test.clj | 27 +-
.../test/clj/backtype/storm/config_test.clj | 28 +-
.../storm/messaging/netty_unit_test.clj | 2 +-
.../test/clj/backtype/storm/nimbus_test.clj | 282 ++--
.../backtype/storm/security/auth/auth_test.clj | 4 +-
.../storm/security/auth/nimbus_auth_test.clj | 14 +-
.../test/clj/backtype/storm/supervisor_test.clj | 3 +
.../test/clj/backtype/storm/utils_test.clj | 12 -
.../storm/topology/TopologyBuilderTest.java | 48 +
.../utils/DisruptorQueueBackpressureTest.java | 115 ++
.../backtype/storm/utils/RateTrackerTest.java | 62 +
storm-dist/binary/src/main/assembly/binary.xml | 16 +
211 files changed, 12789 insertions(+), 2706 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/0a2ed633/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --cc STORM-UI-REST-API.md
index 3e50688,a8f2bd1..2155113
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@@ -84,8 -84,7 +84,6 @@@ Response fields
|Field |Value|Description
|--- |--- |---
|stormVersion|String| Storm version|
--|nimbusUptime|String| Shows how long the cluster is running|
- |nimbusUptimeSeconds|Integer| Shows how long the cluster is running in seconds|
|supervisors|Integer| Number of supervisors running|
|topologies| Integer| Number of topologies running|
|slotsTotal| Integer|Total number of available worker slots|
@@@ -99,8 -98,7 +97,6 @@@ Sample response
```json
{
"stormVersion": "0.9.2-incubating-SNAPSHOT",
-- "nimbusUptime": "3m 53s",
- "nimbusUptimeSeconds": 233,
"supervisors": 1,
"slotsTotal": 4,
"slotsUsed": 3,
@@@ -142,6 -138,38 +138,40 @@@ Sample response
}
```
+ ### /api/v1/nimbus/summary (GET)
+
+ Returns summary information for all nimbus hosts.
+
+ Response fields:
+
+ |Field |Value|Description|
+ |--- |--- |---
+ |host| String | Nimbus' host name|
+ |port| int| Nimbus' port number|
+ |status| String| Possible values are Leader, Not a Leader, Dead|
+ |nimbusUpTime| String| Shows since how long the nimbus has been running|
++|nimbusUpTimeSeconds| String| Shows since how long the nimbus has been running in seconds|
+ |nimbusLogLink| String| Logviewer url to view the nimbus.log|
+ |version| String| Version of storm this nimbus host is running|
+
+ Sample response:
+
+ ```json
+ {
+ "nimbuses":[
+ {
+ "host":"192.168.202.1",
+ "port":6627,
+ "nimbusLogLink":"http:\/\/192.168.202.1:8000\/log?file=nimbus.log",
+ "status":Leader,
+ "version":"0.10.0-SNAPSHOT",
- "nimbusUpTime":"3m 33s"
++ "nimbusUpTime":"3m 33s",
++ "nimbusUpTimeSeconds":"213"
+ }
+ ]
+ }
+ ```
+
### /api/v1/topology/summary (GET)
Returns summary information for all topologies.
@@@ -169,10 -196,10 +199,11 @@@ Sample response
"name": "WordCount3",
"status": "ACTIVE",
"uptime": "6m 5s",
+ "uptimeSeconds": 365,
"tasksTotal": 28,
"workersTotal": 3,
- "executorsTotal": 28
+ "executorsTotal": 28,
+ "replicationCount": 1
}
]
}
http://git-wip-us.apache.org/repos/asf/storm/blob/0a2ed633/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 5b88b21,414bfb1..d9829ca
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -576,9 -605,41 +605,44 @@@
"executorsTotal" total-executors
"tasksTotal" total-tasks })))
+ (defn convert-to-nimbus-summary[nimbus-seed]
+ (let [[host port] (.split nimbus-seed ":")]
+ {
+ "host" host
+ "port" port
+ "nimbusLogLink" (nimbus-log-link host port)
+ "status" "Offline"
+ "version" "Not applicable"
- "nimbusUpTime" "Not applicable"}
++ "nimbusUpTime" "Not applicable"
++ "nimbusUptimeSeconds" "Not applicable"}
+ ))
+
+ (defn nimbus-summary
+ ([]
+ (thrift/with-configured-nimbus-connection nimbus
+ (nimbus-summary
+ (.get_nimbuses (.getClusterInfo ^Nimbus$Client nimbus)))))
+ ([nimbuses]
+ (let [nimbus-seeds (set (map #(str %1 ":" (*STORM-CONF* NIMBUS-THRIFT-PORT)) (set (*STORM-CONF* NIMBUS-SEEDS))))
+ alive-nimbuses (set (map #(str (.get_host %1) ":" (.get_port %1)) nimbuses))
+ offline-nimbuses (clojure.set/difference nimbus-seeds alive-nimbuses)
+ offline-nimbuses-summary (map #(convert-to-nimbus-summary %1) offline-nimbuses)]
+ {"nimbuses"
+ (concat offline-nimbuses-summary
- (for [^NimbusSummary n nimbuses]
++ (for [^NimbusSummary n nimbuses
++ :let [uptime (.get_uptime_secs n)]]
+ {
+ "host" (.get_host n)
+ "port" (.get_port n)
+ "nimbusLogLink" (nimbus-log-link (.get_host n) (.get_port n))
+ "status" (if (.is_isLeader n) "Leader" "Not a Leader")
+ "version" (.get_version n)
- "nimbusUpTime" (pretty-uptime-sec (.get_uptime_secs n))}))})))
++ "nimbusUpTime" (pretty-uptime-sec uptime)
++ "nimbusUpTimeSeconds" uptime}))})))
+
(defn supervisor-summary
([]
- (with-nimbus nimbus
+ (thrift/with-configured-nimbus-connection nimbus
(supervisor-summary
(.get_supervisors (.getClusterInfo ^Nimbus$Client nimbus)))))
([summs]
http://git-wip-us.apache.org/repos/asf/storm/blob/0a2ed633/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index df6debb,1e585ff..9c93e46
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -74,6 -68,43 +68,43 @@@
</tbody>
</table>
</script>
+
+ <script id="nimbus-summary-template" type="text/html">
+ <table class="zebra-striped" id="nimbus-summary-table">
+ <thead>
+ <tr>
+ <th>
+ <span data-toggle="tooltip" data-placement="right" title="nimbus host name.">Host</span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" data-placement="above" title="Nimbus Thrift port number.">Port</span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" data-placement="above" title="Leader if this host is leader, Not a Leader for all other live hosts,
+ note that these hosts may or may not be in leader lock queue, and Dead for hosts that are part of nimbus.seeds list but are not alive.">Status</span>
+ </th>
+ <th>
+ <span data-toggle="tooltip" data-placement="above" title="Storm version this nimbus host is running.">Version</span>
+ </th>
+ <th>
- <span data-toggle="tooltip" data-placement="left" title="Time since this nimbus host has been running.">UpTime Seconds</span>
++ <span data-toggle="tooltip" data-placement="left" title="Time since this nimbus host has been running.">UpTime</span>
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ {{#nimbuses}}
+ <tr>
+ <td>{{host}}</td>
+ <td><a href="{{nimbusLogLink}}">{{port}}</a></td>
+ <td>{{status}}</td>
+ <td>{{version}}</td>
+ <td>{{nimbusUpTime}}</td>
+ </tr>
+ {{/nimbuses}}
+ </tbody>
+ </table>
+ </script>
+
<script id="topology-summary-template" type="text/html">
<table class="table table-striped compact" id="topology-summary-table">
<thead>