You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/11/02 23:19:21 UTC
[14/17] storm git commit: Merge
Merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8cb379fd
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8cb379fd
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8cb379fd
Branch: refs/heads/master
Commit: 8cb379fd995c489f765dac81d334946993ab6bd9
Parents: 94de2a5 f582b8f
Author: Sumit Chawla <su...@gmail.com>
Authored: Sun Oct 25 22:54:50 2015 -0700
Committer: Sumit Chawla <su...@gmail.com>
Committed: Sun Oct 25 22:54:50 2015 -0700
----------------------------------------------------------------------
.gitignore | 3 +
CHANGELOG.md | 50 +-
DEVELOPER.md | 33 +-
LICENSE | 9 +
README.markdown | 4 +
STORM-UI-REST-API.md | 21 +-
bin/storm | 20 +-
bin/storm.py | 51 +-
conf/defaults.yaml | 22 +-
dev-tools/storm-merge.py | 2 +-
dev-tools/travis/travis-install.sh | 4 +-
dev-tools/travis/travis-script.sh | 5 +-
docs/DYNAMIC_LOG_LEVEL_SETTINGS.md | 41 +
docs/documentation/FAQ.md | 2 +-
docs/images/dynamic_log_level_settings_1.png | Bin 0 -> 93689 bytes
docs/images/dynamic_log_level_settings_2.png | Bin 0 -> 78785 bytes
examples/storm-starter/pom.xml | 17 +-
.../storm/starter/MultipleLoggerTopology.java | 105 +
.../starter/ResourceAwareExampleTopology.java | 87 +
.../jvm/storm/starter/ThroughputVsLatency.java | 432 +
.../bolt/IntermediateRankingsBoltTest.java | 2 +-
.../starter/bolt/RollingCountBoltTest.java | 2 +-
.../starter/bolt/TotalRankingsBoltTest.java | 2 +-
.../storm/starter/tools/MockTupleHelpers.java | 40 -
external/flux/flux-core/pom.xml | 1 -
external/flux/flux-examples/pom.xml | 1 -
external/storm-elasticsearch/pom.xml | 5 +
external/storm-eventhubs/pom.xml | 1 -
external/storm-hbase/README.md | 10 +
.../storm/hbase/bolt/AbstractHBaseBolt.java | 2 +
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 75 +-
external/storm-hdfs/pom.xml | 71 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 121 +
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 51 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 42 +-
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 258 +
external/storm-hive/pom.xml | 7 +
.../org/apache/storm/hive/bolt/HiveBolt.java | 9 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 56 +-
external/storm-kafka/README.md | 16 +-
external/storm-kafka/pom.xml | 5 +
.../src/jvm/storm/kafka/KafkaSpout.java | 21 +-
.../src/jvm/storm/kafka/PartitionManager.java | 20 +-
.../src/test/storm/kafka/KafkaUtilsTest.java | 3 +-
.../test/storm/kafka/bolt/KafkaBoltTest.java | 9 +-
external/storm-metrics/pom.xml | 58 +
.../metrics/hdrhistogram/HistogramMetric.java | 79 +
.../apache/storm/metrics/sigar/CPUMetric.java | 60 +
.../resources/libsigar-amd64-freebsd-6.so | Bin 0 -> 210641 bytes
.../resources/resources/libsigar-amd64-linux.so | Bin 0 -> 246605 bytes
.../resources/libsigar-amd64-solaris.so | Bin 0 -> 251360 bytes
.../resources/libsigar-ia64-hpux-11.sl | Bin 0 -> 577452 bytes
.../resources/resources/libsigar-ia64-linux.so | Bin 0 -> 494929 bytes
.../resources/resources/libsigar-pa-hpux-11.sl | Bin 0 -> 516096 bytes
.../resources/resources/libsigar-ppc-aix-5.so | Bin 0 -> 400925 bytes
.../resources/resources/libsigar-ppc-linux.so | Bin 0 -> 258547 bytes
.../resources/resources/libsigar-ppc64-aix-5.so | Bin 0 -> 425077 bytes
.../resources/resources/libsigar-ppc64-linux.so | Bin 0 -> 330767 bytes
.../resources/resources/libsigar-s390x-linux.so | Bin 0 -> 269932 bytes
.../resources/libsigar-sparc-solaris.so | Bin 0 -> 285004 bytes
.../resources/libsigar-sparc64-solaris.so | Bin 0 -> 261896 bytes
.../resources/libsigar-universal-macosx.dylib | Bin 0 -> 377668 bytes
.../resources/libsigar-universal64-macosx.dylib | Bin 0 -> 397440 bytes
.../resources/libsigar-x86-freebsd-5.so | Bin 0 -> 179751 bytes
.../resources/libsigar-x86-freebsd-6.so | Bin 0 -> 179379 bytes
.../resources/resources/libsigar-x86-linux.so | Bin 0 -> 233385 bytes
.../resources/resources/libsigar-x86-solaris.so | Bin 0 -> 242880 bytes
.../resources/resources/sigar-amd64-winnt.dll | Bin 0 -> 402432 bytes
.../resources/resources/sigar-x86-winnt.dll | Bin 0 -> 266240 bytes
.../resources/resources/sigar-x86-winnt.lib | Bin 0 -> 99584 bytes
external/storm-solr/pom.xml | 4 +-
log4j2/cluster.xml | 34 +-
pom.xml | 134 +-
storm-core/pom.xml | 271 +-
storm-core/src/clj/backtype/storm/cluster.clj | 97 +-
.../clj/backtype/storm/command/kill_workers.clj | 33 +
.../backtype/storm/command/set_log_level.clj | 75 +
storm-core/src/clj/backtype/storm/config.clj | 58 +-
storm-core/src/clj/backtype/storm/converter.clj | 54 +-
.../backtype/storm/daemon/builtin_metrics.clj | 84 +-
.../src/clj/backtype/storm/daemon/common.clj | 13 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 28 +-
.../src/clj/backtype/storm/daemon/executor.clj | 25 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 343 +-
.../clj/backtype/storm/daemon/supervisor.clj | 86 +-
.../src/clj/backtype/storm/daemon/task.clj | 10 +-
.../src/clj/backtype/storm/daemon/worker.clj | 138 +-
.../src/clj/backtype/storm/local_state.clj | 20 +-
storm-core/src/clj/backtype/storm/log.clj | 12 +-
storm-core/src/clj/backtype/storm/stats.clj | 1512 ++-
storm-core/src/clj/backtype/storm/testing.clj | 34 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 1064 +--
.../src/clj/backtype/storm/ui/helpers.clj | 35 +-
storm-core/src/clj/backtype/storm/util.clj | 12 +-
storm-core/src/jvm/backtype/storm/Config.java | 648 +-
.../jvm/backtype/storm/ConfigValidation.java | 375 -
.../src/jvm/backtype/storm/StormSubmitter.java | 35 +-
.../storm/coordination/CoordinatedBolt.java | 2 +-
.../storm/generated/AlreadyAliveException.java | 2 +-
.../backtype/storm/generated/Assignment.java | 378 +-
.../storm/generated/AuthorizationException.java | 2 +-
.../src/jvm/backtype/storm/generated/Bolt.java | 2 +-
.../storm/generated/BoltAggregateStats.java | 704 ++
.../jvm/backtype/storm/generated/BoltStats.java | 442 +-
.../storm/generated/ClusterSummary.java | 110 +-
.../storm/generated/ClusterWorkerHeartbeat.java | 54 +-
.../storm/generated/CommonAggregateStats.java | 902 ++
.../generated/ComponentAggregateStats.java | 752 ++
.../storm/generated/ComponentCommon.java | 2 +-
.../storm/generated/ComponentPageInfo.java | 2194 +++++
.../backtype/storm/generated/ComponentType.java | 62 +
.../backtype/storm/generated/Credentials.java | 46 +-
.../storm/generated/DRPCExecutionException.java | 2 +-
.../backtype/storm/generated/DRPCRequest.java | 2 +-
.../backtype/storm/generated/DebugOptions.java | 2 +-
.../storm/generated/DistributedRPC.java | 2 +-
.../generated/DistributedRPCInvocations.java | 2 +-
.../jvm/backtype/storm/generated/ErrorInfo.java | 2 +-
.../storm/generated/ExecutorAggregateStats.java | 526 ++
.../backtype/storm/generated/ExecutorInfo.java | 2 +-
.../backtype/storm/generated/ExecutorStats.java | 170 +-
.../storm/generated/ExecutorSummary.java | 2 +-
.../storm/generated/GetInfoOptions.java | 2 +-
.../storm/generated/GlobalStreamId.java | 2 +-
.../generated/InvalidTopologyException.java | 2 +-
.../backtype/storm/generated/JavaObject.java | 2 +-
.../backtype/storm/generated/KillOptions.java | 2 +-
.../storm/generated/LSApprovedWorkers.java | 46 +-
.../generated/LSSupervisorAssignments.java | 50 +-
.../storm/generated/LSSupervisorId.java | 2 +-
.../storm/generated/LSWorkerHeartbeat.java | 38 +-
.../storm/generated/LocalAssignment.java | 155 +-
.../storm/generated/LocalStateData.java | 50 +-
.../jvm/backtype/storm/generated/LogConfig.java | 475 +
.../jvm/backtype/storm/generated/LogLevel.java | 836 ++
.../storm/generated/LogLevelAction.java | 65 +
.../jvm/backtype/storm/generated/Nimbus.java | 8830 +++++++++++++-----
.../backtype/storm/generated/NimbusSummary.java | 2 +-
.../jvm/backtype/storm/generated/NodeInfo.java | 34 +-
.../storm/generated/NotAliveException.java | 2 +-
.../backtype/storm/generated/NullStruct.java | 2 +-
.../storm/generated/RebalanceOptions.java | 46 +-
.../storm/generated/ShellComponent.java | 2 +-
.../storm/generated/SpecificAggregateStats.java | 387 +
.../storm/generated/SpoutAggregateStats.java | 407 +
.../jvm/backtype/storm/generated/SpoutSpec.java | 2 +-
.../backtype/storm/generated/SpoutStats.java | 254 +-
.../storm/generated/StateSpoutSpec.java | 2 +-
.../jvm/backtype/storm/generated/StormBase.java | 94 +-
.../backtype/storm/generated/StormTopology.java | 2 +-
.../backtype/storm/generated/StreamInfo.java | 2 +-
.../backtype/storm/generated/SubmitOptions.java | 2 +-
.../storm/generated/SupervisorInfo.java | 276 +-
.../storm/generated/SupervisorSummary.java | 168 +-
.../storm/generated/ThriftSerializedObject.java | 2 +-
.../backtype/storm/generated/TopologyInfo.java | 162 +-
.../storm/generated/TopologyPageInfo.java | 2003 ++++
.../backtype/storm/generated/TopologyStats.java | 1094 +++
.../storm/generated/TopologySummary.java | 2 +-
.../storm/generated/WorkerResources.java | 605 ++
.../storm/logging/ThriftAccessLogger.java | 27 +
.../logging/filters/AccessLoggingFilter.java | 52 +
.../backtype/storm/messaging/netty/Client.java | 10 +-
.../metric/HttpForwardingMetricsConsumer.java | 81 +
.../metric/HttpForwardingMetricsServer.java | 119 +
.../metric/internal/CountStatAndMetric.java | 211 +
.../metric/internal/LatencyStatAndMetric.java | 259 +
.../storm/metric/internal/MetricStatTimer.java | 27 +
.../internal/MultiCountStatAndMetric.java | 112 +
.../internal/MultiLatencyStatAndMetric.java | 109 +
.../storm/metric/internal/RateTracker.java | 165 +
.../AbstractDNSToSwitchMapping.java | 95 +
.../networktopography/DNSToSwitchMapping.java | 50 +
.../DefaultRackDNSToSwitchMapping.java | 35 +
.../jvm/backtype/storm/nimbus/NimbusInfo.java | 8 +
.../jvm/backtype/storm/scheduler/Cluster.java | 46 +-
.../storm/scheduler/SupervisorDetails.java | 61 +-
.../backtype/storm/scheduler/Topologies.java | 15 +-
.../storm/scheduler/TopologyDetails.java | 330 +-
.../backtype/storm/scheduler/WorkerSlot.java | 25 +
.../storm/scheduler/resource/Component.java | 54 +
.../storm/scheduler/resource/RAS_Node.java | 575 ++
.../resource/ResourceAwareScheduler.java | 149 +
.../storm/scheduler/resource/ResourceUtils.java | 133 +
.../resource/strategies/IStrategy.java | 37 +
.../strategies/ResourceAwareStrategy.java | 480 +
.../storm/security/auth/ReqContext.java | 7 +
.../auth/authorizer/DenyAuthorizer.java | 5 -
.../auth/authorizer/NoopAuthorizer.java | 5 -
.../auth/authorizer/SimpleACLAuthorizer.java | 21 +-
.../authorizer/SimpleWhitelistAuthorizer.java | 5 -
.../auth/kerberos/ServerCallbackHandler.java | 2 +
.../AlternateRackDNSToSwitchMapping.java | 48 +
.../topology/BaseConfigurationDeclarer.java | 31 +-
.../ComponentConfigurationDeclarer.java | 3 +
.../src/jvm/backtype/storm/tuple/Tuple.java | 8 +
.../src/jvm/backtype/storm/tuple/TupleImpl.java | 7 +-
.../backtype/storm/utils/DisruptorQueue.java | 63 +-
.../jvm/backtype/storm/utils/RateTracker.java | 119 -
.../src/jvm/backtype/storm/utils/Time.java | 12 +-
.../src/jvm/backtype/storm/utils/Utils.java | 283 +-
.../storm/validation/ConfigValidation.java | 609 ++
.../validation/ConfigValidationAnnotations.java | 214 +
.../storm/validation/ConfigValidationUtils.java | 175 +
.../trident/topology/TridentBoltExecutor.java | 4 +-
storm-core/src/py/storm/Nimbus-remote | 28 +
storm-core/src/py/storm/Nimbus.py | 1451 ++-
storm-core/src/py/storm/ttypes.py | 2715 +++++-
storm-core/src/storm.thrift | 134 +
storm-core/src/ui/public/css/style.css | 8 +
storm-core/src/ui/public/js/script.js | 20 +
storm-core/src/ui/public/js/visualization.js | 92 +-
.../templates/component-page-template.html | 2 +-
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 120 +-
storm-core/src/ui/public/topology.html | 149 +-
.../test/clj/backtype/storm/cluster_test.clj | 8 +-
.../test/clj/backtype/storm/config_test.clj | 186 -
.../clj/backtype/storm/integration_test.clj | 8 +-
.../test/clj/backtype/storm/multilang_test.clj | 4 +-
.../test/clj/backtype/storm/nimbus_test.clj | 101 +-
.../scheduler/multitenant_scheduler_test.clj | 34 +-
.../scheduler/resource_aware_scheduler_test.clj | 669 ++
.../test/clj/backtype/storm/scheduler_test.clj | 3 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 40 +-
.../clj/backtype/storm/serialization_test.clj | 14 +-
.../test/clj/backtype/storm/supervisor_test.clj | 199 +-
.../test/clj/backtype/storm/testing4j_test.clj | 1 +
.../clj/backtype/storm/transactional_test.clj | 5 +-
.../test/clj/backtype/storm/worker_test.clj | 179 +-
.../jvm/backtype/storm/TestConfigValidate.java | 642 ++
.../metric/internal/CountStatAndMetricTest.java | 86 +
.../internal/LatencyStatAndMetricTest.java | 83 +
.../storm/metric/internal/RateTrackerTest.java | 94 +
.../backtype/storm/utils/MockTupleHelpers.java | 40 +
.../backtype/storm/utils/RateTrackerTest.java | 62 -
storm-dist/binary/src/main/assembly/binary.xml | 32 -
237 files changed, 34721 insertions(+), 7020 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8cb379fd/external/storm-kafka/README.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8cb379fd/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8cb379fd/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --cc external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 5804ad0,10405fa..f5ea696
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@@ -136,16 -135,16 +136,16 @@@ public class PartitionManager
if (toEmit == null) {
return EmitState.NO_EMITTED;
}
- Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg, _partition.topic);
- if (tups != null) {
- if(_spoutConfig.topicAsStreamId) {
- for (List<Object> tup : tups) {
- collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
- }
- } else {
- for (List<Object> tup : tups) {
- collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
- }
- Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
++ Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg,_partition.topic);
+ if ((tups != null) && tups.iterator().hasNext()) {
+ if(_spoutConfig.topicAsStreamId) {
+ for (List<Object> tup : tups) {
+ collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
+ }
+ } else {
+ for (List<Object> tup : tups) {
+ collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
+ }
}
break;
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/8cb379fd/external/storm-kafka/src/test/storm/kafka/KafkaUtilsTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/8cb379fd/external/storm-kafka/src/test/storm/kafka/bolt/KafkaBoltTest.java
----------------------------------------------------------------------