You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by sr...@apache.org on 2015/11/02 23:19:17 UTC
[10/17] storm git commit: Merge remote-tracking branch
'upstream/master'
Merge remote-tracking branch 'upstream/master'
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/088c244c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/088c244c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/088c244c
Branch: refs/heads/master
Commit: 088c244cd4cd031ea6226842d6f49e94c1370821
Parents: f5fe47b 8eb4509
Author: Sumit Chawla <su...@gmail.com>
Authored: Tue Sep 8 19:41:40 2015 -0700
Committer: Sumit Chawla <su...@gmail.com>
Committed: Tue Sep 8 19:41:40 2015 -0700
----------------------------------------------------------------------
.gitignore | 2 +-
CHANGELOG.md | 35 +-
README.markdown | 13 +
SECURITY.md | 2 +-
STORM-UI-REST-API.md | 41 +-
TODO | 178 -----
bin/storm-config.cmd | 18 +-
bin/storm.py | 22 +-
conf/defaults.yaml | 9 +-
docs/about/multi-language.md | 2 +-
docs/documentation/FAQ.md | 7 +-
.../nimbus_ha_leader_election_and_failover.png | Bin 0 -> 154316 bytes
.../images/nimbus_ha_topology_submission.png | Bin 0 -> 134180 bytes
docs/documentation/nimbus-ha-design.md | 217 +++++
examples/storm-starter/pom.xml | 29 +
.../starter/trident/TridentKafkaWordCount.java | 230 ++++++
.../flux-core/src/test/resources/log4j2.xml | 34 +
.../flux-core/src/test/resources/logback.xml | 30 -
external/storm-elasticsearch/README.md | 91 ++-
external/storm-elasticsearch/pom.xml | 16 +-
.../elasticsearch/ElasticsearchGetRequest.java | 36 +
.../elasticsearch/EsLookupResultOutput.java | 43 +
.../elasticsearch/bolt/AbstractEsBolt.java | 56 +-
.../storm/elasticsearch/bolt/EsIndexBolt.java | 25 +-
.../storm/elasticsearch/bolt/EsLookupBolt.java | 81 ++
.../elasticsearch/bolt/EsPercolateBolt.java | 26 +-
.../common/DefaultEsTupleMapper.java | 42 +
.../storm/elasticsearch/common/EsConfig.java | 64 +-
.../elasticsearch/common/EsTupleMapper.java | 55 ++
.../common/StormElasticSearchClient.java | 48 ++
.../common/TransportAddresses.java | 72 ++
.../storm/elasticsearch/trident/EsState.java | 54 +-
.../elasticsearch/trident/EsStateFactory.java | 27 +-
.../storm/elasticsearch/trident/EsUpdater.java | 6 +-
.../bolt/AbstractEsBoltIntegrationTest.java | 91 +++
.../elasticsearch/bolt/AbstractEsBoltTest.java | 80 +-
.../elasticsearch/bolt/EsIndexBoltTest.java | 43 +-
.../elasticsearch/bolt/EsIndexTopology.java | 8 +-
.../bolt/EsLookupBoltIntegrationTest.java | 137 ++++
.../elasticsearch/bolt/EsLookupBoltTest.java | 125 +++
.../elasticsearch/bolt/EsPercolateBoltTest.java | 35 +-
.../elasticsearch/common/EsConfigTest.java | 71 ++
.../storm/elasticsearch/common/EsTestUtil.java | 5 +
.../common/TransportAddressesTest.java | 81 ++
.../trident/EsStateFactoryTest.java | 32 +
.../trident/TridentEsTopology.java | 8 +-
external/storm-hbase/pom.xml | 2 +-
external/storm-hdfs/README.md | 9 +
external/storm-hdfs/pom.xml | 15 +-
.../ha/codedistributor/HDFSCodeDistributor.java | 101 +++
.../apache/storm/hdfs/trident/HdfsState.java | 392 +++++++--
.../trident/rotation/FileRotationPolicy.java | 14 +
.../rotation/FileSizeRotationPolicy.java | 13 +
.../hdfs/trident/rotation/NoRotationPolicy.java | 10 +
.../trident/rotation/TimedRotationPolicy.java | 31 +-
.../storm/hdfs/trident/HdfsStateTest.java | 206 +++++
external/storm-hive/README.md | 3 +-
.../org/apache/storm/hive/bolt/HiveBolt.java | 39 +-
.../apache/storm/hive/common/HiveOptions.java | 11 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 12 +-
external/storm-kafka/CHANGELOG.md | 13 -
external/storm-kafka/README.md | 3 +
.../ExponentialBackoffMsgRetryManager.java | 4 +-
.../src/jvm/storm/kafka/KafkaSpout.java | 6 +-
.../src/jvm/storm/kafka/PartitionManager.java | 10 +-
.../src/jvm/storm/kafka/SpoutConfig.java | 3 +
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 17 +-
.../test/storm/kafka/bolt/KafkaBoltTest.java | 32 +
external/storm-solr/README.md | 201 +++++
external/storm-solr/pom.xml | 98 +++
.../apache/storm/solr/bolt/SolrUpdateBolt.java | 136 ++++
.../storm/solr/config/CountBasedCommit.java | 59 ++
.../storm/solr/config/SolrCommitStrategy.java | 30 +
.../apache/storm/solr/config/SolrConfig.java | 42 +
.../storm/solr/mapper/SolrFieldsMapper.java | 182 +++++
.../storm/solr/mapper/SolrJsonMapper.java | 160 ++++
.../apache/storm/solr/mapper/SolrMapper.java | 32 +
.../storm/solr/mapper/SolrMapperException.java | 24 +
.../org/apache/storm/solr/schema/CopyField.java | 50 ++
.../org/apache/storm/solr/schema/Field.java | 50 ++
.../org/apache/storm/solr/schema/FieldType.java | 63 ++
.../org/apache/storm/solr/schema/Schema.java | 116 +++
.../storm/solr/schema/SolrFieldTypeFinder.java | 182 +++++
.../schema/builder/RestJsonSchemaBuilder.java | 69 ++
.../solr/schema/builder/SchemaBuilder.java | 27 +
.../apache/storm/solr/trident/SolrState.java | 67 ++
.../storm/solr/trident/SolrStateFactory.java | 44 +
.../apache/storm/solr/trident/SolrUpdater.java | 33 +
.../storm/solr/spout/SolrFieldsSpout.java | 76 ++
.../apache/storm/solr/spout/SolrJsonSpout.java | 120 +++
.../storm/solr/topology/SolrFieldsTopology.java | 56 ++
.../storm/solr/topology/SolrJsonTopology.java | 48 ++
.../storm/solr/topology/SolrTopology.java | 82 ++
.../solr/trident/SolrFieldsTridentTopology.java | 45 ++
.../solr/trident/SolrJsonTridentTopology.java | 45 ++
.../org/apache/storm/solr/util/TestUtil.java | 30 +
pom.xml | 19 +-
storm-core/pom.xml | 90 +--
storm-core/src/clj/backtype/storm/cluster.clj | 100 ++-
.../backtype/storm/command/shell_submission.clj | 9 +-
storm-core/src/clj/backtype/storm/config.clj | 15 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 2 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 252 ++++--
.../clj/backtype/storm/daemon/supervisor.clj | 63 +-
storm-core/src/clj/backtype/storm/thrift.clj | 23 +-
storm-core/src/clj/backtype/storm/timer.clj | 7 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 95 ++-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
storm-core/src/clj/backtype/storm/zookeeper.clj | 108 ++-
storm-core/src/jvm/backtype/storm/Config.java | 63 +-
.../jvm/backtype/storm/ConfigValidation.java | 13 +-
.../storm/codedistributor/ICodeDistributor.java | 56 ++
.../LocalFileSystemCodeDistributor.java | 106 +++
.../coordination/BatchSubtopologyBuilder.java | 2 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 2 +-
.../storm/generated/ClusterSummary.java | 292 ++++---
.../backtype/storm/generated/NimbusSummary.java | 796 +++++++++++++++++++
.../backtype/storm/generated/TopologyInfo.java | 221 +++--
.../storm/generated/TopologySummary.java | 107 ++-
.../backtype/storm/messaging/netty/Client.java | 37 +-
.../backtype/storm/nimbus/ILeaderElector.java | 60 ++
.../jvm/backtype/storm/nimbus/NimbusInfo.java | 93 +++
.../security/auth/SaslTransportPlugin.java | 8 +-
.../authorizer/DRPCSimpleACLAuthorizer.java | 2 +-
.../authorizer/ImpersonationAuthorizer.java | 2 +-
.../kerberos/KerberosSaslTransportPlugin.java | 5 +-
.../security/auth/kerberos/NoOpTTrasport.java | 40 +
.../src/jvm/backtype/storm/task/ShellBolt.java | 8 +-
.../testing/OpaqueMemoryTransactionalSpout.java | 2 +-
.../TransactionalTopologyBuilder.java | 2 +-
.../src/jvm/backtype/storm/tuple/Fields.java | 21 +-
.../src/jvm/backtype/storm/tuple/ITuple.java | 126 ++-
.../backtype/storm/utils/DisruptorQueue.java | 127 +--
.../jvm/backtype/storm/utils/NimbusClient.java | 78 +-
.../jvm/backtype/storm/utils/RotatingMap.java | 12 +-
.../jvm/backtype/storm/utils/TimeCacheMap.java | 60 +-
.../src/jvm/backtype/storm/utils/Utils.java | 32 +-
.../storm/trident/planner/SubtopologyBolt.java | 2 +-
.../jvm/storm/trident/spout/ITridentSpout.java | 4 +-
.../OpaquePartitionedTridentSpoutExecutor.java | 20 +-
.../spout/PartitionedTridentSpoutExecutor.java | 26 +-
.../trident/spout/RichSpoutBatchExecutor.java | 6 +-
.../trident/spout/RichSpoutBatchTriggerer.java | 14 +-
.../trident/spout/TridentSpoutCoordinator.java | 6 +-
.../trident/spout/TridentSpoutExecutor.java | 10 +-
.../storm/trident/testing/FeederBatchSpout.java | 2 +-
.../trident/topology/TridentBoltExecutor.java | 17 +-
.../topology/TridentTopologyBuilder.java | 4 +-
storm-core/src/py/storm/ttypes.py | 613 ++++++++------
storm-core/src/storm.thrift | 12 +-
storm-core/src/ui/public/index.html | 21 +
.../public/templates/index-page-template.html | 59 +-
.../templates/topology-page-template.html | 18 +-
storm-core/src/ui/public/topology.html | 35 +-
.../test/clj/backtype/storm/cluster_test.clj | 23 +-
.../test/clj/backtype/storm/config_test.clj | 4 +-
.../storm/messaging/netty_unit_test.clj | 2 +-
.../test/clj/backtype/storm/nimbus_test.clj | 210 +++--
.../backtype/storm/security/auth/auth_test.clj | 4 +-
.../storm/security/auth/nimbus_auth_test.clj | 14 +-
.../test/clj/backtype/storm/supervisor_test.clj | 3 +
.../test/clj/backtype/storm/utils_test.clj | 12 -
storm-dist/binary/src/main/assembly/binary.xml | 16 +
163 files changed, 8205 insertions(+), 1550 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/088c244c/external/storm-kafka/README.md
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/088c244c/external/storm-kafka/src/jvm/storm/kafka/KafkaSpout.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/088c244c/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --cc external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
index 18356bb,052d525..5804ad0
--- a/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/storm/kafka/PartitionManager.java
@@@ -136,10 -135,16 +136,16 @@@ public class PartitionManager
if (toEmit == null) {
return EmitState.NO_EMITTED;
}
- Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg);
+ Iterable<List<Object>> tups = KafkaUtils.generateTuples(_spoutConfig, toEmit.msg, _partition.topic);
if (tups != null) {
- for (List<Object> tup : tups) {
- collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
+ if(_spoutConfig.topicAsStreamId) {
+ for (List<Object> tup : tups) {
+ collector.emit(_spoutConfig.topic, tup, new KafkaMessageId(_partition, toEmit.offset));
+ }
+ } else {
+ for (List<Object> tup : tups) {
+ collector.emit(tup, new KafkaMessageId(_partition, toEmit.offset));
+ }
}
break;
} else {
http://git-wip-us.apache.org/repos/asf/storm/blob/088c244c/external/storm-kafka/src/test/storm/kafka/bolt/KafkaBoltTest.java
----------------------------------------------------------------------