You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/03/18 20:39:15 UTC
[08/16] storm git commit: Merge remote-tracking branch
'upstream/master' into ru
Merge remote-tracking branch 'upstream/master' into ru
Conflicts:
storm-core/src/clj/backtype/storm/bootstrap.clj
storm-core/src/clj/backtype/storm/config.clj
storm-core/src/clj/backtype/storm/daemon/executor.clj
storm-core/src/clj/backtype/storm/daemon/nimbus.clj
storm-core/src/jvm/backtype/storm/generated/ExecutorStats.java
storm-core/src/py/storm/ttypes.py
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5b67b571
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5b67b571
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5b67b571
Branch: refs/heads/master
Commit: 5b67b571fe45e14fd52b835bc1de056538de1e70
Parents: f23b3c8 5eff2e7
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Thu Mar 5 17:05:33 2015 -0800
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Thu Mar 5 17:05:33 2015 -0800
----------------------------------------------------------------------
.gitignore | 4 +-
BYLAWS.md | 31 +-
CHANGELOG.md | 19 +
DEVELOPER.md | 24 +-
README.markdown | 3 +
SECURITY.md | 10 +-
STORM-UI-REST-API.md | 45 +-
bin/storm-config.cmd | 14 +-
bin/storm.cmd | 33 +-
conf/defaults.yaml | 2 +-
docs/documentation/Powered-By.md | 83 +
docs/documentation/Trident-API-Overview.md | 2 +-
docs/images/architecture.png | Bin 0 -> 69825 bytes
docs/images/architecture.svg | 1458 +++
docs/images/bolt.png | Bin 0 -> 24796 bytes
docs/images/bolt.svg | 743 ++
docs/images/security.png | Bin 0 -> 72415 bytes
docs/images/security.svg | 1779 ++++
docs/images/spout.png | Bin 0 -> 22911 bytes
docs/images/spout.svg | 833 ++
docs/images/storm.svg | 1326 +++
docs/images/topology.png | Bin 59837 -> 64740 bytes
docs/images/topology.svg | 1044 ++
docs/images/topology_dark.png | Bin 0 -> 49692 bytes
docs/images/topology_dark.svg | 1101 +++
.../jvm/storm/starter/BasicDRPCTopology.java | 3 +-
external/storm-hdfs/pom.xml | 18 +-
.../storm/hdfs/bolt/HdfsFileTopology.java | 6 +-
.../storm/hdfs/bolt/SequenceFileTopology.java | 4 +-
.../storm/hdfs/trident/TridentFileTopology.java | 2 +-
.../hdfs/trident/TridentSequenceTopology.java | 6 +-
external/storm-hive/README.md | 113 +
external/storm-hive/pom.xml | 143 +
.../org/apache/storm/hive/bolt/HiveBolt.java | 289 +
.../bolt/mapper/DelimitedRecordHiveMapper.java | 143 +
.../storm/hive/bolt/mapper/HiveMapper.java | 81 +
.../hive/bolt/mapper/JsonRecordHiveMapper.java | 132 +
.../apache/storm/hive/common/HiveOptions.java | 146 +
.../org/apache/storm/hive/common/HiveUtils.java | 76 +
.../apache/storm/hive/common/HiveWriter.java | 420 +
.../apache/storm/hive/trident/HiveState.java | 306 +
.../storm/hive/trident/HiveStateFactory.java | 31 +
.../apache/storm/hive/trident/HiveUpdater.java | 14 +
.../apache/storm/hive/bolt/HiveSetupUtil.java | 220 +
.../apache/storm/hive/bolt/HiveTopology.java | 150 +
.../hive/bolt/HiveTopologyPartitioned.java | 153 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 330 +
.../storm/hive/common/TestHiveWriter.java | 193 +
.../storm/hive/trident/TridentHiveTopology.java | 190 +
external/storm-jdbc/LICENSE | 202 +
external/storm-jdbc/README.md | 240 +
external/storm-jdbc/pom.xml | 125 +
.../storm/jdbc/bolt/AbstractJdbcBolt.java | 57 +
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 71 +
.../apache/storm/jdbc/bolt/JdbcLookupBolt.java | 76 +
.../org/apache/storm/jdbc/common/Column.java | 111 +
.../apache/storm/jdbc/common/JdbcClient.java | 228 +
.../java/org/apache/storm/jdbc/common/Util.java | 75 +
.../storm/jdbc/mapper/JdbcLookupMapper.java | 26 +
.../apache/storm/jdbc/mapper/JdbcMapper.java | 33 +
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 46 +
.../storm/jdbc/mapper/SimpleJdbcMapper.java | 92 +
.../storm/jdbc/trident/state/JdbcQuery.java | 40 +
.../storm/jdbc/trident/state/JdbcState.java | 145 +
.../jdbc/trident/state/JdbcStateFactory.java | 40 +
.../storm/jdbc/trident/state/JdbcUpdater.java | 32 +
.../storm/jdbc/common/JdbcClientTest.java | 88 +
.../org/apache/storm/jdbc/common/UtilTest.java | 69 +
.../org/apache/storm/jdbc/spout/UserSpout.java | 90 +
.../jdbc/topology/AbstractUserTopology.java | 106 +
.../jdbc/topology/UserPersistanceTopology.java | 48 +
.../UserPersistanceTridentTopology.java | 61 +
external/storm-jdbc/src/test/sql/test.sql | 1 +
.../src/jvm/storm/kafka/PartitionManager.java | 5 +-
external/storm-redis/LICENSE | 202 +
external/storm-redis/README.md | 137 +
external/storm-redis/pom.xml | 65 +
.../storm/redis/bolt/AbstractRedisBolt.java | 67 +
.../trident/mapper/TridentTupleMapper.java | 27 +
.../trident/state/RedisClusterMapState.java | 294 +
.../redis/trident/state/RedisClusterState.java | 80 +
.../trident/state/RedisClusterStateQuerier.java | 78 +
.../trident/state/RedisClusterStateUpdater.java | 76 +
.../redis/trident/state/RedisMapState.java | 323 +
.../storm/redis/trident/state/RedisState.java | 83 +
.../redis/trident/state/RedisStateQuerier.java | 70 +
.../state/RedisStateSetCountQuerier.java | 74 +
.../trident/state/RedisStateSetUpdater.java | 80 +
.../redis/trident/state/RedisStateUpdater.java | 75 +
.../redis/util/config/JedisClusterConfig.java | 82 +
.../redis/util/config/JedisPoolConfig.java | 97 +
.../util/container/JedisClusterContainer.java | 47 +
.../JedisCommandsContainerBuilder.java | 38 +
.../JedisCommandsInstanceContainer.java | 25 +
.../redis/util/container/JedisContainer.java | 65 +
.../storm/redis/topology/LookupWordCount.java | 127 +
.../redis/topology/PersistentWordCount.java | 117 +
.../storm/redis/topology/WordCounter.java | 58 +
.../apache/storm/redis/topology/WordSpout.java | 88 +
.../storm/redis/trident/PrintFunction.java | 40 +
.../redis/trident/WordCountTridentRedis.java | 97 +
.../trident/WordCountTridentRedisCluster.java | 103 +
.../WordCountTridentRedisClusterMap.java | 101 +
.../redis/trident/WordCountTridentRedisMap.java | 95 +
.../redis/trident/WordCountTupleMapper.java | 16 +
pom.xml | 16 +-
storm-core/pom.xml | 13 +-
storm-core/src/clj/backtype/storm/bootstrap.clj | 64 -
storm-core/src/clj/backtype/storm/clojure.clj | 2 +-
storm-core/src/clj/backtype/storm/config.clj | 3 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 6 +-
.../src/clj/backtype/storm/daemon/executor.clj | 30 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 29 +-
.../clj/backtype/storm/daemon/supervisor.clj | 191 +-
.../src/clj/backtype/storm/daemon/task.clj | 20 +-
.../src/clj/backtype/storm/daemon/worker.clj | 70 +-
.../src/clj/backtype/storm/messaging/local.clj | 2 +-
storm-core/src/clj/backtype/storm/tuple.clj | 4 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 63 +-
.../src/clj/backtype/storm/ui/helpers.clj | 1 +
storm-core/src/clj/backtype/storm/util.clj | 3 +
.../storm/drpc/DRPCInvocationsClient.java | 6 +
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 14 +-
.../jvm/backtype/storm/drpc/ReturnResults.java | 8 +-
.../storm/generated/AlreadyAliveException.java | 149 +-
.../storm/generated/AuthorizationException.java | 149 +-
.../src/jvm/backtype/storm/generated/Bolt.java | 194 +-
.../jvm/backtype/storm/generated/BoltStats.java | 908 +-
.../storm/generated/ClusterSummary.java | 339 +-
.../storm/generated/ComponentCommon.java | 426 +-
.../storm/generated/ComponentObject.java | 86 +-
.../backtype/storm/generated/Credentials.java | 220 +-
.../storm/generated/DRPCExecutionException.java | 149 +-
.../backtype/storm/generated/DRPCRequest.java | 185 +-
.../storm/generated/DistributedRPC.java | 529 +-
.../generated/DistributedRPCInvocations.java | 1199 ++-
.../jvm/backtype/storm/generated/ErrorInfo.java | 300 +-
.../backtype/storm/generated/ExecutorInfo.java | 198 +-
.../storm/generated/ExecutorSpecificStats.java | 72 +-
.../backtype/storm/generated/ExecutorStats.java | 609 +-
.../storm/generated/ExecutorSummary.java | 371 +-
.../storm/generated/GetInfoOptions.java | 166 +-
.../storm/generated/GlobalStreamId.java | 185 +-
.../jvm/backtype/storm/generated/Grouping.java | 163 +-
.../generated/InvalidTopologyException.java | 149 +-
.../backtype/storm/generated/JavaObject.java | 239 +-
.../backtype/storm/generated/JavaObjectArg.java | 108 +-
.../backtype/storm/generated/KillOptions.java | 176 +-
.../jvm/backtype/storm/generated/Nimbus.java | 9177 +++++++++++++-----
.../storm/generated/NotAliveException.java | 149 +-
.../backtype/storm/generated/NullStruct.java | 112 +-
.../storm/generated/NumErrorsChoice.java | 3 +-
.../storm/generated/RebalanceOptions.java | 348 +-
.../storm/generated/ShellComponent.java | 202 +-
.../jvm/backtype/storm/generated/SpoutSpec.java | 194 +-
.../backtype/storm/generated/SpoutStats.java | 614 +-
.../storm/generated/StateSpoutSpec.java | 194 +-
.../backtype/storm/generated/StormTopology.java | 410 +-
.../backtype/storm/generated/StreamInfo.java | 249 +-
.../backtype/storm/generated/SubmitOptions.java | 208 +-
.../storm/generated/SupervisorSummary.java | 309 +-
.../backtype/storm/generated/TopologyInfo.java | 609 +-
.../storm/generated/TopologyInitialStatus.java | 3 +-
.../storm/generated/TopologySummary.java | 486 +-
.../storm/messaging/ConnectionWithStatus.java | 32 +
.../backtype/storm/messaging/netty/Client.java | 712 +-
.../messaging/netty/SaslStormClientHandler.java | 5 +-
.../backtype/storm/messaging/netty/Server.java | 182 +-
.../netty/StormClientPipelineFactory.java | 5 +-
.../security/auth/SimpleTransportPlugin.java | 2 +-
.../trident/drpc/ReturnResultsReducer.java | 4 +-
.../trident/spout/RichSpoutBatchExecutor.java | 1 +
storm-core/src/py/storm/DistributedRPC-remote | 35 +-
storm-core/src/py/storm/DistributedRPC.py | 38 +-
.../py/storm/DistributedRPCInvocations-remote | 43 +-
.../src/py/storm/DistributedRPCInvocations.py | 95 +-
storm-core/src/py/storm/Nimbus-remote | 111 +-
storm-core/src/py/storm/Nimbus.py | 640 +-
storm-core/src/py/storm/constants.py | 6 +-
storm-core/src/py/storm/ttypes.py | 1290 +--
storm-core/src/ui/public/index.html | 1 +
.../test/clj/backtype/storm/clojure_test.clj | 9 +-
.../test/clj/backtype/storm/drpc_test.clj | 8 +-
.../test/clj/backtype/storm/grouping_test.clj | 9 +-
.../clj/backtype/storm/integration_test.clj | 10 +-
.../storm/messaging/netty_integration_test.clj | 5 +-
.../storm/messaging/netty_unit_test.clj | 73 +-
.../test/clj/backtype/storm/messaging_test.clj | 6 +-
.../test/clj/backtype/storm/metrics_test.clj | 8 +-
.../test/clj/backtype/storm/multilang_test.clj | 6 +-
.../test/clj/backtype/storm/nimbus_test.clj | 21 +-
.../scheduler/multitenant_scheduler_test.clj | 4 +-
.../test/clj/backtype/storm/scheduler_test.clj | 4 +-
.../storm/security/auth/AuthUtils_test.clj | 6 +-
.../backtype/storm/security/auth/auth_test.clj | 13 +-
.../storm/security/auth/drpc_auth_test.clj | 7 +-
.../storm/security/auth/nimbus_auth_test.clj | 10 +-
.../clj/backtype/storm/subtopology_test.clj | 9 +-
.../test/clj/backtype/storm/supervisor_test.clj | 16 +-
.../test/clj/backtype/storm/tick_tuple_test.clj | 7 +-
.../clj/backtype/storm/transactional_test.clj | 14 +-
.../test/clj/backtype/storm/worker_test.clj | 37 +
storm-dist/binary/src/main/assembly/binary.xml | 38 +-
203 files changed, 32972 insertions(+), 8388 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/conf/defaults.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/config.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/config.clj
index 00396c7,483c6ff..5e7893b
--- a/storm-core/src/clj/backtype/storm/config.clj
+++ b/storm-core/src/clj/backtype/storm/config.clj
@@@ -212,8 -211,7 +212,7 @@@
(let [stormroot (supervisor-stormdist-root conf storm-id)
conf-path (supervisor-stormconf-path stormroot)
topology-path (supervisor-stormcode-path stormroot)]
- (merge conf (Utils/javaDeserialize (FileUtils/readFileToByteArray (File. conf-path)) java.util.Map))
- ))
- (merge conf (clojurify-structure (Utils/deserialize (FileUtils/readFileToByteArray (File. conf-path)))))))
++ (merge conf (clojurify-structure (Utils/javaDeserialize (FileUtils/readFileToByteArray (File. conf-path)))))))
(defn read-supervisor-topology
[conf storm-id]
http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/daemon/executor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/executor.clj
index a1ecc4a,ec68d45..0e083ea
--- a/storm-core/src/clj/backtype/storm/daemon/executor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/executor.clj
@@@ -15,23 -15,29 +15,31 @@@
;; limitations under the License.
(ns backtype.storm.daemon.executor
(:use [backtype.storm.daemon common])
- (:use [backtype.storm bootstrap])
- (:import [backtype.storm ICredentialsListener]
- [backtype.storm.generated Grouping]
++ (:import [backtype.storm.generated Grouping]
+ [java.io Serializable])
+ (:use [backtype.storm util config log timer stats])
+ (:import [java.util List Random HashMap ArrayList LinkedList Map])
+ (:import [backtype.storm ICredentialsListener])
(:import [backtype.storm.hooks ITaskHook])
- (:import [backtype.storm.tuple Tuple])
- (:import [backtype.storm.spout ISpoutWaitStrategy])
+ (:import [backtype.storm.tuple Tuple Fields TupleImpl MessageId])
+ (:import [backtype.storm.spout ISpoutWaitStrategy ISpout SpoutOutputCollector ISpoutOutputCollector])
(:import [backtype.storm.hooks.info SpoutAckInfo SpoutFailInfo
EmitInfo BoltFailInfo BoltAckInfo BoltExecuteInfo])
- (:import [backtype.storm.metric.api IMetric IMetricsConsumer$TaskInfo IMetricsConsumer$DataPoint])
- (:import [backtype.storm Config])
+ (:import [backtype.storm.grouping CustomStreamGrouping])
+ (:import [backtype.storm.task WorkerTopologyContext IBolt OutputCollector IOutputCollector])
+ (:import [backtype.storm.generated GlobalStreamId])
+ (:import [backtype.storm.utils Utils MutableObject RotatingMap RotatingMap$ExpiredCallback MutableLong Time])
+ (:import [com.lmax.disruptor InsufficientCapacityException])
+ (:import [backtype.storm.serialization KryoTupleSerializer KryoTupleDeserializer])
+ (:import [backtype.storm.daemon Shutdownable])
+ (:import [backtype.storm.metric.api IMetric IMetricsConsumer$TaskInfo IMetricsConsumer$DataPoint StateMetric])
+ (:import [backtype.storm Config Constants])
(:import [java.util.concurrent ConcurrentLinkedQueue])
- (:require [backtype.storm [tuple :as tuple]])
+ (:require [backtype.storm [tuple :as tuple] [thrift :as thrift]
+ [cluster :as cluster] [disruptor :as disruptor] [stats :as stats]])
(:require [backtype.storm.daemon [task :as task]])
- (:require [backtype.storm.daemon.builtin-metrics :as builtin-metrics]))
-
- (bootstrap)
+ (:require [backtype.storm.daemon.builtin-metrics :as builtin-metrics])
+ (:require [clojure.set :as set]))
(defn- mk-fields-grouper [^Fields out-fields ^Fields group-fields ^List target-tasks]
(let [num-tasks (count target-tasks)
http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index d79b542,f1b4082..2c7b82c
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@@ -14,10 -14,14 +14,14 @@@
;; See the License for the specific language governing permissions and
;; limitations under the License.
(ns backtype.storm.daemon.nimbus
- (:import [java.nio ByteBuffer]
- [java.util Collections]
- [backtype.storm.generated StormTopology])
- (:import [java.io FileNotFoundException])
+ (:import [org.apache.thrift.server THsHaServer THsHaServer$Args])
+ (:import [org.apache.thrift.protocol TBinaryProtocol TBinaryProtocol$Factory])
+ (:import [org.apache.thrift.exception])
+ (:import [org.apache.thrift.transport TNonblockingServerTransport TNonblockingServerSocket])
+ (:import [org.apache.commons.io FileUtils])
- (:import [java.nio ByteBuffer]
++ (:import [java.nio ByteBuffer])
+ [java.util Collections HashMap])
+ (:import [java.io FileNotFoundException File FileOutputStream])
(:import [java.nio.channels Channels WritableByteChannel])
(:import [backtype.storm.security.auth ThriftServer ThriftConnectionType ReqContext AuthUtils])
(:use [backtype.storm.scheduler.DefaultScheduler])
http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/test/clj/backtype/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/nimbus_test.clj
index 7671f58,99d1e32..f9502c2
--- a/storm-core/test/clj/backtype/storm/nimbus_test.clj
+++ b/storm-core/test/clj/backtype/storm/nimbus_test.clj
@@@ -15,18 -15,25 +15,25 @@@
;; limitations under the License.
(ns backtype.storm.nimbus-test
(:use [clojure test])
- (:require [backtype.storm [util :as util]])
+ (:require [backtype.storm [util :as util] [stats :as stats]])
(:require [backtype.storm.daemon [nimbus :as nimbus]])
- (:import [backtype.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter])
+ (:import [backtype.storm.testing TestWordCounter TestWordSpout TestGlobalCount
+ TestAggregatesCounter TestPlannerSpout TestPlannerBolt])
(:import [backtype.storm.scheduler INimbus])
- (:import [backtype.storm.generated Credentials])
- (:use [backtype.storm bootstrap testing MockAutoCred])
+ (:import [backtype.storm.generated Credentials NotAliveException SubmitOptions
+ TopologyInitialStatus AlreadyAliveException KillOptions RebalanceOptions
+ InvalidTopologyException AuthorizationException])
+ (:import [java.util HashMap])
+ (:import [java.io File])
+ (:import [backtype.storm.utils Time])
+ (:import [org.apache.commons.io FileUtils])
+ (:use [backtype.storm testing MockAutoCred util config log timer])
(:use [backtype.storm.daemon common])
(:require [conjure.core])
- (:use [conjure core])
- )
-
- (bootstrap)
+ (:require [backtype.storm
+ [thrift :as thrift]
+ [cluster :as cluster]])
+ (:use [conjure core]))
(defn storm-component->task-info [cluster storm-name]
(let [storm-id (get-storm-id (:storm-cluster-state cluster) storm-name)