You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/05/22 21:54:46 UTC
[01/18] storm git commit: STORM-596: remove config
topology.receiver.buffer.size
Repository: storm
Updated Branches:
refs/heads/master 972ab8559 -> 2195fd606
STORM-596: remove config topology.receiver.buffer.size
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cad1505d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cad1505d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cad1505d
Branch: refs/heads/master
Commit: cad1505deb214facb0fb70e24034e30f79a973da
Parents: 97ae36e
Author: caofangkun <ca...@gmail.com>
Authored: Wed Dec 24 11:44:25 2014 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Dec 24 11:44:25 2014 +0800
----------------------------------------------------------------------
conf/defaults.yaml | 1 -
conf/defaults.yaml.orig | 201 +++
.../src/clj/backtype/storm/daemon/worker.clj | 3 +-
.../src/clj/backtype/storm/messaging/loader.clj | 13 +-
storm-core/src/jvm/backtype/storm/Config.java | 7 -
.../src/jvm/backtype/storm/Config.java.orig | 1434 ++++++++++++++++++
6 files changed, 1642 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/cad1505d/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index d03c2c9..08086ff 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -181,7 +181,6 @@ topology.fall.back.on.java.serialization: true
topology.worker.childopts: null
topology.executor.receive.buffer.size: 1024 #batched
topology.executor.send.buffer.size: 1024 #individual messages
-topology.receiver.buffer.size: 8 # setting it too high causes a lot of problems (heartbeat thread gets starved, throughput plummets)
topology.transfer.buffer.size: 1024 # batched
topology.tick.tuple.freq.secs: null
topology.worker.shared.thread.pool.size: 4
http://git-wip-us.apache.org/repos/asf/storm/blob/cad1505d/conf/defaults.yaml.orig
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml.orig b/conf/defaults.yaml.orig
new file mode 100644
index 0000000..d03c2c9
--- /dev/null
+++ b/conf/defaults.yaml.orig
@@ -0,0 +1,201 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+########### These all have default values as shown
+########### Additional configuration goes into storm.yaml
+
+java.library.path: "/usr/local/lib:/opt/local/lib:/usr/lib"
+
+### storm.* configs are general configurations
+# the local dir is where jars are kept
+storm.local.dir: "storm-local"
+storm.zookeeper.servers:
+ - "localhost"
+storm.zookeeper.port: 2181
+storm.zookeeper.root: "/storm"
+storm.zookeeper.session.timeout: 20000
+storm.zookeeper.connection.timeout: 15000
+storm.zookeeper.retry.times: 5
+storm.zookeeper.retry.interval: 1000
+storm.zookeeper.retry.intervalceiling.millis: 30000
+storm.zookeeper.auth.user: null
+storm.zookeeper.auth.password: null
+storm.cluster.mode: "distributed" # can be distributed or local
+storm.local.mode.zmq: false
+storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin"
+storm.principal.tolocal: "backtype.storm.security.auth.DefaultPrincipalToLocal"
+storm.group.mapping.service: "backtype.storm.security.auth.ShellBasedGroupsMapping"
+storm.messaging.transport: "backtype.storm.messaging.netty.Context"
+storm.nimbus.retry.times: 5
+storm.nimbus.retry.interval.millis: 2000
+storm.nimbus.retry.intervalceiling.millis: 60000
+storm.auth.simple-white-list.users: []
+storm.auth.simple-acl.users: []
+storm.auth.simple-acl.users.commands: []
+storm.auth.simple-acl.admins: []
+storm.meta.serialization.delegate: "backtype.storm.serialization.DefaultSerializationDelegate"
+
+### nimbus.* configs are for the master
+nimbus.host: "localhost"
+nimbus.thrift.port: 6627
+nimbus.thrift.threads: 64
+nimbus.thrift.max_buffer_size: 1048576
+nimbus.childopts: "-Xmx1024m"
+nimbus.task.timeout.secs: 30
+nimbus.supervisor.timeout.secs: 60
+nimbus.monitor.freq.secs: 10
+nimbus.cleanup.inbox.freq.secs: 600
+nimbus.inbox.jar.expiration.secs: 3600
+nimbus.task.launch.secs: 120
+nimbus.reassign: true
+nimbus.file.copy.expiration.secs: 600
+nimbus.topology.validator: "backtype.storm.nimbus.DefaultTopologyValidator"
+nimbus.credential.renewers.freq.secs: 600
+
+### ui.* configs are for the master
+ui.port: 8080
+ui.childopts: "-Xmx768m"
+ui.actions.enabled: true
+ui.filter: null
+ui.filter.params: null
+ui.users: null
+ui.header.buffer.bytes: 4096
+ui.http.creds.plugin: backtype.storm.security.auth.DefaultHttpCredentialsPlugin
+
+logviewer.port: 8000
+logviewer.childopts: "-Xmx128m"
+logviewer.cleanup.age.mins: 10080
+logviewer.appender.name: "A1"
+
+logs.users: null
+
+drpc.port: 3772
+drpc.worker.threads: 64
+drpc.max_buffer_size: 1048576
+drpc.queue.size: 128
+drpc.invocations.port: 3773
+drpc.invocations.threads: 64
+drpc.request.timeout.secs: 600
+drpc.childopts: "-Xmx768m"
+drpc.http.port: 3774
+drpc.https.port: -1
+drpc.https.keystore.password: ""
+drpc.https.keystore.type: "JKS"
+drpc.http.creds.plugin: backtype.storm.security.auth.DefaultHttpCredentialsPlugin
+drpc.authorizer.acl.filename: "drpc-auth-acl.yaml"
+drpc.authorizer.acl.strict: false
+
+transactional.zookeeper.root: "/transactional"
+transactional.zookeeper.servers: null
+transactional.zookeeper.port: null
+
+### supervisor.* configs are for node supervisors
+# Define the amount of workers that can be run on this machine. Each worker is assigned a port to use for communication
+supervisor.slots.ports:
+ - 6700
+ - 6701
+ - 6702
+ - 6703
+supervisor.childopts: "-Xmx256m"
+supervisor.run.worker.as.user: false
+#how long supervisor will wait to ensure that a worker process is started
+supervisor.worker.start.timeout.secs: 120
+#how long between heartbeats until supervisor considers that worker dead and tries to restart it
+supervisor.worker.timeout.secs: 30
+#how frequently the supervisor checks on the status of the processes it's monitoring and restarts if necessary
+supervisor.monitor.frequency.secs: 3
+#how frequently the supervisor heartbeats to the cluster state (for nimbus)
+supervisor.heartbeat.frequency.secs: 5
+supervisor.enable: true
+supervisor.supervisors: []
+supervisor.supervisors.commands: []
+
+
+### worker.* configs are for task workers
+worker.childopts: "-Xmx768m"
+worker.gc.childopts: ""
+worker.heartbeat.frequency.secs: 1
+
+# control how many worker receiver threads we need per worker
+topology.worker.receiver.thread.count: 1
+
+task.heartbeat.frequency.secs: 3
+task.refresh.poll.secs: 10
+task.credentials.poll.secs: 30
+
+zmq.threads: 1
+zmq.linger.millis: 5000
+zmq.hwm: 0
+
+
+storm.messaging.netty.server_worker_threads: 1
+storm.messaging.netty.client_worker_threads: 1
+storm.messaging.netty.buffer_size: 5242880 #5MB buffer
+# Since nimbus.task.launch.secs and supervisor.worker.start.timeout.secs are 120, other workers should also wait at least that long before giving up on connecting to the other worker.
+storm.messaging.netty.max_retries: 300
+storm.messaging.netty.max_wait_ms: 1000
+storm.messaging.netty.min_wait_ms: 100
+
+# If the Netty messaging layer is busy(netty internal buffer not writable), the Netty client will try to batch message as more as possible up to the size of storm.messaging.netty.transfer.batch.size bytes, otherwise it will try to flush message as soon as possible to reduce latency.
+storm.messaging.netty.transfer.batch.size: 262144
+
+# We check with this interval that whether the Netty channel is writable and try to write pending messages if it is.
+storm.messaging.netty.flush.check.interval.ms: 10
+
+# By default, the Netty SASL authentication is set to false. Users can override and set it true for a specific topology.
+storm.messaging.netty.authentication: false
+
+# default number of seconds group mapping service will cache user group
+storm.group.mapping.service.cache.duration.secs: 120
+
+### topology.* configs are for specific executing storms
+topology.enable.message.timeouts: true
+topology.debug: false
+topology.workers: 1
+topology.acker.executors: null
+topology.tasks: null
+# maximum amount of time a message has to complete before it's considered failed
+topology.message.timeout.secs: 30
+topology.multilang.serializer: "backtype.storm.multilang.JsonSerializer"
+topology.skip.missing.kryo.registrations: false
+topology.max.task.parallelism: null
+topology.max.spout.pending: null
+topology.state.synchronization.timeout.secs: 60
+topology.stats.sample.rate: 0.05
+topology.builtin.metrics.bucket.size.secs: 60
+topology.fall.back.on.java.serialization: true
+topology.worker.childopts: null
+topology.executor.receive.buffer.size: 1024 #batched
+topology.executor.send.buffer.size: 1024 #individual messages
+topology.receiver.buffer.size: 8 # setting it too high causes a lot of problems (heartbeat thread gets starved, throughput plummets)
+topology.transfer.buffer.size: 1024 # batched
+topology.tick.tuple.freq.secs: null
+topology.worker.shared.thread.pool.size: 4
+topology.disruptor.wait.strategy: "com.lmax.disruptor.BlockingWaitStrategy"
+topology.spout.wait.strategy: "backtype.storm.spout.SleepSpoutWaitStrategy"
+topology.sleep.spout.wait.strategy.time.ms: 1
+topology.error.throttle.interval.secs: 10
+topology.max.error.report.per.interval: 5
+topology.kryo.factory: "backtype.storm.serialization.DefaultKryoFactory"
+topology.tuple.serializer: "backtype.storm.serialization.types.ListDelegateSerializer"
+topology.trident.batch.emit.interval.millis: 500
+topology.testing.always.try.serialize: false
+topology.classpath: null
+topology.environment: null
+topology.bolts.outgoing.overflow.buffer.enable: false
+
+dev.zookeeper.path: "/tmp/dev-storm-zookeeper"
http://git-wip-us.apache.org/repos/asf/storm/blob/cad1505d/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj
index e717ce4..991116b 100644
--- a/storm-core/src/clj/backtype/storm/daemon/worker.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj
@@ -355,8 +355,7 @@
(:storm-id worker)
(:receiver-thread-count worker)
(:port worker)
- (:transfer-local-fn worker)
- (-> worker :storm-conf (get TOPOLOGY-RECEIVER-BUFFER-SIZE))
+ (:transfer-local-fn worker))
:kill-fn (fn [t] (exit-process! 11))))
(defn- close-resources [worker]
http://git-wip-us.apache.org/repos/asf/storm/blob/cad1505d/storm-core/src/clj/backtype/storm/messaging/loader.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/messaging/loader.clj b/storm-core/src/clj/backtype/storm/messaging/loader.clj
index 13130af..c154ed8 100644
--- a/storm-core/src/clj/backtype/storm/messaging/loader.clj
+++ b/storm-core/src/clj/backtype/storm/messaging/loader.clj
@@ -24,7 +24,7 @@
(defn mk-local-context []
(local/mk-context))
-(defn- mk-receive-thread [storm-id port transfer-local-fn daemon kill-fn priority socket max-buffer-size thread-id]
+(defn- mk-receive-thread [storm-id port transfer-local-fn daemon kill-fn priority socket thread-id]
(async-loop
(fn []
(log-message "Starting receive-thread: [stormId: " storm-id ", port: " port ", thread-id: " thread-id " ]")
@@ -54,20 +54,19 @@
:priority priority
:thread-name (str "worker-receiver-thread-" thread-id)))
-(defn- mk-receive-threads [storm-id port transfer-local-fn daemon kill-fn priority socket max-buffer-size thread-count]
+(defn- mk-receive-threads [storm-id port transfer-local-fn daemon kill-fn priority socket thread-count]
(into [] (for [thread-id (range thread-count)]
- (mk-receive-thread storm-id port transfer-local-fn daemon kill-fn priority socket max-buffer-size thread-id))))
+ (mk-receive-thread storm-id port transfer-local-fn daemon kill-fn priority socket thread-id))))
(defnk launch-receive-thread!
- [context socket storm-id receiver-thread-count port transfer-local-fn max-buffer-size
+ [context socket storm-id receiver-thread-count port transfer-local-fn
:daemon true
:kill-fn (fn [t] (System/exit 1))
:priority Thread/NORM_PRIORITY]
- (let [max-buffer-size (int max-buffer-size)
- local-hostname (memoized-local-hostname)
+ (let [local-hostname (memoized-local-hostname)
thread-count (if receiver-thread-count receiver-thread-count 1)
- vthreads (mk-receive-threads storm-id port transfer-local-fn daemon kill-fn priority socket max-buffer-size thread-count)]
+ vthreads (mk-receive-threads storm-id port transfer-local-fn daemon kill-fn priority socket thread-count)]
(fn []
(let [kill-socket (.connect ^IContext context storm-id local-hostname port)]
(log-message "Shutting down receiving-thread: [" storm-id ", " port "]")
http://git-wip-us.apache.org/repos/asf/storm/blob/cad1505d/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index c680354..192f49d 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -1064,13 +1064,6 @@ public class Config extends HashMap<String, Object> {
public static final Object TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
/**
- * The maximum number of messages to batch from the thread receiving off the network to the
- * executor queues. Must be a power of 2.
- */
- public static final String TOPOLOGY_RECEIVER_BUFFER_SIZE="topology.receiver.buffer.size";
- public static final Object TOPOLOGY_RECEIVER_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
-
- /**
* The size of the Disruptor send queue for each executor. Must be a power of 2.
*/
public static final String TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE="topology.executor.send.buffer.size";
http://git-wip-us.apache.org/repos/asf/storm/blob/cad1505d/storm-core/src/jvm/backtype/storm/Config.java.orig
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java.orig b/storm-core/src/jvm/backtype/storm/Config.java.orig
new file mode 100644
index 0000000..c680354
--- /dev/null
+++ b/storm-core/src/jvm/backtype/storm/Config.java.orig
@@ -0,0 +1,1434 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package backtype.storm;
+
+import backtype.storm.serialization.IKryoDecorator;
+import backtype.storm.serialization.IKryoFactory;
+import com.esotericsoftware.kryo.Serializer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Topology configs are specified as a plain old map. This class provides a
+ * convenient way to create a topology config map by providing setter methods for
+ * all the configs that can be set. It also makes it easier to do things like add
+ * serializations.
+ *
+ * <p>This class also provides constants for all the configurations possible on
+ * a Storm cluster and Storm topology. Each constant is paired with a schema
+ * that defines the validity criterion of the corresponding field. Default
+ * values for these configs can be found in defaults.yaml.</p>
+ *
+ * <p>Note that you may put other configurations in any of the configs. Storm
+ * will ignore anything it doesn't recognize, but your topologies are free to make
+ * use of them by reading them in the prepare method of Bolts or the open method of
+ * Spouts.</p>
+ */
+public class Config extends HashMap<String, Object> {
+ /**
+ * This is part of a temporary workaround to a ZK bug, it is the 'scheme:acl' for
+ * the user Nimbus and Supervisors use to authenticate with ZK.
+ */
+ public static final String STORM_ZOOKEEPER_SUPERACL = "storm.zookeeper.superACL";
+ public static final Object STORM_ZOOKEEPER_SUPERACL_SCHEMA = String.class;
+
+ /**
+ * The transporter for communication among Storm tasks
+ */
+ public static final String STORM_MESSAGING_TRANSPORT = "storm.messaging.transport";
+ public static final Object STORM_MESSAGING_TRANSPORT_SCHEMA = String.class;
+
+ /**
+ * Netty based messaging: The buffer size for send/recv buffer
+ */
+ public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size";
+ public static final Object STORM_MESSAGING_NETTY_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible
+ */
+ public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries";
+ public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Netty based messaging: The min # of milliseconds that a peer will wait.
+ */
+ public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms";
+ public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Netty based messaging: The max # of milliseconds that a peer will wait.
+ */
+ public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms";
+ public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Netty based messaging: The # of worker threads for the server.
+ */
+ public static final String STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS = "storm.messaging.netty.server_worker_threads";
+ public static final Object STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Netty based messaging: The # of worker threads for the client.
+ */
+ public static final String STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS = "storm.messaging.netty.client_worker_threads";
+ public static final Object STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * If the Netty messaging layer is busy, the Netty client will try to batch message as more as possible up to the size of STORM_NETTY_MESSAGE_BATCH_SIZE bytes
+ */
+ public static final String STORM_NETTY_MESSAGE_BATCH_SIZE = "storm.messaging.netty.transfer.batch.size";
+ public static final Object STORM_NETTY_MESSAGE_BATCH_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * We check with this interval that whether the Netty channel is writable and try to write pending messages
+ */
+ public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms";
+ public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Netty based messaging: Is authentication required for Netty messaging from client worker process to server worker process.
+ */
+ public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication";
+ public static final Object STORM_MESSAGING_NETTY_AUTHENTICATION_SCHEMA = Boolean.class;
+
+ /**
+ * The delegate for serializing metadata, should be used for serialized objects stored in zookeeper and on disk.
+ * This is NOT used for compressing serialized tuples sent between topologies.
+ */
+ public static final String STORM_META_SERIALIZATION_DELEGATE = "storm.meta.serialization.delegate";
+ public static final Object STORM_META_SERIALIZATION_DELEGATE_SCHEMA = String.class;
+
+ /**
+ * A list of hosts of ZooKeeper servers used to manage the cluster.
+ */
+ public static final String STORM_ZOOKEEPER_SERVERS = "storm.zookeeper.servers";
+ public static final Object STORM_ZOOKEEPER_SERVERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * The port Storm will use to connect to each of the ZooKeeper servers.
+ */
+ public static final String STORM_ZOOKEEPER_PORT = "storm.zookeeper.port";
+ public static final Object STORM_ZOOKEEPER_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * A directory on the local filesystem used by Storm for any local
+ * filesystem usage it needs. The directory must exist and the Storm daemons must
+ * have permission to read/write from this location.
+ */
+ public static final String STORM_LOCAL_DIR = "storm.local.dir";
+ public static final Object STORM_LOCAL_DIR_SCHEMA = String.class;
+
+ /**
+ * A global task scheduler used to assign topologies's tasks to supervisors' wokers.
+ *
+ * If this is not set, a default system scheduler will be used.
+ */
+ public static final String STORM_SCHEDULER = "storm.scheduler";
+ public static final Object STORM_SCHEDULER_SCHEMA = String.class;
+
+ /**
+ * The mode this Storm cluster is running in. Either "distributed" or "local".
+ */
+ public static final String STORM_CLUSTER_MODE = "storm.cluster.mode";
+ public static final Object STORM_CLUSTER_MODE_SCHEMA = String.class;
+
+ /**
+ * The hostname the supervisors/workers should report to nimbus. If unset, Storm will
+ * get the hostname to report by calling <code>InetAddress.getLocalHost().getCanonicalHostName()</code>.
+ *
+ * You should set this config when you dont have a DNS which supervisors/workers
+ * can utilize to find each other based on hostname got from calls to
+ * <code>InetAddress.getLocalHost().getCanonicalHostName()</code>.
+ */
+ public static final String STORM_LOCAL_HOSTNAME = "storm.local.hostname";
+ public static final Object STORM_LOCAL_HOSTNAME_SCHEMA = String.class;
+
+ /**
+ * The plugin that will convert a principal to a local user.
+ */
+ public static final String STORM_PRINCIPAL_TO_LOCAL_PLUGIN = "storm.principal.tolocal";
+ public static final Object STORM_PRINCIPAL_TO_LOCAL_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * The plugin that will provide user groups service
+ */
+ public static final String STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN = "storm.group.mapping.service";
+ public static final Object STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * Max no.of seconds group mapping service will cache user groups
+ */
+ public static final String STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS = "storm.group.mapping.service.cache.duration.secs";
+ public static final Object STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS_SCHEMA = Number.class;
+
+ /**
+ * The default transport plug-in for Thrift client/server communication
+ */
+ public static final String STORM_THRIFT_TRANSPORT_PLUGIN = "storm.thrift.transport";
+ public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * The serializer class for ListDelegate (tuple payload).
+ * The default serializer will be ListDelegateSerializer
+ */
+ public static final String TOPOLOGY_TUPLE_SERIALIZER = "topology.tuple.serializer";
+ public static final Object TOPOLOGY_TUPLE_SERIALIZER_SCHEMA = String.class;
+
+ /**
+ * Try to serialize all tuples, even for local transfers. This should only be used
+ * for testing, as a sanity check that all of your tuples are setup properly.
+ */
+ public static final String TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE = "topology.testing.always.try.serialize";
+ public static final Object TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE_SCHEMA = Boolean.class;
+
+ /**
+ * Whether or not to use ZeroMQ for messaging in local mode. If this is set
+ * to false, then Storm will use a pure-Java messaging system. The purpose
+ * of this flag is to make it easy to run Storm in local mode by eliminating
+ * the need for native dependencies, which can be difficult to install.
+ *
+ * Defaults to false.
+ */
+ public static final String STORM_LOCAL_MODE_ZMQ = "storm.local.mode.zmq";
+ public static final Object STORM_LOCAL_MODE_ZMQ_SCHEMA = Boolean.class;
+
+ /**
+ * The root location at which Storm stores data in ZooKeeper.
+ */
+ public static final String STORM_ZOOKEEPER_ROOT = "storm.zookeeper.root";
+ public static final Object STORM_ZOOKEEPER_ROOT_SCHEMA = String.class;
+
+ /**
+ * The session timeout for clients to ZooKeeper.
+ */
+ public static final String STORM_ZOOKEEPER_SESSION_TIMEOUT = "storm.zookeeper.session.timeout";
+ public static final Object STORM_ZOOKEEPER_SESSION_TIMEOUT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The connection timeout for clients to ZooKeeper.
+ */
+ public static final String STORM_ZOOKEEPER_CONNECTION_TIMEOUT = "storm.zookeeper.connection.timeout";
+ public static final Object STORM_ZOOKEEPER_CONNECTION_TIMEOUT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The number of times to retry a Zookeeper operation.
+ */
+ public static final String STORM_ZOOKEEPER_RETRY_TIMES="storm.zookeeper.retry.times";
+ public static final Object STORM_ZOOKEEPER_RETRY_TIMES_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The interval between retries of a Zookeeper operation.
+ */
+ public static final String STORM_ZOOKEEPER_RETRY_INTERVAL="storm.zookeeper.retry.interval";
+ public static final Object STORM_ZOOKEEPER_RETRY_INTERVAL_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The ceiling of the interval between retries of a Zookeeper operation.
+ */
+ public static final String STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING="storm.zookeeper.retry.intervalceiling.millis";
+ public static final Object STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The cluster Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication.
+ */
+ public static final String STORM_ZOOKEEPER_AUTH_SCHEME="storm.zookeeper.auth.scheme";
+ public static final Object STORM_ZOOKEEPER_AUTH_SCHEME_SCHEMA = String.class;
+
+ /**
+ * A string representing the payload for cluster Zookeeper authentication.
+ * It gets serialized using UTF-8 encoding during authentication.
+ * Note that if this is set to something with a secret (as when using
+ * digest authentication) then it should only be set in the
+ * storm-cluster-auth.yaml file.
+ * This file storm-cluster-auth.yaml should then be protected with
+ * appropriate permissions that deny access from workers.
+ */
+ public static final String STORM_ZOOKEEPER_AUTH_PAYLOAD="storm.zookeeper.auth.payload";
+ public static final Object STORM_ZOOKEEPER_AUTH_PAYLOAD_SCHEMA = String.class;
+
+ /**
+ * The topology Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication.
+ */
+ public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME="storm.zookeeper.topology.auth.scheme";
+ public static final Object STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME_SCHEMA = String.class;
+
+ /**
+ * A string representing the payload for topology Zookeeper authentication. It gets serialized using UTF-8 encoding during authentication.
+ */
+ public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD="storm.zookeeper.topology.auth.payload";
+ public static final Object STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD_SCHEMA = String.class;
+
+ /**
+ * The id assigned to a running topology. The id is the storm name with a unique nonce appended.
+ */
+ public static final String STORM_ID = "storm.id";
+ public static final Object STORM_ID_SCHEMA = String.class;
+
+ /**
+ * The number of times to retry a Nimbus operation.
+ */
+ public static final String STORM_NIMBUS_RETRY_TIMES="storm.nimbus.retry.times";
+ public static final Object STORM_NIMBUS_RETRY_TIMES_SCHEMA = Number.class;
+
+ /**
+ * The starting interval between exponential backoff retries of a Nimbus operation.
+ */
+ public static final String STORM_NIMBUS_RETRY_INTERVAL="storm.nimbus.retry.interval.millis";
+ public static final Object STORM_NIMBUS_RETRY_INTERVAL_SCHEMA = Number.class;
+
+ /**
+ * The ceiling of the interval between retries of a client connect to Nimbus operation.
+ */
+ public static final String STORM_NIMBUS_RETRY_INTERVAL_CEILING="storm.nimbus.retry.intervalceiling.millis";
+ public static final Object STORM_NIMBUS_RETRY_INTERVAL_CEILING_SCHEMA = Number.class;
+
+ /**
+ * The host that the master server is running on.
+ */
+ public static final String NIMBUS_HOST = "nimbus.host";
+ public static final Object NIMBUS_HOST_SCHEMA = String.class;
+
+ /**
+ * The Nimbus transport plug-in for Thrift client/server communication
+ */
+ public static final String NIMBUS_THRIFT_TRANSPORT_PLUGIN = "nimbus.thrift.transport";
+ public static final Object NIMBUS_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * Which port the Thrift interface of Nimbus should run on. Clients should
+ * connect to this port to upload jars and submit topologies.
+ */
+ public static final String NIMBUS_THRIFT_PORT = "nimbus.thrift.port";
+ public static final Object NIMBUS_THRIFT_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The number of threads that should be used by the nimbus thrift server.
+ */
+ public static final String NIMBUS_THRIFT_THREADS = "nimbus.thrift.threads";
+ public static final Object NIMBUS_THRIFT_THREADS_SCHEMA = Number.class;
+
+ /**
+ * A list of users that are cluster admins and can run any command. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String NIMBUS_ADMINS = "nimbus.admins";
+ public static final Object NIMBUS_ADMINS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * A list of users that run the supervisors and should be authorized to interact with
+ * nimbus as a supervisor would. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String NIMBUS_SUPERVISOR_USERS = "nimbus.supervisor.users";
+ public static final Object NIMBUS_SUPERVISOR_USERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * The maximum buffer size thrift should use when reading messages.
+ */
+ public static final String NIMBUS_THRIFT_MAX_BUFFER_SIZE = "nimbus.thrift.max_buffer_size";
+ public static final Object NIMBUS_THRIFT_MAX_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * This parameter is used by the storm-deploy project to configure the
+ * jvm options for the nimbus daemon.
+ */
+ public static final String NIMBUS_CHILDOPTS = "nimbus.childopts";
+ public static final Object NIMBUS_CHILDOPTS_SCHEMA = String.class;
+
+
+ /**
+ * How long without heartbeating a task can go before nimbus will consider the
+ * task dead and reassign it to another location.
+ */
+ public static final String NIMBUS_TASK_TIMEOUT_SECS = "nimbus.task.timeout.secs";
+ public static final Object NIMBUS_TASK_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * How often nimbus should wake up to check heartbeats and do reassignments. Note
+ * that if a machine ever goes down Nimbus will immediately wake up and take action.
+ * This parameter is for checking for failures when there's no explicit event like that
+ * occuring.
+ */
+ public static final String NIMBUS_MONITOR_FREQ_SECS = "nimbus.monitor.freq.secs";
+ public static final Object NIMBUS_MONITOR_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How often nimbus should wake the cleanup thread to clean the inbox.
+ * @see NIMBUS_INBOX_JAR_EXPIRATION_SECS
+ */
+ public static final String NIMBUS_CLEANUP_INBOX_FREQ_SECS = "nimbus.cleanup.inbox.freq.secs";
+ public static final Object NIMBUS_CLEANUP_INBOX_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The length of time a jar file lives in the inbox before being deleted by the cleanup thread.
+ *
+ * Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS.
+ * Note that the time it takes to delete an inbox jar file is going to be somewhat more than
+ * NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS
+ * is set to).
+ * @see NIMBUS_CLEANUP_FREQ_SECS
+ */
+ public static final String NIMBUS_INBOX_JAR_EXPIRATION_SECS = "nimbus.inbox.jar.expiration.secs";
+ public static final Object NIMBUS_INBOX_JAR_EXPIRATION_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How long before a supervisor can go without heartbeating before nimbus considers it dead
+ * and stops assigning new work to it.
+ */
+ public static final String NIMBUS_SUPERVISOR_TIMEOUT_SECS = "nimbus.supervisor.timeout.secs";
+ public static final Object NIMBUS_SUPERVISOR_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * A special timeout used when a task is initially launched. During launch, this is the timeout
+ * used until the first heartbeat, overriding nimbus.task.timeout.secs.
+ *
+ * <p>A separate timeout exists for launch because there can be quite a bit of overhead
+ * to launching new JVM's and configuring them.</p>
+ */
+ public static final String NIMBUS_TASK_LAUNCH_SECS = "nimbus.task.launch.secs";
+ public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Whether or not nimbus should reassign tasks if it detects that a task goes down.
+ * Defaults to true, and it's not recommended to change this value.
+ */
+ public static final String NIMBUS_REASSIGN = "nimbus.reassign";
+ public static final Object NIMBUS_REASSIGN_SCHEMA = Boolean.class;
+
+ /**
+ * During upload/download with the master, how long an upload or download connection is idle
+ * before nimbus considers it dead and drops the connection.
+ */
+ public static final String NIMBUS_FILE_COPY_EXPIRATION_SECS = "nimbus.file.copy.expiration.secs";
+ public static final Object NIMBUS_FILE_COPY_EXPIRATION_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * A custom class that implements ITopologyValidator that is run whenever a
+ * topology is submitted. Can be used to provide business-specific logic for
+ * whether topologies are allowed to run or not.
+ */
+ public static final String NIMBUS_TOPOLOGY_VALIDATOR = "nimbus.topology.validator";
+ public static final Object NIMBUS_TOPOLOGY_VALIDATOR_SCHEMA = String.class;
+
+ /**
+ * Class name for authorization plugin for Nimbus
+ */
+ public static final String NIMBUS_AUTHORIZER = "nimbus.authorizer";
+ public static final Object NIMBUS_AUTHORIZER_SCHEMA = String.class;
+
+ /**
+ * How often nimbus should wake up to renew credentials if needed.
+ */
+ public static final String NIMBUS_CREDENTIAL_RENEW_FREQ_SECS = "nimbus.credential.renewers.freq.secs";
+ public static final Object NIMBUS_CREDENTIAL_RENEW_FREQ_SECS_SCHEMA = Number.class;
+
+ /**
+ * A list of credential renewers that nimbus should load.
+ */
+ public static final String NIMBUS_CREDENTIAL_RENEWERS = "nimbus.credential.renewers.classes";
+ public static final Object NIMBUS_CREDENTIAL_RENEWERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * A list of plugins that nimbus should load during submit topology to populate
+ * credentials on user's behalf.
+ */
+ public static final String NIMBUS_AUTO_CRED_PLUGINS = "nimbus.autocredential.plugins.classes";
+ public static final Object NIMBUS_AUTO_CRED_PLUGINS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * Storm UI binds to this port.
+ */
+ public static final String UI_PORT = "ui.port";
+ public static final Object UI_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * HTTP UI port for log viewer
+ */
+ public static final String LOGVIEWER_PORT = "logviewer.port";
+ public static final Object LOGVIEWER_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Childopts for log viewer java process.
+ */
+ public static final String LOGVIEWER_CHILDOPTS = "logviewer.childopts";
+ public static final Object LOGVIEWER_CHILDOPTS_SCHEMA = String.class;
+
+ /**
+ * How often to clean up old log files
+ */
+ public static final String LOGVIEWER_CLEANUP_INTERVAL_SECS = "logviewer.cleanup.interval.secs";
+ public static final Object LOGVIEWER_CLEANUP_INTERVAL_SECS_SCHEMA = ConfigValidation.PositiveIntegerValidator;
+
+ /**
+ * How many minutes since a log was last modified for the log to be considered for clean-up
+ */
+ public static final String LOGVIEWER_CLEANUP_AGE_MINS = "logviewer.cleanup.age.mins";
+ public static final Object LOGVIEWER_CLEANUP_AGE_MINS_SCHEMA = ConfigValidation.PositiveIntegerValidator;
+
+ /**
+ * A list of users allowed to view logs via the Log Viewer
+ */
+ public static final String LOGS_USERS = "logs.users";
+ public static final Object LOGS_USERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * Appender name used by log viewer to determine log directory.
+ */
+ public static final String LOGVIEWER_APPENDER_NAME = "logviewer.appender.name";
+ public static final Object LOGVIEWER_APPENDER_NAME_SCHEMA = String.class;
+
+ /**
+ * Childopts for Storm UI Java process.
+ */
+ public static final String UI_CHILDOPTS = "ui.childopts";
+ public static final Object UI_CHILDOPTS_SCHEMA = String.class;
+
+ /**
+ * A class implementing javax.servlet.Filter for authenticating/filtering UI requests
+ */
+ public static final String UI_FILTER = "ui.filter";
+ public static final Object UI_FILTER_SCHEMA = String.class;
+
+ /**
+ * Initialization parameters for the javax.servlet.Filter
+ */
+ public static final String UI_FILTER_PARAMS = "ui.filter.params";
+ public static final Object UI_FILTER_PARAMS_SCHEMA = Map.class;
+
+ /**
+ * The size of the header buffer for the UI in bytes
+ */
+ public static final String UI_HEADER_BUFFER_BYTES = "ui.header.buffer.bytes";
+ public static final Object UI_HEADER_BUFFER_BYTES_SCHEMA = Number.class;
+
+ /**
+ * A list of users allowed to view topologies via the UI
+ */
+ public static final String UI_USERS = "ui.users";
+ public static final Object UI_USERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * List of DRPC servers so that the DRPCSpout knows who to talk to.
+ */
+ public static final String DRPC_SERVERS = "drpc.servers";
+ public static final Object DRPC_SERVERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * This port is used by Storm DRPC for receiving HTTP DPRC requests from clients.
+ */
+ public static final String DRPC_HTTP_PORT = "drpc.http.port";
+ public static final Object DRPC_HTTP_PORT_SCHEMA = Number.class;
+
+ /**
+ * This port is used by Storm DRPC for receiving HTTPS (SSL) DPRC requests from clients.
+ */
+ public static final String DRPC_HTTPS_PORT = "drpc.https.port";
+ public static final Object DRPC_HTTPS_PORT_SCHEMA = Number.class;
+
+ /**
+ * Path to the keystore used by Storm DRPC for setting up HTTPS (SSL).
+ */
+ public static final String DRPC_HTTPS_KEYSTORE_PATH = "drpc.https.keystore.path";
+ public static final Object DRPC_HTTPS_KEYSTORE_PATH_SCHEMA = String.class;
+
+ /**
+ * Password to the keystore used by Storm DRPC for setting up HTTPS (SSL).
+ */
+ public static final String DRPC_HTTPS_KEYSTORE_PASSWORD = "drpc.https.keystore.password";
+ public static final Object DRPC_HTTPS_KEYSTORE_PASSWORD_SCHEMA = String.class;
+
+ /**
+ * Type of keystore used by Storm DRPC for setting up HTTPS (SSL).
+ * see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore.html for more details.
+ */
+ public static final String DRPC_HTTPS_KEYSTORE_TYPE = "drpc.https.keystore.type";
+ public static final Object DRPC_HTTPS_KEYSTORE_TYPE_SCHEMA = String.class;
+
+ /**
+ * The DRPC transport plug-in for Thrift client/server communication
+ */
+ public static final String DRPC_THRIFT_TRANSPORT_PLUGIN = "drpc.thrift.transport";
+ public static final Object DRPC_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * This port is used by Storm DRPC for receiving DPRC requests from clients.
+ */
+ public static final String DRPC_PORT = "drpc.port";
+ public static final Object DRPC_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Class name for authorization plugin for DRPC client
+ */
+ public static final String DRPC_AUTHORIZER = "drpc.authorizer";
+ public static final Object DRPC_AUTHORIZER_SCHEMA = String.class;
+
+ /**
+ * The Access Control List for the DRPC Authorizer.
+ * @see DRPCSimpleAclAuthorizer
+ */
+ public static final String DRPC_AUTHORIZER_ACL = "drpc.authorizer.acl";
+ public static final Object DRPC_AUTHORIZER_ACL_SCHEMA = Map.class;
+
+ /**
+ * File name of the DRPC Authorizer ACL.
+ * @see DRPCSimpleAclAuthorizer
+ */
+ public static final String DRPC_AUTHORIZER_ACL_FILENAME = "drpc.authorizer.acl.filename";
+ public static final Object DRPC_AUTHORIZER_ACL_FILENAME_SCHEMA = String.class;
+
+ /**
+ * Whether the DRPCSimpleAclAuthorizer should deny requests for operations
+ * involving functions that have no explicit ACL entry. When set to false
+ * (the default) DRPC functions that have no entry in the ACL will be
+ * permitted, which is appropriate for a development environment. When set
+ * to true, explicit ACL entries are required for every DRPC function, and
+ * any request for functions will be denied.
+ * @see DRPCSimpleAclAuthorizer
+ */
+ public static final String DRPC_AUTHORIZER_ACL_STRICT = "drpc.authorizer.acl.strict";
+ public static final Object DRPC_AUTHORIZER_ACL_STRICT_SCHEMA = Boolean.class;
+
+ /**
+ * DRPC thrift server worker threads
+ */
+ public static final String DRPC_WORKER_THREADS = "drpc.worker.threads";
+ public static final Object DRPC_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The maximum buffer size thrift should use when reading messages for DRPC.
+ */
+ public static final String DRPC_MAX_BUFFER_SIZE = "drpc.max_buffer_size";
+ public static final Object DRPC_MAX_BUFFER_SIZE_SCHEMA = Number.class;
+
+ /**
+ * DRPC thrift server queue size
+ */
+ public static final String DRPC_QUEUE_SIZE = "drpc.queue.size";
+ public static final Object DRPC_QUEUE_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The DRPC invocations transport plug-in for Thrift client/server communication
+ */
+ public static final String DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN = "drpc.invocations.thrift.transport";
+ public static final Object DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.
+ */
+ public static final String DRPC_INVOCATIONS_PORT = "drpc.invocations.port";
+ public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * DRPC invocations thrift server worker threads
+ */
+ public static final String DRPC_INVOCATIONS_THREADS = "drpc.invocations.threads";
+ public static final Object DRPC_INVOCATIONS_THREADS_SCHEMA = Number.class;
+
+ /**
+ * The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also
+ * timeout based on the socket timeout on the DRPC client, and separately based on the topology message
+ * timeout for the topology implementing the DRPC function.
+ */
+ public static final String DRPC_REQUEST_TIMEOUT_SECS = "drpc.request.timeout.secs";
+ public static final Object DRPC_REQUEST_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Childopts for Storm DRPC Java process.
+ */
+ public static final String DRPC_CHILDOPTS = "drpc.childopts";
+ public static final Object DRPC_CHILDOPTS_SCHEMA = String.class;
+
+ /**
+ * Class name of the HTTP credentials plugin for the UI.
+ */
+ public static final String UI_HTTP_CREDS_PLUGIN = "ui.http.creds.plugin";
+ public static final Object UI_HTTP_CREDS_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * Class name of the HTTP credentials plugin for DRPC.
+ */
+ public static final String DRPC_HTTP_CREDS_PLUGIN = "drpc.http.creds.plugin";
+ public static final Object DRPC_HTTP_CREDS_PLUGIN_SCHEMA = String.class;
+
+ /**
+ * the metadata configured on the supervisor
+ */
+ public static final String SUPERVISOR_SCHEDULER_META = "supervisor.scheduler.meta";
+ public static final Object SUPERVISOR_SCHEDULER_META_SCHEMA = Map.class;
+ /**
+ * A list of ports that can run workers on this supervisor. Each worker uses one port, and
+ * the supervisor will only run one worker per port. Use this configuration to tune
+ * how many workers run on each machine.
+ */
+ public static final String SUPERVISOR_SLOTS_PORTS = "supervisor.slots.ports";
+ public static final Object SUPERVISOR_SLOTS_PORTS_SCHEMA = ConfigValidation.IntegersValidator;
+
+ /**
+ * A number representing the maximum number of workers any single topology can acquire.
+ */
+ public static final String NIMBUS_SLOTS_PER_TOPOLOGY = "nimbus.slots.perTopology";
+ public static final Object NIMBUS_SLOTS_PER_TOPOLOGY_SCHEMA = Number.class;
+
+ /**
+ * A class implementing javax.servlet.Filter for DRPC HTTP requests
+ */
+ public static final String DRPC_HTTP_FILTER = "drpc.http.filter";
+ public static final Object DRPC_HTTP_FILTER_SCHEMA = String.class;
+
+ /**
+ * Initialization parameters for the javax.servlet.Filter of the DRPC HTTP
+ * service
+ */
+ public static final String DRPC_HTTP_FILTER_PARAMS = "drpc.http.filter.params";
+ public static final Object DRPC_HTTP_FILTER_PARAMS_SCHEMA = Map.class;
+
+ /**
+ * A number representing the maximum number of executors any single topology can acquire.
+ */
+ public static final String NIMBUS_EXECUTORS_PER_TOPOLOGY = "nimbus.executors.perTopology";
+ public static final Object NIMBUS_EXECUTORS_PER_TOPOLOGY_SCHEMA = Number.class;
+
+ /**
+ * This parameter is used by the storm-deploy project to configure the
+ * jvm options for the supervisor daemon.
+ */
+ public static final String SUPERVISOR_CHILDOPTS = "supervisor.childopts";
+ public static final Object SUPERVISOR_CHILDOPTS_SCHEMA = String.class;
+
+ /**
+ * How long a worker can go without heartbeating before the supervisor tries to
+ * restart the worker process.
+ */
+ public static final String SUPERVISOR_WORKER_TIMEOUT_SECS = "supervisor.worker.timeout.secs";
+ public static final Object SUPERVISOR_WORKER_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How long a worker can go without heartbeating during the initial launch before
+ * the supervisor tries to restart the worker process. This value override
+ * supervisor.worker.timeout.secs during launch because there is additional
+ * overhead to starting and configuring the JVM on launch.
+ */
+ public static final String SUPERVISOR_WORKER_START_TIMEOUT_SECS = "supervisor.worker.start.timeout.secs";
+ public static final Object SUPERVISOR_WORKER_START_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Whether or not the supervisor should launch workers assigned to it. Defaults
+ * to true -- and you should probably never change this value. This configuration
+ * is used in the Storm unit tests.
+ */
+ public static final String SUPERVISOR_ENABLE = "supervisor.enable";
+ public static final Object SUPERVISOR_ENABLE_SCHEMA = Boolean.class;
+
+ /**
+ * how often the supervisor sends a heartbeat to the master.
+ */
+ public static final String SUPERVISOR_HEARTBEAT_FREQUENCY_SECS = "supervisor.heartbeat.frequency.secs";
+ public static final Object SUPERVISOR_HEARTBEAT_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * How often the supervisor checks the worker heartbeats to see if any of them
+ * need to be restarted.
+ */
+ public static final String SUPERVISOR_MONITOR_FREQUENCY_SECS = "supervisor.monitor.frequency.secs";
+ public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Should the supervior try to run the worker as the lauching user or not. Defaults to false.
+ */
+ public static final String SUPERVISOR_RUN_WORKER_AS_USER = "supervisor.run.worker.as.user";
+ public static final Object SUPERVISOR_RUN_WORKER_AS_USER_SCHEMA = Boolean.class;
+
+ /**
+ * Full path to the worker-laucher executable that will be used to lauch workers when
+ * SUPERVISOR_RUN_WORKER_AS_USER is set to true.
+ */
+ public static final String SUPERVISOR_WORKER_LAUNCHER = "supervisor.worker.launcher";
+ public static final Object SUPERVISOR_WORKER_LAUNCHER_SCHEMA = String.class;
+
+ /**
+ * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%TOPOLOGY-ID%"
+ * and "%WORKER-PORT%" substrings are replaced with:
+ * %ID% -> port (for backward compatibility),
+ * %WORKER-ID% -> worker-id,
+ * %TOPOLOGY-ID% -> topology-id,
+ * %WORKER-PORT% -> port.
+ */
+ public static final String WORKER_CHILDOPTS = "worker.childopts";
+ public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * The jvm opts provided to workers launched by this supervisor for GC. All "%ID%" substrings are replaced
+ * with an identifier for this worker. Because the JVM complains about multiple GC opts the topology
+ * can override this default value by setting topology.worker.gc.childopts.
+ */
+ public static final String WORKER_GC_CHILDOPTS = "worker.gc.childopts";
+ public static final Object WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * control how many worker receiver threads we need per worker
+ */
+ public static final String WORKER_RECEIVER_THREAD_COUNT = "topology.worker.receiver.thread.count";
+ public static final Object WORKER_RECEIVER_THREAD_COUNT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How often this worker should heartbeat to the supervisor.
+ */
+ public static final String WORKER_HEARTBEAT_FREQUENCY_SECS = "worker.heartbeat.frequency.secs";
+ public static final Object WORKER_HEARTBEAT_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How often a task should heartbeat its status to the master.
+ */
+ public static final String TASK_HEARTBEAT_FREQUENCY_SECS = "task.heartbeat.frequency.secs";
+ public static final Object TASK_HEARTBEAT_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * How often a task should sync its connections with other tasks (if a task is
+ * reassigned, the other tasks sending messages to it need to refresh their connections).
+ * In general though, when a reassignment happens other tasks will be notified
+ * almost immediately. This configuration is here just in case that notification doesn't
+ * come through.
+ */
+ public static final String TASK_REFRESH_POLL_SECS = "task.refresh.poll.secs";
+ public static final Object TASK_REFRESH_POLL_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * How often a task should sync credentials, worst case.
+ */
+ public static final String TASK_CREDENTIALS_POLL_SECS = "task.credentials.poll.secs";
+ public static final Object TASK_CREDENTIALS_POLL_SECS_SCHEMA = Number.class;
+
+
+ /**
+ * A list of users that are allowed to interact with the topology. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String TOPOLOGY_USERS = "topology.users";
+ public static final Object TOPOLOGY_USERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * A list of groups that are allowed to interact with the topology. To use this set
+ * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
+ */
+ public static final String TOPOLOGY_GROUPS = "topology.groups";
+ public static final Object TOPOLOGY_GROUPS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * True if Storm should timeout messages or not. Defaults to true. This is meant to be used
+ * in unit tests to prevent tuples from being accidentally timed out during the test.
+ */
+ public static final String TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS = "topology.enable.message.timeouts";
+ public static final Object TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS_SCHEMA = Boolean.class;
+
+ /**
+ * When set to true, Storm will log every message that's emitted.
+ */
+ public static final String TOPOLOGY_DEBUG = "topology.debug";
+ public static final Object TOPOLOGY_DEBUG_SCHEMA = Boolean.class;
+
+ /**
+ * The serializer for communication between shell components and non-JVM
+ * processes
+ */
+ public static final String TOPOLOGY_MULTILANG_SERIALIZER = "topology.multilang.serializer";
+ public static final Object TOPOLOGY_MULTILANG_SERIALIZER_SCHEMA = String.class;
+
+ /**
+ * How many processes should be spawned around the cluster to execute this
+ * topology. Each process will execute some number of tasks as threads within
+ * them. This parameter should be used in conjunction with the parallelism hints
+ * on each component in the topology to tune the performance of a topology.
+ */
+ public static final String TOPOLOGY_WORKERS = "topology.workers";
+ public static final Object TOPOLOGY_WORKERS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How many instances to create for a spout/bolt. A task runs on a thread with zero or more
+ * other tasks for the same spout/bolt. The number of tasks for a spout/bolt is always
+ * the same throughout the lifetime of a topology, but the number of executors (threads) for
+ * a spout/bolt can change over time. This allows a topology to scale to more or less resources
+ * without redeploying the topology or violating the constraints of Storm (such as a fields grouping
+ * guaranteeing that the same value goes to the same task).
+ */
+ public static final String TOPOLOGY_TASKS = "topology.tasks";
+ public static final Object TOPOLOGY_TASKS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How many executors to spawn for ackers.
+ *
+ * <p>If this is set to 0, then Storm will immediately ack tuples as soon
+ * as they come off the spout, effectively disabling reliability.</p>
+ */
+ public static final String TOPOLOGY_ACKER_EXECUTORS = "topology.acker.executors";
+ public static final Object TOPOLOGY_ACKER_EXECUTORS_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * The maximum amount of time given to the topology to fully process a message
+ * emitted by a spout. If the message is not acked within this time frame, Storm
+ * will fail the message on the spout. Some spouts implementations will then replay
+ * the message at a later time.
+ */
+ public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs";
+ public static final Object TOPOLOGY_MESSAGE_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * A list of serialization registrations for Kryo ( http://code.google.com/p/kryo/ ),
+ * the underlying serialization framework for Storm. A serialization can either
+ * be the name of a class (in which case Kryo will automatically create a serializer for the class
+ * that saves all the object's fields), or an implementation of com.esotericsoftware.kryo.Serializer.
+ *
+ * See Kryo's documentation for more information about writing custom serializers.
+ */
+ public static final String TOPOLOGY_KRYO_REGISTER = "topology.kryo.register";
+ public static final Object TOPOLOGY_KRYO_REGISTER_SCHEMA = ConfigValidation.KryoRegValidator;
+
+ /**
+ * A list of classes that customize storm's kryo instance during start-up.
+ * Each listed class name must implement IKryoDecorator. During start-up the
+ * listed class is instantiated with 0 arguments, then its 'decorate' method
+ * is called with storm's kryo instance as the only argument.
+ */
+ public static final String TOPOLOGY_KRYO_DECORATORS = "topology.kryo.decorators";
+ public static final Object TOPOLOGY_KRYO_DECORATORS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * Class that specifies how to create a Kryo instance for serialization. Storm will then apply
+ * topology.kryo.register and topology.kryo.decorators on top of this. The default implementation
+ * implements topology.fall.back.on.java.serialization and turns references off.
+ */
+ public static final String TOPOLOGY_KRYO_FACTORY = "topology.kryo.factory";
+ public static final Object TOPOLOGY_KRYO_FACTORY_SCHEMA = String.class;
+
+
+ /**
+ * Whether or not Storm should skip the loading of kryo registrations for which it
+ * does not know the class or have the serializer implementation. Otherwise, the task will
+ * fail to load and will throw an error at runtime. The use case of this is if you want to
+ * declare your serializations on the storm.yaml files on the cluster rather than every single
+ * time you submit a topology. Different applications may use different serializations and so
+ * a single application may not have the code for the other serializers used by other apps.
+ * By setting this config to true, Storm will ignore that it doesn't have those other serializations
+ * rather than throw an error.
+ */
+ public static final String TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS= "topology.skip.missing.kryo.registrations";
+ public static final Object TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS_SCHEMA = Boolean.class;
+
+ /*
+ * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config format).
+ * Each listed class will be routed all the metrics data generated by the storm metrics API.
+ * Each listed class maps 1:1 to a system bolt named __metrics_ClassName#N, and it's parallelism is configurable.
+ */
+ public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = "topology.metrics.consumer.register";
+ public static final Object TOPOLOGY_METRICS_CONSUMER_REGISTER_SCHEMA = ConfigValidation.MapsValidator;
+
+
+ /**
+ * The maximum parallelism allowed for a component in this topology. This configuration is
+ * typically used in testing to limit the number of threads spawned in local mode.
+ */
+ public static final String TOPOLOGY_MAX_TASK_PARALLELISM="topology.max.task.parallelism";
+ public static final Object TOPOLOGY_MAX_TASK_PARALLELISM_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * The maximum number of tuples that can be pending on a spout task at any given time.
+ * This config applies to individual tasks, not to spouts or topologies as a whole.
+ *
+ * A pending tuple is one that has been emitted from a spout but has not been acked or failed yet.
+ * Note that this config parameter has no effect for unreliable spouts that don't tag
+ * their tuples with a message id.
+ */
+ public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending";
+ public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * A class that implements a strategy for what to do when a spout needs to wait. Waiting is
+ * triggered in one of two conditions:
+ *
+ * 1. nextTuple emits no tuples
+ * 2. The spout has hit maxSpoutPending and can't emit any more tuples
+ */
+ public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy";
+ public static final Object TOPOLOGY_SPOUT_WAIT_STRATEGY_SCHEMA = String.class;
+
+ /**
+ * The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.
+ */
+ public static final String TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS="topology.sleep.spout.wait.strategy.time.ms";
+ public static final Object TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The maximum amount of time a component gives a source of state to synchronize before it requests
+ * synchronization again.
+ */
+ public static final String TOPOLOGY_STATE_SYNCHRONIZATION_TIMEOUT_SECS="topology.state.synchronization.timeout.secs";
+ public static final Object TOPOLOGY_STATE_SYNCHRONIZATION_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The percentage of tuples to sample to produce stats for a task.
+ */
+ public static final String TOPOLOGY_STATS_SAMPLE_RATE="topology.stats.sample.rate";
+ public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = ConfigValidation.DoubleValidator;
+
+ /**
+ * The time period that builtin metrics data in bucketed into.
+ */
+ public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS="topology.builtin.metrics.bucket.size.secs";
+ public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Whether or not to use Java serialization in a topology.
+ */
+ public static final String TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION="topology.fall.back.on.java.serialization";
+ public static final Object TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_SCHEMA = Boolean.class;
+
+ /**
+ * Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.
+ */
+ public static final String TOPOLOGY_WORKER_CHILDOPTS="topology.worker.childopts";
+ public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * Topology-specific options GC for the worker child process. This overrides WORKER_GC_CHILDOPTS.
+ */
+ public static final String TOPOLOGY_WORKER_GC_CHILDOPTS="topology.worker.gc.childopts";
+ public static final Object TOPOLOGY_WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * Topology-specific classpath for the worker child process. This is combined to the usual classpath.
+ */
+ public static final String TOPOLOGY_CLASSPATH="topology.classpath";
+ public static final Object TOPOLOGY_CLASSPATH_SCHEMA = ConfigValidation.StringOrStringListValidator;
+
+ /**
+ * Topology-specific environment variables for the worker child process.
+ * This is added to the existing environment (that of the supervisor)
+ */
+ public static final String TOPOLOGY_ENVIRONMENT="topology.environment";
+ public static final Object TOPOLOGY_ENVIRONMENT_SCHEMA = Map.class;
+
+ /*
+ * Topology-specific option to disable/enable bolt's outgoing overflow buffer.
+ * Enabling this option ensures that the bolt can always clear the incoming messages,
+ * preventing live-lock for the topology with cyclic flow.
+ * The overflow buffer can fill degrading the performance gradually,
+ * eventually running out of memory.
+ */
+ public static final String TOPOLOGY_BOLTS_OUTGOING_OVERFLOW_BUFFER_ENABLE="topology.bolts.outgoing.overflow.buffer.enable";
+ public static final Object TOPOLOGY_BOLTS_OUTGOING_OVERFLOW_BUFFER_ENABLE_SCHEMA = Boolean.class;
+
+ /**
+ * This config is available for TransactionalSpouts, and contains the id ( a String) for
+ * the transactional topology. This id is used to store the state of the transactional
+ * topology in Zookeeper.
+ */
+ public static final String TOPOLOGY_TRANSACTIONAL_ID="topology.transactional.id";
+ public static final Object TOPOLOGY_TRANSACTIONAL_ID_SCHEMA = String.class;
+
+ /**
+ * A list of task hooks that are automatically added to every spout and bolt in the topology. An example
+ * of when you'd do this is to add a hook that integrates with your internal
+ * monitoring system. These hooks are instantiated using the zero-arg constructor.
+ */
+ public static final String TOPOLOGY_AUTO_TASK_HOOKS="topology.auto.task.hooks";
+ public static final Object TOPOLOGY_AUTO_TASK_HOOKS_SCHEMA = ConfigValidation.StringsValidator;
+
+
+ /**
+ * The size of the Disruptor receive queue for each executor. Must be a power of 2.
+ */
+ public static final String TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE="topology.executor.receive.buffer.size";
+ public static final Object TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
+
+ /**
+ * The maximum number of messages to batch from the thread receiving off the network to the
+ * executor queues. Must be a power of 2.
+ */
+ public static final String TOPOLOGY_RECEIVER_BUFFER_SIZE="topology.receiver.buffer.size";
+ public static final Object TOPOLOGY_RECEIVER_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
+
+ /**
+ * The size of the Disruptor send queue for each executor. Must be a power of 2.
+ */
+ public static final String TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE="topology.executor.send.buffer.size";
+ public static final Object TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
+
+ /**
+ * The size of the Disruptor transfer queue for each worker.
+ */
+ public static final String TOPOLOGY_TRANSFER_BUFFER_SIZE="topology.transfer.buffer.size";
+ public static final Object TOPOLOGY_TRANSFER_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How often a tick tuple from the "__system" component and "__tick" stream should be sent
+ * to tasks. Meant to be used as a component-specific configuration.
+ */
+ public static final String TOPOLOGY_TICK_TUPLE_FREQ_SECS="topology.tick.tuple.freq.secs";
+ public static final Object TOPOLOGY_TICK_TUPLE_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * Configure the wait strategy used for internal queuing. Can be used to tradeoff latency
+ * vs. throughput
+ */
+ public static final String TOPOLOGY_DISRUPTOR_WAIT_STRATEGY="topology.disruptor.wait.strategy";
+ public static final Object TOPOLOGY_DISRUPTOR_WAIT_STRATEGY_SCHEMA = String.class;
+
+ /**
+ * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed
+ * via the TopologyContext.
+ */
+ public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE="topology.worker.shared.thread.pool.size";
+ public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
+ * an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
+ * reported to Zookeeper per task for every 10 second interval of time.
+ */
+ public static final String TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS="topology.error.throttle.interval.secs";
+ public static final Object TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * See doc for TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS
+ */
+ public static final String TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL="topology.max.error.report.per.interval";
+ public static final Object TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL_SCHEMA = ConfigValidation.IntegerValidator;
+
+
+ /**
+ * How often a batch can be emitted in a Trident topology.
+ */
+ public static final String TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS="topology.trident.batch.emit.interval.millis";
+ public static final Object TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * Name of the topology. This config is automatically set by Storm when the topology is submitted.
+ */
+ public final static String TOPOLOGY_NAME="topology.name";
+ public static final Object TOPOLOGY_NAME_SCHEMA = String.class;
+
+ /**
+ * The principal who submitted a topology
+ */
+ public final static String TOPOLOGY_SUBMITTER_PRINCIPAL = "topology.submitter.principal";
+ public static final Object TOPOLOGY_SUBMITTER_PRINCIPAL_SCHEMA = String.class;
+
+ /**
+ * The local user name of the user who submitted a topology.
+ */
+ public static final String TOPOLOGY_SUBMITTER_USER = "topology.submitter.user";
+ public static final Object TOPOLOGY_SUBMITTER_USER_SCHEMA = String.class;
+
+ /**
+ * Array of components that scheduler should try to place on separate hosts.
+ */
+ public static final String TOPOLOGY_SPREAD_COMPONENTS = "topology.spread.components";
+ public static final Object TOPOLOGY_SPREAD_COMPONENTS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * A list of IAutoCredentials that the topology should load and use.
+ */
+ public static final String TOPOLOGY_AUTO_CREDENTIALS = "topology.auto-credentials";
+ public static final Object TOPOLOGY_AUTO_CREDENTIALS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * Max pending tuples in one ShellBolt
+ */
+ public static final String TOPOLOGY_SHELLBOLT_MAX_PENDING="topology.shellbolt.max.pending";
+ public static final Object TOPOLOGY_SHELLBOLT_MAX_PENDING_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The root directory in ZooKeeper for metadata about TransactionalSpouts.
+ */
+ public static final String TRANSACTIONAL_ZOOKEEPER_ROOT="transactional.zookeeper.root";
+ public static final Object TRANSACTIONAL_ZOOKEEPER_ROOT_SCHEMA = String.class;
+
+ /**
+ * The list of zookeeper servers in which to keep the transactional state. If null (which is default),
+ * will use storm.zookeeper.servers
+ */
+ public static final String TRANSACTIONAL_ZOOKEEPER_SERVERS="transactional.zookeeper.servers";
+ public static final Object TRANSACTIONAL_ZOOKEEPER_SERVERS_SCHEMA = ConfigValidation.StringsValidator;
+
+ /**
+ * The port to use to connect to the transactional zookeeper servers. If null (which is default),
+ * will use storm.zookeeper.port
+ */
+ public static final String TRANSACTIONAL_ZOOKEEPER_PORT="transactional.zookeeper.port";
+ public static final Object TRANSACTIONAL_ZOOKEEPER_PORT_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The number of threads that should be used by the zeromq context in each worker process.
+ */
+ public static final String ZMQ_THREADS = "zmq.threads";
+ public static final Object ZMQ_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * How long a connection should retry sending messages to a target host when
+ * the connection is closed. This is an advanced configuration and can almost
+ * certainly be ignored.
+ */
+ public static final String ZMQ_LINGER_MILLIS = "zmq.linger.millis";
+ public static final Object ZMQ_LINGER_MILLIS_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion
+ * on the networking layer.
+ */
+ public static final String ZMQ_HWM = "zmq.hwm";
+ public static final Object ZMQ_HWM_SCHEMA = ConfigValidation.IntegerValidator;
+
+ /**
+ * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers)
+ * for the java.library.path value. java.library.path tells the JVM where
+ * to look for native libraries. It is necessary to set this config correctly since
+ * Storm uses the ZeroMQ and JZMQ native libs.
+ */
+ public static final String JAVA_LIBRARY_PATH = "java.library.path";
+ public static final Object JAVA_LIBRARY_PATH_SCHEMA = String.class;
+
+ /**
+ * The path to use as the zookeeper dir when running a zookeeper server via
+ * "storm dev-zookeeper". This zookeeper instance is only intended for development;
+ * it is not a production grade zookeeper setup.
+ */
+ public static final String DEV_ZOOKEEPER_PATH = "dev.zookeeper.path";
+ public static final Object DEV_ZOOKEEPER_PATH_SCHEMA = String.class;
+
+ /**
+ * A map from topology name to the number of machines that should be dedicated for that topology. Set storm.scheduler
+ * to backtype.storm.scheduler.IsolationScheduler to make use of the isolation scheduler.
+ */
+ public static final String ISOLATION_SCHEDULER_MACHINES = "isolation.scheduler.machines";
+ public static final Object ISOLATION_SCHEDULER_MACHINES_SCHEMA = ConfigValidation.MapOfStringToNumberValidator;
+
+ /**
+ * A map from the user name to the number of machines that should that user is allowed to use. Set storm.scheduler
+ * to backtype.storm.scheduler.multitenant.MultitenantScheduler
+ */
+ public static final String MULTITENANT_SCHEDULER_USER_POOLS = "multitenant.scheduler.user.pools";
+ public static final Object MULTITENANT_SCHEDULER_USER_POOLS_SCHEMA = ConfigValidation.MapOfStringToNumberValidator;
+
+ /**
+ * The number of machines that should be used by this topology to isolate it from all others. Set storm.scheduler
+ * to backtype.storm.scheduler.multitenant.MultitenantScheduler
+ */
+ public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines";
+ public static final Object TOPOLOGY_ISOLATED_MACHINES_SCHEMA = Number.class;
+
+ /**
+ * HDFS information, used to get the delegation token on behalf of the topology
+ * submitter user and renew the tokens. see {@link backtype.storm.security.auth.hadoop.AutoHDFS}
+ * kerberos principal name with realm should be provided.
+ */
+ public static final Object TOPOLOGY_HDFS_PRINCIPAL = "topology.hdfs.user";
+ public static final Object TOPOLOGY_HDFS_PRINCIPAL_SCHEMA = String.class;
+
+ /**
+ * The HDFS URI to be used by AutoHDFS.java to grab the delegation token on topology
+ * submitter user's behalf by the nimbus. If this is not provided the default URI provided
+ * in the hdfs configuration files will be used.
+ */
+ public static final Object TOPOLOGY_HDFS_URI = "topology.hdfs.uri";
+ public static final Object TOPOLOGY_HDFS_URI_SCHEMA = String.class;
+
+ public static void setClasspath(Map conf, String cp) {
+ conf.put(Config.TOPOLOGY_CLASSPATH, cp);
+ }
+
+ public void setClasspath(String cp) {
+ setClasspath(this, cp);
+ }
+
+ public static void setEnvironment(Map conf, Map env) {
+ conf.put(Config.TOPOLOGY_ENVIRONMENT, env);
+ }
+
+ public void setEnvironment(Map env) {
+ setEnvironment(this, env);
+ }
+
+ public static void setDebug(Map conf, boolean isOn) {
+ conf.put(Config.TOPOLOGY_DEBUG, isOn);
+ }
+
+ public void setDebug(boolean isOn) {
+ setDebug(this, isOn);
+ }
+
+ public static void setNumWorkers(Map conf, int workers) {
+ conf.put(Config.TOPOLOGY_WORKERS, workers);
+ }
+
+ public void setNumWorkers(int workers) {
+ setNumWorkers(this, workers);
+ }
+
+ public static void setNumAckers(Map conf, int numExecutors) {
+ conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, numExecutors);
+ }
+
+ public void setNumAckers(int numExecutors) {
+ setNumAckers(this, numExecutors);
+ }
+
+ public static void setMessageTimeoutSecs(Map conf, int secs) {
+ conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, secs);
+ }
+
+ public void setMessageTimeoutSecs(int secs) {
+ setMessageTimeoutSecs(this, secs);
+ }
+
+ public static void registerSerialization(Map conf, Class klass) {
+ getRegisteredSerializations(conf).add(klass.getName());
+ }
+
+ public void registerSerialization(Class klass) {
+ registerSerialization(this, klass);
+ }
+
+ public static void registerSerialization(Map conf, Class klass, Class<? extends Serializer> serializerClass) {
+ Map<String, String> register = new HashMap<String, String>();
+ register.put(klass.getName(), serializerClass.getName());
+ getRegisteredSerializations(conf).add(register);
+ }
+
+ public void registerSerialization(Class klass, Class<? extends Serializer> serializerClass) {
+ registerSerialization(this, klass, serializerClass);
+ }
+
+ public static void registerMetricsConsumer(Map conf, Class klass, Object argument, long parallelismHint) {
+ HashMap m = new HashMap();
+ m.put("class", klass.getCanonicalName());
+ m.put("parallelism.hint", parallelismHint);
+ m.put("argument", argument);
+
+ List l = (List)conf.get(TOPOLOGY_METRICS_CONSUMER_REGISTER);
+ if (l == null) { l = new ArrayList(); }
+ l.add(m);
+ conf.put(TOPOLOGY_METRICS_CONSUMER_REGISTER, l);
+ }
+
+ public void registerMetricsConsumer(Class klass, Object argument, long parallelismHint) {
+ registerMetricsConsumer(this, klass, argument, parallelismHint);
+ }
+
+ public static void registerMetricsConsumer(Map conf, Class klass, long parallelismHint) {
+ registerMetricsConsumer(conf, klass, null, parallelismHint);
+ }
+
+ public void registerMetricsConsumer(Class klass, long parallelismHint) {
+ registerMetricsConsumer(this, klass, parallelismHint);
+ }
+
+ public static void registerMetricsConsumer(Map conf, Class klass) {
+ registerMetricsConsumer(conf, klass, null, 1L);
+ }
+
+ public void registerMetricsConsumer(Class klass) {
+ registerMetricsConsumer(this, klass);
+ }
+
+ public static void registerDecorator(Map conf, Class<? extends IKryoDecorator> klass) {
+ getRegisteredDecorators(conf).add(klass.getName());
+ }
+
+ public void registerDecorator(Class<? extends IKryoDecorator> klass) {
+ registerDecorator(this, klass);
+ }
+
+ public static void setKryoFactory(Map conf, Class<? extends IKryoFactory> klass) {
+ conf.put(Config.TOPOLOGY_KRYO_FACTORY, klass.getName());
+ }
+
+ public void setKryoFactory(Class<? extends IKryoFactory> klass) {
+ setKryoFactory(this, klass);
+ }
+
+ public static void setSkipMissingKryoRegistrations(Map conf, boolean skip) {
+ conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, skip);
+ }
+
+ public void setSkipMissingKryoRegistrations(boolean skip) {
+ setSkipMissingKryoRegistrations(this, skip);
+ }
+
+ public static void setMaxTaskParallelism(Map conf, int max) {
+ conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, max);
+ }
+
+ public void setMaxTaskParallelism(int max) {
+ setMaxTaskParallelism(this, max);
+ }
+
+ public static void setMaxSpoutPending(Map conf, int max) {
+ conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, max);
+ }
+
+ public void setMaxSpoutPending(int max) {
+ setMaxSpoutPending(this, max);
+ }
+
+ public static void setStatsSampleRate(Map conf, double rate) {
+ conf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, rate);
+ }
+
+ public void setStatsSampleRate(double rate) {
+ setStatsSampleRate(this, rate);
+ }
+
+ public static void setFallBackOnJavaSerialization(Map conf, boolean fallback) {
+ conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, fallback);
+ }
+
+ public void setFallBackOnJavaSerialization(boolean fallback) {
+ setFallBackOnJavaSerialization(this, fallback);
+ }
+
+ private static List getRegisteredSerializations(Map conf) {
+ List ret;
+ if(!conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) {
+ ret = new ArrayList();
+ } else {
+ ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_REGISTER));
+ }
+ conf.put(Config.TOPOLOGY_KRYO_REGISTER, ret);
+ return ret;
+ }
+
+ private static List getRegisteredDecorators(Map conf) {
+ List ret;
+ if(!conf.containsKey(Config.TOPOLOGY_KRYO_DECORATORS)) {
+ ret = new ArrayList();
+ } else {
+ ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS));
+ }
+ conf.put(Config.TOPOLOGY_KRYO_DECORATORS, ret);
+ return ret;
+ }
+}
[13/18] storm git commit: Merge pull request #21 from apache/master
Posted by bo...@apache.org.
Merge pull request #21 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/232e4ace
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/232e4ace
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/232e4ace
Branch: refs/heads/master
Commit: 232e4ace33f4424031eef64fa64abd20c9735f9d
Parents: 8696317 44e9aaf
Author: caofangkun <ca...@gmail.com>
Authored: Wed Apr 29 08:53:32 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Apr 29 08:53:32 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 4 ++
README.markdown | 4 ++
.../documentation/Setting-up-a-Storm-cluster.md | 7 ++-
.../mapper/SimpleTridentHBaseMapMapper.java | 50 ++++++++++++++++++++
.../trident/mapper/TridentHBaseMapMapper.java | 40 ++++++++++++++++
.../hbase/trident/state/HBaseMapState.java | 45 ++++++++----------
.../src/jvm/backtype/storm/task/IBolt.java | 4 +-
.../storm/utils/DisruptorQueueTest.java | 38 ++++++++-------
8 files changed, 149 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
[09/18] storm git commit: Merge pull request #18 from apache/master
Posted by bo...@apache.org.
Merge pull request #18 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a3602f25
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a3602f25
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a3602f25
Branch: refs/heads/master
Commit: a3602f2552a0c1f88f0323d27076df16a26d09a4
Parents: 503a04b a7c8310
Author: caofangkun <ca...@gmail.com>
Authored: Thu Apr 16 14:52:15 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Thu Apr 16 14:52:15 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 9 +
DEVELOPER.md | 4 +-
STORM-UI-REST-API.md | 44 +-
.../storm/starter/bolt/AbstractRankerBolt.java | 4 +-
.../storm/starter/bolt/RollingCountAggBolt.java | 1 -
.../storm/starter/bolt/RollingCountBolt.java | 4 +-
.../jvm/storm/starter/util/TupleHelpers.java | 33 -
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 5 +
pom.xml | 2 +-
storm-core/pom.xml | 5 -
storm-core/src/clj/backtype/storm/converter.clj | 3 +-
.../src/clj/backtype/storm/daemon/common.clj | 13 -
.../clj/backtype/storm/daemon/supervisor.clj | 51 +-
.../src/clj/backtype/storm/daemon/worker.clj | 14 +-
.../src/clj/backtype/storm/local_state.clj | 99 +++
storm-core/src/clj/backtype/storm/testing.clj | 25 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 13 +-
storm-core/src/clj/backtype/storm/util.clj | 5 -
storm-core/src/jvm/backtype/storm/Config.java | 3 +
.../storm/generated/ClusterWorkerHeartbeat.java | 101 ++-
.../storm/generated/LSApprovedWorkers.java | 458 +++++++++++
.../generated/LSSupervisorAssignments.java | 471 ++++++++++++
.../storm/generated/LSSupervisorId.java | 406 ++++++++++
.../storm/generated/LSWorkerHeartbeat.java | 755 +++++++++++++++++++
.../storm/generated/LocalAssignment.java | 561 ++++++++++++++
.../storm/generated/LocalStateData.java | 471 ++++++++++++
.../storm/generated/ThriftSerializedObject.java | 516 +++++++++++++
.../security/auth/SimpleTransportPlugin.java | 3 +-
.../jvm/backtype/storm/utils/LocalState.java | 163 +++-
.../jvm/backtype/storm/utils/TupleUtils.java | 35 +
.../src/jvm/backtype/storm/utils/Utils.java | 119 +--
.../trident/topology/TridentBoltExecutor.java | 3 +-
storm-core/src/py/__init__.py | 2 +
storm-core/src/py/storm/DistributedRPC.py | 2 +
.../src/py/storm/DistributedRPCInvocations.py | 2 +
storm-core/src/py/storm/Nimbus.py | 2 +
storm-core/src/py/storm/__init__.py | 2 +
storm-core/src/py/storm/constants.py | 2 +
storm-core/src/py/storm/ttypes.py | 613 ++++++++++++++-
storm-core/src/storm.thrift | 34 +
storm-core/src/ui/public/css/style.css | 4 -
storm-core/src/ui/public/js/script.js | 3 +-
.../public/templates/anti-forgery-template.html | 19 -
storm-core/src/ui/public/topology.html | 8 -
.../clj/backtype/storm/local_state_test.clj | 40 +-
.../storm/security/auth/nimbus_auth_test.clj | 2 +-
46 files changed, 4828 insertions(+), 306 deletions(-)
----------------------------------------------------------------------
[11/18] storm git commit: Merge pull request #20 from apache/master
Posted by bo...@apache.org.
Merge pull request #20 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/8696317b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/8696317b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/8696317b
Branch: refs/heads/master
Commit: 8696317ba1f3ac19befd53ff6d892282948596e8
Parents: bb8583a 7712102
Author: caofangkun <ca...@gmail.com>
Authored: Tue Apr 28 15:33:25 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Tue Apr 28 15:33:25 2015 +0800
----------------------------------------------------------------------
.gitignore | 1 +
.travis.yml | 9 +
CHANGELOG.md | 14 +
SECURITY.md | 50 +++
bin/storm.py | 24 +-
conf/defaults.yaml | 2 +-
dev-tools/test-ns.py | 19 +-
.../print-errors-from-clojure-test-reports.py | 58 +++
dev-tools/travis/travis-build.sh | 50 +++
docs/documentation/Multilang-protocol.md | 63 +++-
docs/documentation/Trident-API-Overview.md | 2 +-
examples/storm-starter/README.markdown | 8 +-
.../storm-starter/multilang/resources/storm.js | 373 -------------------
.../storm-starter/multilang/resources/storm.py | 260 -------------
.../storm-starter/multilang/resources/storm.rb | 236 ------------
examples/storm-starter/pom.xml | 67 ++--
external/storm-eventhubs/README.md | 41 ++
external/storm-eventhubs/pom.xml | 122 ++++++
.../storm/eventhubs/bolt/EventHubBolt.java | 81 ++++
.../client/ConnectionStringBuilder.java | 116 ++++++
.../storm/eventhubs/client/Constants.java | 32 ++
.../storm/eventhubs/client/EventHubClient.java | 92 +++++
.../eventhubs/client/EventHubConsumerGroup.java | 72 ++++
.../eventhubs/client/EventHubException.java | 37 ++
.../eventhubs/client/EventHubReceiver.java | 139 +++++++
.../eventhubs/client/EventHubSendClient.java | 70 ++++
.../storm/eventhubs/client/EventHubSender.java | 95 +++++
.../storm/eventhubs/client/SelectorFilter.java | 38 ++
.../eventhubs/client/SelectorFilterWriter.java | 64 ++++
.../eventhubs/samples/AtMostOnceEventCount.java | 54 +++
.../storm/eventhubs/samples/EventCount.java | 155 ++++++++
.../storm/eventhubs/samples/EventHubLoop.java | 51 +++
.../samples/OpaqueTridentEventCount.java | 53 +++
.../samples/TransactionalTridentEventCount.java | 81 ++++
.../eventhubs/samples/bolt/GlobalCountBolt.java | 83 +++++
.../samples/bolt/PartialCountBolt.java | 63 ++++
.../apache/storm/eventhubs/spout/EventData.java | 48 +++
.../storm/eventhubs/spout/EventDataScheme.java | 55 +++
.../eventhubs/spout/EventHubReceiverFilter.java | 56 +++
.../eventhubs/spout/EventHubReceiverImpl.java | 150 ++++++++
.../storm/eventhubs/spout/EventHubSpout.java | 258 +++++++++++++
.../eventhubs/spout/EventHubSpoutConfig.java | 165 ++++++++
.../eventhubs/spout/EventHubSpoutException.java | 37 ++
.../storm/eventhubs/spout/FieldConstants.java | 25 ++
.../storm/eventhubs/spout/IEventDataScheme.java | 30 ++
.../eventhubs/spout/IEventHubReceiver.java | 35 ++
.../spout/IEventHubReceiverFactory.java | 30 ++
.../spout/IEventHubReceiverFilter.java | 35 ++
.../eventhubs/spout/IPartitionCoordinator.java | 27 ++
.../eventhubs/spout/IPartitionManager.java | 37 ++
.../spout/IPartitionManagerFactory.java | 33 ++
.../storm/eventhubs/spout/IStateStore.java | 31 ++
.../apache/storm/eventhubs/spout/MessageId.java | 56 +++
.../storm/eventhubs/spout/PartitionManager.java | 101 +++++
.../eventhubs/spout/SimplePartitionManager.java | 136 +++++++
.../spout/StaticPartitionCoordinator.java | 85 +++++
.../eventhubs/spout/ZookeeperStateStore.java | 95 +++++
.../storm/eventhubs/trident/Coordinator.java | 60 +++
.../trident/ITridentPartitionManager.java | 35 ++
.../ITridentPartitionManagerFactory.java | 26 ++
.../trident/OpaqueTridentEventHubEmitter.java | 69 ++++
.../trident/OpaqueTridentEventHubSpout.java | 64 ++++
.../storm/eventhubs/trident/Partition.java | 39 ++
.../storm/eventhubs/trident/Partitions.java | 41 ++
.../TransactionalTridentEventHubEmitter.java | 167 +++++++++
.../TransactionalTridentEventHubSpout.java | 66 ++++
.../trident/TridentPartitionManager.java | 91 +++++
.../src/main/resources/config.properties | 27 ++
.../eventhubs/spout/EventHubReceiverMock.java | 105 ++++++
.../spout/EventHubSpoutCallerMock.java | 96 +++++
.../spout/PartitionManagerCallerMock.java | 105 ++++++
.../spout/SpoutOutputCollectorMock.java | 61 +++
.../storm/eventhubs/spout/StateStoreMock.java | 54 +++
.../storm/eventhubs/spout/TestEventData.java | 47 +++
.../eventhubs/spout/TestEventHubSpout.java | 70 ++++
.../eventhubs/spout/TestPartitionManager.java | 117 ++++++
.../TestTransactionalTridentEmitter.java | 93 +++++
.../eventhubs/trident/TridentCollectorMock.java | 52 +++
external/storm-kafka/README.md | 49 ++-
external/storm-redis/pom.xml | 2 +-
.../redis/common/container/JedisContainer.java | 7 +-
.../redis/trident/state/RedisMapState.java | 10 +-
pom.xml | 4 +
storm-core/pom.xml | 35 ++
storm-core/src/clj/backtype/storm/cluster.clj | 5 +-
storm-core/src/clj/backtype/storm/converter.clj | 7 +-
.../src/clj/backtype/storm/daemon/common.clj | 2 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 21 +-
.../src/clj/backtype/storm/daemon/executor.clj | 3 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 1 +
.../src/clj/backtype/storm/daemon/nimbus.clj | 9 +-
.../clj/backtype/storm/daemon/supervisor.clj | 11 +-
.../src/clj/backtype/storm/daemon/worker.clj | 1 +
storm-core/src/clj/backtype/storm/ui/core.clj | 27 +-
.../src/clj/backtype/storm/ui/helpers.clj | 51 ++-
storm-core/src/clj/backtype/storm/util.clj | 32 ++
storm-core/src/dev/resources/storm.js | 373 -------------------
storm-core/src/dev/resources/storm.py | 260 -------------
storm-core/src/dev/resources/storm.rb | 236 ------------
storm-core/src/jvm/backtype/storm/Config.java | 94 +++++
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 8 +-
.../storm/generated/ClusterWorkerHeartbeat.java | 1 +
.../jvm/backtype/storm/generated/Nimbus.java | 12 +-
.../storm/generated/SupervisorInfo.java | 116 +++++-
.../storm/generated/SupervisorSummary.java | 117 +++++-
.../storm/generated/TopologySummary.java | 2 +-
.../security/auth/SaslTransportPlugin.java | 17 +-
.../GzipBridgeThriftSerializationDelegate.java | 64 ++++
.../GzipThriftSerializationDelegate.java | 57 +++
.../src/jvm/backtype/storm/task/ShellBolt.java | 5 +-
.../backtype/storm/task/TopologyContext.java | 104 ++++--
.../storm/utils/ExtendedThreadPoolExecutor.java | 67 ++++
.../src/jvm/backtype/storm/utils/Utils.java | 49 +++
storm-core/src/multilang/js/storm.js | 366 ------------------
storm-core/src/multilang/py/storm.py | 260 -------------
storm-core/src/multilang/rb/storm.rb | 236 ------------
storm-core/src/py/storm/Nimbus.py | 8 +
storm-core/src/py/storm/ttypes.py | 32 +-
storm-core/src/storm.thrift | 9 +
.../templates/component-page-template.html | 2 +-
.../public/templates/index-page-template.html | 6 +
.../test/clj/backtype/storm/cluster_test.clj | 4 +-
...ipBridgeThriftSerializationDelegateTest.java | 71 ++++
storm-core/test/resources/logback-test.xml | 26 ++
storm-dist/binary/src/main/assembly/binary.xml | 41 +-
storm-multilang/javascript/pom.xml | 32 ++
.../src/main/resources/resources/storm.js | 373 +++++++++++++++++++
storm-multilang/python/pom.xml | 32 ++
.../src/main/resources/resources/storm.py | 260 +++++++++++++
storm-multilang/ruby/pom.xml | 32 ++
.../ruby/src/main/resources/resources/storm.rb | 236 ++++++++++++
131 files changed, 6900 insertions(+), 2770 deletions(-)
----------------------------------------------------------------------
[08/18] storm git commit: Merge pull request #17 from apache/master
Posted by bo...@apache.org.
Merge pull request #17 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/503a04ba
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/503a04ba
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/503a04ba
Branch: refs/heads/master
Commit: 503a04ba1f470ab3f1f6b32109d27a401a8a2510
Parents: 2bc9447 2aaa718
Author: caofangkun <ca...@gmail.com>
Authored: Wed Apr 1 14:13:00 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Apr 1 14:13:00 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 7 +
README.markdown | 1 +
STORM-UI-REST-API.md | 30 +++
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 2 +-
.../storm/hbase/bolt/HBaseLookupBolt.java | 4 +-
.../hbase/trident/state/HBaseMapState.java | 2 +
.../storm/hbase/trident/state/HBaseState.java | 2 +-
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 2 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 16 +-
external/storm-redis/README.md | 108 ++++++++-
.../redis/topology/PersistentWordCount.java | 35 ---
storm-core/src/clj/backtype/storm/ui/core.clj | 62 ++++-
.../src/clj/backtype/storm/ui/helpers.clj | 15 +-
storm-core/src/jvm/backtype/storm/Config.java | 6 +
.../jvm/backtype/storm/utils/NimbusClient.java | 7 +
.../storm/security/auth/nimbus_auth_test.clj | 231 ++++++++++---------
17 files changed, 364 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
[02/18] storm git commit: STORM-596: rm defaults.yaml.orig
Config.java.orig
Posted by bo...@apache.org.
STORM-596: rm defaults.yaml.orig Config.java.orig
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/986ad60b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/986ad60b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/986ad60b
Branch: refs/heads/master
Commit: 986ad60bf065c8c3dd965b5c2cfd33d71229385b
Parents: cad1505
Author: caofangkun <ca...@gmail.com>
Authored: Wed Dec 24 11:56:12 2014 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Dec 24 11:56:12 2014 +0800
----------------------------------------------------------------------
conf/defaults.yaml.orig | 201 ---
.../src/jvm/backtype/storm/Config.java.orig | 1434 ------------------
2 files changed, 1635 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/986ad60b/conf/defaults.yaml.orig
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml.orig b/conf/defaults.yaml.orig
deleted file mode 100644
index d03c2c9..0000000
--- a/conf/defaults.yaml.orig
+++ /dev/null
@@ -1,201 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-########### These all have default values as shown
-########### Additional configuration goes into storm.yaml
-
-java.library.path: "/usr/local/lib:/opt/local/lib:/usr/lib"
-
-### storm.* configs are general configurations
-# the local dir is where jars are kept
-storm.local.dir: "storm-local"
-storm.zookeeper.servers:
- - "localhost"
-storm.zookeeper.port: 2181
-storm.zookeeper.root: "/storm"
-storm.zookeeper.session.timeout: 20000
-storm.zookeeper.connection.timeout: 15000
-storm.zookeeper.retry.times: 5
-storm.zookeeper.retry.interval: 1000
-storm.zookeeper.retry.intervalceiling.millis: 30000
-storm.zookeeper.auth.user: null
-storm.zookeeper.auth.password: null
-storm.cluster.mode: "distributed" # can be distributed or local
-storm.local.mode.zmq: false
-storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin"
-storm.principal.tolocal: "backtype.storm.security.auth.DefaultPrincipalToLocal"
-storm.group.mapping.service: "backtype.storm.security.auth.ShellBasedGroupsMapping"
-storm.messaging.transport: "backtype.storm.messaging.netty.Context"
-storm.nimbus.retry.times: 5
-storm.nimbus.retry.interval.millis: 2000
-storm.nimbus.retry.intervalceiling.millis: 60000
-storm.auth.simple-white-list.users: []
-storm.auth.simple-acl.users: []
-storm.auth.simple-acl.users.commands: []
-storm.auth.simple-acl.admins: []
-storm.meta.serialization.delegate: "backtype.storm.serialization.DefaultSerializationDelegate"
-
-### nimbus.* configs are for the master
-nimbus.host: "localhost"
-nimbus.thrift.port: 6627
-nimbus.thrift.threads: 64
-nimbus.thrift.max_buffer_size: 1048576
-nimbus.childopts: "-Xmx1024m"
-nimbus.task.timeout.secs: 30
-nimbus.supervisor.timeout.secs: 60
-nimbus.monitor.freq.secs: 10
-nimbus.cleanup.inbox.freq.secs: 600
-nimbus.inbox.jar.expiration.secs: 3600
-nimbus.task.launch.secs: 120
-nimbus.reassign: true
-nimbus.file.copy.expiration.secs: 600
-nimbus.topology.validator: "backtype.storm.nimbus.DefaultTopologyValidator"
-nimbus.credential.renewers.freq.secs: 600
-
-### ui.* configs are for the master
-ui.port: 8080
-ui.childopts: "-Xmx768m"
-ui.actions.enabled: true
-ui.filter: null
-ui.filter.params: null
-ui.users: null
-ui.header.buffer.bytes: 4096
-ui.http.creds.plugin: backtype.storm.security.auth.DefaultHttpCredentialsPlugin
-
-logviewer.port: 8000
-logviewer.childopts: "-Xmx128m"
-logviewer.cleanup.age.mins: 10080
-logviewer.appender.name: "A1"
-
-logs.users: null
-
-drpc.port: 3772
-drpc.worker.threads: 64
-drpc.max_buffer_size: 1048576
-drpc.queue.size: 128
-drpc.invocations.port: 3773
-drpc.invocations.threads: 64
-drpc.request.timeout.secs: 600
-drpc.childopts: "-Xmx768m"
-drpc.http.port: 3774
-drpc.https.port: -1
-drpc.https.keystore.password: ""
-drpc.https.keystore.type: "JKS"
-drpc.http.creds.plugin: backtype.storm.security.auth.DefaultHttpCredentialsPlugin
-drpc.authorizer.acl.filename: "drpc-auth-acl.yaml"
-drpc.authorizer.acl.strict: false
-
-transactional.zookeeper.root: "/transactional"
-transactional.zookeeper.servers: null
-transactional.zookeeper.port: null
-
-### supervisor.* configs are for node supervisors
-# Define the amount of workers that can be run on this machine. Each worker is assigned a port to use for communication
-supervisor.slots.ports:
- - 6700
- - 6701
- - 6702
- - 6703
-supervisor.childopts: "-Xmx256m"
-supervisor.run.worker.as.user: false
-#how long supervisor will wait to ensure that a worker process is started
-supervisor.worker.start.timeout.secs: 120
-#how long between heartbeats until supervisor considers that worker dead and tries to restart it
-supervisor.worker.timeout.secs: 30
-#how frequently the supervisor checks on the status of the processes it's monitoring and restarts if necessary
-supervisor.monitor.frequency.secs: 3
-#how frequently the supervisor heartbeats to the cluster state (for nimbus)
-supervisor.heartbeat.frequency.secs: 5
-supervisor.enable: true
-supervisor.supervisors: []
-supervisor.supervisors.commands: []
-
-
-### worker.* configs are for task workers
-worker.childopts: "-Xmx768m"
-worker.gc.childopts: ""
-worker.heartbeat.frequency.secs: 1
-
-# control how many worker receiver threads we need per worker
-topology.worker.receiver.thread.count: 1
-
-task.heartbeat.frequency.secs: 3
-task.refresh.poll.secs: 10
-task.credentials.poll.secs: 30
-
-zmq.threads: 1
-zmq.linger.millis: 5000
-zmq.hwm: 0
-
-
-storm.messaging.netty.server_worker_threads: 1
-storm.messaging.netty.client_worker_threads: 1
-storm.messaging.netty.buffer_size: 5242880 #5MB buffer
-# Since nimbus.task.launch.secs and supervisor.worker.start.timeout.secs are 120, other workers should also wait at least that long before giving up on connecting to the other worker.
-storm.messaging.netty.max_retries: 300
-storm.messaging.netty.max_wait_ms: 1000
-storm.messaging.netty.min_wait_ms: 100
-
-# If the Netty messaging layer is busy(netty internal buffer not writable), the Netty client will try to batch message as more as possible up to the size of storm.messaging.netty.transfer.batch.size bytes, otherwise it will try to flush message as soon as possible to reduce latency.
-storm.messaging.netty.transfer.batch.size: 262144
-
-# We check with this interval that whether the Netty channel is writable and try to write pending messages if it is.
-storm.messaging.netty.flush.check.interval.ms: 10
-
-# By default, the Netty SASL authentication is set to false. Users can override and set it true for a specific topology.
-storm.messaging.netty.authentication: false
-
-# default number of seconds group mapping service will cache user group
-storm.group.mapping.service.cache.duration.secs: 120
-
-### topology.* configs are for specific executing storms
-topology.enable.message.timeouts: true
-topology.debug: false
-topology.workers: 1
-topology.acker.executors: null
-topology.tasks: null
-# maximum amount of time a message has to complete before it's considered failed
-topology.message.timeout.secs: 30
-topology.multilang.serializer: "backtype.storm.multilang.JsonSerializer"
-topology.skip.missing.kryo.registrations: false
-topology.max.task.parallelism: null
-topology.max.spout.pending: null
-topology.state.synchronization.timeout.secs: 60
-topology.stats.sample.rate: 0.05
-topology.builtin.metrics.bucket.size.secs: 60
-topology.fall.back.on.java.serialization: true
-topology.worker.childopts: null
-topology.executor.receive.buffer.size: 1024 #batched
-topology.executor.send.buffer.size: 1024 #individual messages
-topology.receiver.buffer.size: 8 # setting it too high causes a lot of problems (heartbeat thread gets starved, throughput plummets)
-topology.transfer.buffer.size: 1024 # batched
-topology.tick.tuple.freq.secs: null
-topology.worker.shared.thread.pool.size: 4
-topology.disruptor.wait.strategy: "com.lmax.disruptor.BlockingWaitStrategy"
-topology.spout.wait.strategy: "backtype.storm.spout.SleepSpoutWaitStrategy"
-topology.sleep.spout.wait.strategy.time.ms: 1
-topology.error.throttle.interval.secs: 10
-topology.max.error.report.per.interval: 5
-topology.kryo.factory: "backtype.storm.serialization.DefaultKryoFactory"
-topology.tuple.serializer: "backtype.storm.serialization.types.ListDelegateSerializer"
-topology.trident.batch.emit.interval.millis: 500
-topology.testing.always.try.serialize: false
-topology.classpath: null
-topology.environment: null
-topology.bolts.outgoing.overflow.buffer.enable: false
-
-dev.zookeeper.path: "/tmp/dev-storm-zookeeper"
http://git-wip-us.apache.org/repos/asf/storm/blob/986ad60b/storm-core/src/jvm/backtype/storm/Config.java.orig
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java.orig b/storm-core/src/jvm/backtype/storm/Config.java.orig
deleted file mode 100644
index c680354..0000000
--- a/storm-core/src/jvm/backtype/storm/Config.java.orig
+++ /dev/null
@@ -1,1434 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package backtype.storm;
-
-import backtype.storm.serialization.IKryoDecorator;
-import backtype.storm.serialization.IKryoFactory;
-import com.esotericsoftware.kryo.Serializer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Topology configs are specified as a plain old map. This class provides a
- * convenient way to create a topology config map by providing setter methods for
- * all the configs that can be set. It also makes it easier to do things like add
- * serializations.
- *
- * <p>This class also provides constants for all the configurations possible on
- * a Storm cluster and Storm topology. Each constant is paired with a schema
- * that defines the validity criterion of the corresponding field. Default
- * values for these configs can be found in defaults.yaml.</p>
- *
- * <p>Note that you may put other configurations in any of the configs. Storm
- * will ignore anything it doesn't recognize, but your topologies are free to make
- * use of them by reading them in the prepare method of Bolts or the open method of
- * Spouts.</p>
- */
-public class Config extends HashMap<String, Object> {
- /**
- * This is part of a temporary workaround to a ZK bug, it is the 'scheme:acl' for
- * the user Nimbus and Supervisors use to authenticate with ZK.
- */
- public static final String STORM_ZOOKEEPER_SUPERACL = "storm.zookeeper.superACL";
- public static final Object STORM_ZOOKEEPER_SUPERACL_SCHEMA = String.class;
-
- /**
- * The transporter for communication among Storm tasks
- */
- public static final String STORM_MESSAGING_TRANSPORT = "storm.messaging.transport";
- public static final Object STORM_MESSAGING_TRANSPORT_SCHEMA = String.class;
-
- /**
- * Netty based messaging: The buffer size for send/recv buffer
- */
- public static final String STORM_MESSAGING_NETTY_BUFFER_SIZE = "storm.messaging.netty.buffer_size";
- public static final Object STORM_MESSAGING_NETTY_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Netty based messaging: The max # of retries that a peer will perform when a remote is not accessible
- */
- public static final String STORM_MESSAGING_NETTY_MAX_RETRIES = "storm.messaging.netty.max_retries";
- public static final Object STORM_MESSAGING_NETTY_MAX_RETRIES_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Netty based messaging: The min # of milliseconds that a peer will wait.
- */
- public static final String STORM_MESSAGING_NETTY_MIN_SLEEP_MS = "storm.messaging.netty.min_wait_ms";
- public static final Object STORM_MESSAGING_NETTY_MIN_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Netty based messaging: The max # of milliseconds that a peer will wait.
- */
- public static final String STORM_MESSAGING_NETTY_MAX_SLEEP_MS = "storm.messaging.netty.max_wait_ms";
- public static final Object STORM_MESSAGING_NETTY_MAX_SLEEP_MS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Netty based messaging: The # of worker threads for the server.
- */
- public static final String STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS = "storm.messaging.netty.server_worker_threads";
- public static final Object STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Netty based messaging: The # of worker threads for the client.
- */
- public static final String STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS = "storm.messaging.netty.client_worker_threads";
- public static final Object STORM_MESSAGING_NETTY_CLIENT_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * If the Netty messaging layer is busy, the Netty client will try to batch message as more as possible up to the size of STORM_NETTY_MESSAGE_BATCH_SIZE bytes
- */
- public static final String STORM_NETTY_MESSAGE_BATCH_SIZE = "storm.messaging.netty.transfer.batch.size";
- public static final Object STORM_NETTY_MESSAGE_BATCH_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * We check with this interval that whether the Netty channel is writable and try to write pending messages
- */
- public static final String STORM_NETTY_FLUSH_CHECK_INTERVAL_MS = "storm.messaging.netty.flush.check.interval.ms";
- public static final Object STORM_NETTY_FLUSH_CHECK_INTERVAL_MS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Netty based messaging: Is authentication required for Netty messaging from client worker process to server worker process.
- */
- public static final String STORM_MESSAGING_NETTY_AUTHENTICATION = "storm.messaging.netty.authentication";
- public static final Object STORM_MESSAGING_NETTY_AUTHENTICATION_SCHEMA = Boolean.class;
-
- /**
- * The delegate for serializing metadata, should be used for serialized objects stored in zookeeper and on disk.
- * This is NOT used for compressing serialized tuples sent between topologies.
- */
- public static final String STORM_META_SERIALIZATION_DELEGATE = "storm.meta.serialization.delegate";
- public static final Object STORM_META_SERIALIZATION_DELEGATE_SCHEMA = String.class;
-
- /**
- * A list of hosts of ZooKeeper servers used to manage the cluster.
- */
- public static final String STORM_ZOOKEEPER_SERVERS = "storm.zookeeper.servers";
- public static final Object STORM_ZOOKEEPER_SERVERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * The port Storm will use to connect to each of the ZooKeeper servers.
- */
- public static final String STORM_ZOOKEEPER_PORT = "storm.zookeeper.port";
- public static final Object STORM_ZOOKEEPER_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * A directory on the local filesystem used by Storm for any local
- * filesystem usage it needs. The directory must exist and the Storm daemons must
- * have permission to read/write from this location.
- */
- public static final String STORM_LOCAL_DIR = "storm.local.dir";
- public static final Object STORM_LOCAL_DIR_SCHEMA = String.class;
-
- /**
- * A global task scheduler used to assign topologies's tasks to supervisors' wokers.
- *
- * If this is not set, a default system scheduler will be used.
- */
- public static final String STORM_SCHEDULER = "storm.scheduler";
- public static final Object STORM_SCHEDULER_SCHEMA = String.class;
-
- /**
- * The mode this Storm cluster is running in. Either "distributed" or "local".
- */
- public static final String STORM_CLUSTER_MODE = "storm.cluster.mode";
- public static final Object STORM_CLUSTER_MODE_SCHEMA = String.class;
-
- /**
- * The hostname the supervisors/workers should report to nimbus. If unset, Storm will
- * get the hostname to report by calling <code>InetAddress.getLocalHost().getCanonicalHostName()</code>.
- *
- * You should set this config when you dont have a DNS which supervisors/workers
- * can utilize to find each other based on hostname got from calls to
- * <code>InetAddress.getLocalHost().getCanonicalHostName()</code>.
- */
- public static final String STORM_LOCAL_HOSTNAME = "storm.local.hostname";
- public static final Object STORM_LOCAL_HOSTNAME_SCHEMA = String.class;
-
- /**
- * The plugin that will convert a principal to a local user.
- */
- public static final String STORM_PRINCIPAL_TO_LOCAL_PLUGIN = "storm.principal.tolocal";
- public static final Object STORM_PRINCIPAL_TO_LOCAL_PLUGIN_SCHEMA = String.class;
-
- /**
- * The plugin that will provide user groups service
- */
- public static final String STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN = "storm.group.mapping.service";
- public static final Object STORM_GROUP_MAPPING_SERVICE_PROVIDER_PLUGIN_SCHEMA = String.class;
-
- /**
- * Max no.of seconds group mapping service will cache user groups
- */
- public static final String STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS = "storm.group.mapping.service.cache.duration.secs";
- public static final Object STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS_SCHEMA = Number.class;
-
- /**
- * The default transport plug-in for Thrift client/server communication
- */
- public static final String STORM_THRIFT_TRANSPORT_PLUGIN = "storm.thrift.transport";
- public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
-
- /**
- * The serializer class for ListDelegate (tuple payload).
- * The default serializer will be ListDelegateSerializer
- */
- public static final String TOPOLOGY_TUPLE_SERIALIZER = "topology.tuple.serializer";
- public static final Object TOPOLOGY_TUPLE_SERIALIZER_SCHEMA = String.class;
-
- /**
- * Try to serialize all tuples, even for local transfers. This should only be used
- * for testing, as a sanity check that all of your tuples are setup properly.
- */
- public static final String TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE = "topology.testing.always.try.serialize";
- public static final Object TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE_SCHEMA = Boolean.class;
-
- /**
- * Whether or not to use ZeroMQ for messaging in local mode. If this is set
- * to false, then Storm will use a pure-Java messaging system. The purpose
- * of this flag is to make it easy to run Storm in local mode by eliminating
- * the need for native dependencies, which can be difficult to install.
- *
- * Defaults to false.
- */
- public static final String STORM_LOCAL_MODE_ZMQ = "storm.local.mode.zmq";
- public static final Object STORM_LOCAL_MODE_ZMQ_SCHEMA = Boolean.class;
-
- /**
- * The root location at which Storm stores data in ZooKeeper.
- */
- public static final String STORM_ZOOKEEPER_ROOT = "storm.zookeeper.root";
- public static final Object STORM_ZOOKEEPER_ROOT_SCHEMA = String.class;
-
- /**
- * The session timeout for clients to ZooKeeper.
- */
- public static final String STORM_ZOOKEEPER_SESSION_TIMEOUT = "storm.zookeeper.session.timeout";
- public static final Object STORM_ZOOKEEPER_SESSION_TIMEOUT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The connection timeout for clients to ZooKeeper.
- */
- public static final String STORM_ZOOKEEPER_CONNECTION_TIMEOUT = "storm.zookeeper.connection.timeout";
- public static final Object STORM_ZOOKEEPER_CONNECTION_TIMEOUT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The number of times to retry a Zookeeper operation.
- */
- public static final String STORM_ZOOKEEPER_RETRY_TIMES="storm.zookeeper.retry.times";
- public static final Object STORM_ZOOKEEPER_RETRY_TIMES_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The interval between retries of a Zookeeper operation.
- */
- public static final String STORM_ZOOKEEPER_RETRY_INTERVAL="storm.zookeeper.retry.interval";
- public static final Object STORM_ZOOKEEPER_RETRY_INTERVAL_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The ceiling of the interval between retries of a Zookeeper operation.
- */
- public static final String STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING="storm.zookeeper.retry.intervalceiling.millis";
- public static final Object STORM_ZOOKEEPER_RETRY_INTERVAL_CEILING_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The cluster Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication.
- */
- public static final String STORM_ZOOKEEPER_AUTH_SCHEME="storm.zookeeper.auth.scheme";
- public static final Object STORM_ZOOKEEPER_AUTH_SCHEME_SCHEMA = String.class;
-
- /**
- * A string representing the payload for cluster Zookeeper authentication.
- * It gets serialized using UTF-8 encoding during authentication.
- * Note that if this is set to something with a secret (as when using
- * digest authentication) then it should only be set in the
- * storm-cluster-auth.yaml file.
- * This file storm-cluster-auth.yaml should then be protected with
- * appropriate permissions that deny access from workers.
- */
- public static final String STORM_ZOOKEEPER_AUTH_PAYLOAD="storm.zookeeper.auth.payload";
- public static final Object STORM_ZOOKEEPER_AUTH_PAYLOAD_SCHEMA = String.class;
-
- /**
- * The topology Zookeeper authentication scheme to use, e.g. "digest". Defaults to no authentication.
- */
- public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME="storm.zookeeper.topology.auth.scheme";
- public static final Object STORM_ZOOKEEPER_TOPOLOGY_AUTH_SCHEME_SCHEMA = String.class;
-
- /**
- * A string representing the payload for topology Zookeeper authentication. It gets serialized using UTF-8 encoding during authentication.
- */
- public static final String STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD="storm.zookeeper.topology.auth.payload";
- public static final Object STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD_SCHEMA = String.class;
-
- /**
- * The id assigned to a running topology. The id is the storm name with a unique nonce appended.
- */
- public static final String STORM_ID = "storm.id";
- public static final Object STORM_ID_SCHEMA = String.class;
-
- /**
- * The number of times to retry a Nimbus operation.
- */
- public static final String STORM_NIMBUS_RETRY_TIMES="storm.nimbus.retry.times";
- public static final Object STORM_NIMBUS_RETRY_TIMES_SCHEMA = Number.class;
-
- /**
- * The starting interval between exponential backoff retries of a Nimbus operation.
- */
- public static final String STORM_NIMBUS_RETRY_INTERVAL="storm.nimbus.retry.interval.millis";
- public static final Object STORM_NIMBUS_RETRY_INTERVAL_SCHEMA = Number.class;
-
- /**
- * The ceiling of the interval between retries of a client connect to Nimbus operation.
- */
- public static final String STORM_NIMBUS_RETRY_INTERVAL_CEILING="storm.nimbus.retry.intervalceiling.millis";
- public static final Object STORM_NIMBUS_RETRY_INTERVAL_CEILING_SCHEMA = Number.class;
-
- /**
- * The host that the master server is running on.
- */
- public static final String NIMBUS_HOST = "nimbus.host";
- public static final Object NIMBUS_HOST_SCHEMA = String.class;
-
- /**
- * The Nimbus transport plug-in for Thrift client/server communication
- */
- public static final String NIMBUS_THRIFT_TRANSPORT_PLUGIN = "nimbus.thrift.transport";
- public static final Object NIMBUS_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
-
- /**
- * Which port the Thrift interface of Nimbus should run on. Clients should
- * connect to this port to upload jars and submit topologies.
- */
- public static final String NIMBUS_THRIFT_PORT = "nimbus.thrift.port";
- public static final Object NIMBUS_THRIFT_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The number of threads that should be used by the nimbus thrift server.
- */
- public static final String NIMBUS_THRIFT_THREADS = "nimbus.thrift.threads";
- public static final Object NIMBUS_THRIFT_THREADS_SCHEMA = Number.class;
-
- /**
- * A list of users that are cluster admins and can run any command. To use this set
- * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
- */
- public static final String NIMBUS_ADMINS = "nimbus.admins";
- public static final Object NIMBUS_ADMINS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * A list of users that run the supervisors and should be authorized to interact with
- * nimbus as a supervisor would. To use this set
- * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
- */
- public static final String NIMBUS_SUPERVISOR_USERS = "nimbus.supervisor.users";
- public static final Object NIMBUS_SUPERVISOR_USERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * The maximum buffer size thrift should use when reading messages.
- */
- public static final String NIMBUS_THRIFT_MAX_BUFFER_SIZE = "nimbus.thrift.max_buffer_size";
- public static final Object NIMBUS_THRIFT_MAX_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * This parameter is used by the storm-deploy project to configure the
- * jvm options for the nimbus daemon.
- */
- public static final String NIMBUS_CHILDOPTS = "nimbus.childopts";
- public static final Object NIMBUS_CHILDOPTS_SCHEMA = String.class;
-
-
- /**
- * How long without heartbeating a task can go before nimbus will consider the
- * task dead and reassign it to another location.
- */
- public static final String NIMBUS_TASK_TIMEOUT_SECS = "nimbus.task.timeout.secs";
- public static final Object NIMBUS_TASK_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * How often nimbus should wake up to check heartbeats and do reassignments. Note
- * that if a machine ever goes down Nimbus will immediately wake up and take action.
- * This parameter is for checking for failures when there's no explicit event like that
- * occuring.
- */
- public static final String NIMBUS_MONITOR_FREQ_SECS = "nimbus.monitor.freq.secs";
- public static final Object NIMBUS_MONITOR_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How often nimbus should wake the cleanup thread to clean the inbox.
- * @see NIMBUS_INBOX_JAR_EXPIRATION_SECS
- */
- public static final String NIMBUS_CLEANUP_INBOX_FREQ_SECS = "nimbus.cleanup.inbox.freq.secs";
- public static final Object NIMBUS_CLEANUP_INBOX_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The length of time a jar file lives in the inbox before being deleted by the cleanup thread.
- *
- * Probably keep this value greater than or equal to NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS.
- * Note that the time it takes to delete an inbox jar file is going to be somewhat more than
- * NIMBUS_CLEANUP_INBOX_JAR_EXPIRATION_SECS (depending on how often NIMBUS_CLEANUP_FREQ_SECS
- * is set to).
- * @see NIMBUS_CLEANUP_FREQ_SECS
- */
- public static final String NIMBUS_INBOX_JAR_EXPIRATION_SECS = "nimbus.inbox.jar.expiration.secs";
- public static final Object NIMBUS_INBOX_JAR_EXPIRATION_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How long before a supervisor can go without heartbeating before nimbus considers it dead
- * and stops assigning new work to it.
- */
- public static final String NIMBUS_SUPERVISOR_TIMEOUT_SECS = "nimbus.supervisor.timeout.secs";
- public static final Object NIMBUS_SUPERVISOR_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * A special timeout used when a task is initially launched. During launch, this is the timeout
- * used until the first heartbeat, overriding nimbus.task.timeout.secs.
- *
- * <p>A separate timeout exists for launch because there can be quite a bit of overhead
- * to launching new JVM's and configuring them.</p>
- */
- public static final String NIMBUS_TASK_LAUNCH_SECS = "nimbus.task.launch.secs";
- public static final Object NIMBUS_TASK_LAUNCH_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Whether or not nimbus should reassign tasks if it detects that a task goes down.
- * Defaults to true, and it's not recommended to change this value.
- */
- public static final String NIMBUS_REASSIGN = "nimbus.reassign";
- public static final Object NIMBUS_REASSIGN_SCHEMA = Boolean.class;
-
- /**
- * During upload/download with the master, how long an upload or download connection is idle
- * before nimbus considers it dead and drops the connection.
- */
- public static final String NIMBUS_FILE_COPY_EXPIRATION_SECS = "nimbus.file.copy.expiration.secs";
- public static final Object NIMBUS_FILE_COPY_EXPIRATION_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * A custom class that implements ITopologyValidator that is run whenever a
- * topology is submitted. Can be used to provide business-specific logic for
- * whether topologies are allowed to run or not.
- */
- public static final String NIMBUS_TOPOLOGY_VALIDATOR = "nimbus.topology.validator";
- public static final Object NIMBUS_TOPOLOGY_VALIDATOR_SCHEMA = String.class;
-
- /**
- * Class name for authorization plugin for Nimbus
- */
- public static final String NIMBUS_AUTHORIZER = "nimbus.authorizer";
- public static final Object NIMBUS_AUTHORIZER_SCHEMA = String.class;
-
- /**
- * How often nimbus should wake up to renew credentials if needed.
- */
- public static final String NIMBUS_CREDENTIAL_RENEW_FREQ_SECS = "nimbus.credential.renewers.freq.secs";
- public static final Object NIMBUS_CREDENTIAL_RENEW_FREQ_SECS_SCHEMA = Number.class;
-
- /**
- * A list of credential renewers that nimbus should load.
- */
- public static final String NIMBUS_CREDENTIAL_RENEWERS = "nimbus.credential.renewers.classes";
- public static final Object NIMBUS_CREDENTIAL_RENEWERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * A list of plugins that nimbus should load during submit topology to populate
- * credentials on user's behalf.
- */
- public static final String NIMBUS_AUTO_CRED_PLUGINS = "nimbus.autocredential.plugins.classes";
- public static final Object NIMBUS_AUTO_CRED_PLUGINS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * Storm UI binds to this port.
- */
- public static final String UI_PORT = "ui.port";
- public static final Object UI_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * HTTP UI port for log viewer
- */
- public static final String LOGVIEWER_PORT = "logviewer.port";
- public static final Object LOGVIEWER_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Childopts for log viewer java process.
- */
- public static final String LOGVIEWER_CHILDOPTS = "logviewer.childopts";
- public static final Object LOGVIEWER_CHILDOPTS_SCHEMA = String.class;
-
- /**
- * How often to clean up old log files
- */
- public static final String LOGVIEWER_CLEANUP_INTERVAL_SECS = "logviewer.cleanup.interval.secs";
- public static final Object LOGVIEWER_CLEANUP_INTERVAL_SECS_SCHEMA = ConfigValidation.PositiveIntegerValidator;
-
- /**
- * How many minutes since a log was last modified for the log to be considered for clean-up
- */
- public static final String LOGVIEWER_CLEANUP_AGE_MINS = "logviewer.cleanup.age.mins";
- public static final Object LOGVIEWER_CLEANUP_AGE_MINS_SCHEMA = ConfigValidation.PositiveIntegerValidator;
-
- /**
- * A list of users allowed to view logs via the Log Viewer
- */
- public static final String LOGS_USERS = "logs.users";
- public static final Object LOGS_USERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * Appender name used by log viewer to determine log directory.
- */
- public static final String LOGVIEWER_APPENDER_NAME = "logviewer.appender.name";
- public static final Object LOGVIEWER_APPENDER_NAME_SCHEMA = String.class;
-
- /**
- * Childopts for Storm UI Java process.
- */
- public static final String UI_CHILDOPTS = "ui.childopts";
- public static final Object UI_CHILDOPTS_SCHEMA = String.class;
-
- /**
- * A class implementing javax.servlet.Filter for authenticating/filtering UI requests
- */
- public static final String UI_FILTER = "ui.filter";
- public static final Object UI_FILTER_SCHEMA = String.class;
-
- /**
- * Initialization parameters for the javax.servlet.Filter
- */
- public static final String UI_FILTER_PARAMS = "ui.filter.params";
- public static final Object UI_FILTER_PARAMS_SCHEMA = Map.class;
-
- /**
- * The size of the header buffer for the UI in bytes
- */
- public static final String UI_HEADER_BUFFER_BYTES = "ui.header.buffer.bytes";
- public static final Object UI_HEADER_BUFFER_BYTES_SCHEMA = Number.class;
-
- /**
- * A list of users allowed to view topologies via the UI
- */
- public static final String UI_USERS = "ui.users";
- public static final Object UI_USERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * List of DRPC servers so that the DRPCSpout knows who to talk to.
- */
- public static final String DRPC_SERVERS = "drpc.servers";
- public static final Object DRPC_SERVERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * This port is used by Storm DRPC for receiving HTTP DPRC requests from clients.
- */
- public static final String DRPC_HTTP_PORT = "drpc.http.port";
- public static final Object DRPC_HTTP_PORT_SCHEMA = Number.class;
-
- /**
- * This port is used by Storm DRPC for receiving HTTPS (SSL) DPRC requests from clients.
- */
- public static final String DRPC_HTTPS_PORT = "drpc.https.port";
- public static final Object DRPC_HTTPS_PORT_SCHEMA = Number.class;
-
- /**
- * Path to the keystore used by Storm DRPC for setting up HTTPS (SSL).
- */
- public static final String DRPC_HTTPS_KEYSTORE_PATH = "drpc.https.keystore.path";
- public static final Object DRPC_HTTPS_KEYSTORE_PATH_SCHEMA = String.class;
-
- /**
- * Password to the keystore used by Storm DRPC for setting up HTTPS (SSL).
- */
- public static final String DRPC_HTTPS_KEYSTORE_PASSWORD = "drpc.https.keystore.password";
- public static final Object DRPC_HTTPS_KEYSTORE_PASSWORD_SCHEMA = String.class;
-
- /**
- * Type of keystore used by Storm DRPC for setting up HTTPS (SSL).
- * see http://docs.oracle.com/javase/7/docs/api/java/security/KeyStore.html for more details.
- */
- public static final String DRPC_HTTPS_KEYSTORE_TYPE = "drpc.https.keystore.type";
- public static final Object DRPC_HTTPS_KEYSTORE_TYPE_SCHEMA = String.class;
-
- /**
- * The DRPC transport plug-in for Thrift client/server communication
- */
- public static final String DRPC_THRIFT_TRANSPORT_PLUGIN = "drpc.thrift.transport";
- public static final Object DRPC_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
-
- /**
- * This port is used by Storm DRPC for receiving DPRC requests from clients.
- */
- public static final String DRPC_PORT = "drpc.port";
- public static final Object DRPC_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Class name for authorization plugin for DRPC client
- */
- public static final String DRPC_AUTHORIZER = "drpc.authorizer";
- public static final Object DRPC_AUTHORIZER_SCHEMA = String.class;
-
- /**
- * The Access Control List for the DRPC Authorizer.
- * @see DRPCSimpleAclAuthorizer
- */
- public static final String DRPC_AUTHORIZER_ACL = "drpc.authorizer.acl";
- public static final Object DRPC_AUTHORIZER_ACL_SCHEMA = Map.class;
-
- /**
- * File name of the DRPC Authorizer ACL.
- * @see DRPCSimpleAclAuthorizer
- */
- public static final String DRPC_AUTHORIZER_ACL_FILENAME = "drpc.authorizer.acl.filename";
- public static final Object DRPC_AUTHORIZER_ACL_FILENAME_SCHEMA = String.class;
-
- /**
- * Whether the DRPCSimpleAclAuthorizer should deny requests for operations
- * involving functions that have no explicit ACL entry. When set to false
- * (the default) DRPC functions that have no entry in the ACL will be
- * permitted, which is appropriate for a development environment. When set
- * to true, explicit ACL entries are required for every DRPC function, and
- * any request for functions will be denied.
- * @see DRPCSimpleAclAuthorizer
- */
- public static final String DRPC_AUTHORIZER_ACL_STRICT = "drpc.authorizer.acl.strict";
- public static final Object DRPC_AUTHORIZER_ACL_STRICT_SCHEMA = Boolean.class;
-
- /**
- * DRPC thrift server worker threads
- */
- public static final String DRPC_WORKER_THREADS = "drpc.worker.threads";
- public static final Object DRPC_WORKER_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The maximum buffer size thrift should use when reading messages for DRPC.
- */
- public static final String DRPC_MAX_BUFFER_SIZE = "drpc.max_buffer_size";
- public static final Object DRPC_MAX_BUFFER_SIZE_SCHEMA = Number.class;
-
- /**
- * DRPC thrift server queue size
- */
- public static final String DRPC_QUEUE_SIZE = "drpc.queue.size";
- public static final Object DRPC_QUEUE_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The DRPC invocations transport plug-in for Thrift client/server communication
- */
- public static final String DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN = "drpc.invocations.thrift.transport";
- public static final Object DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
-
- /**
- * This port on Storm DRPC is used by DRPC topologies to receive function invocations and send results back.
- */
- public static final String DRPC_INVOCATIONS_PORT = "drpc.invocations.port";
- public static final Object DRPC_INVOCATIONS_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * DRPC invocations thrift server worker threads
- */
- public static final String DRPC_INVOCATIONS_THREADS = "drpc.invocations.threads";
- public static final Object DRPC_INVOCATIONS_THREADS_SCHEMA = Number.class;
-
- /**
- * The timeout on DRPC requests within the DRPC server. Defaults to 10 minutes. Note that requests can also
- * timeout based on the socket timeout on the DRPC client, and separately based on the topology message
- * timeout for the topology implementing the DRPC function.
- */
- public static final String DRPC_REQUEST_TIMEOUT_SECS = "drpc.request.timeout.secs";
- public static final Object DRPC_REQUEST_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Childopts for Storm DRPC Java process.
- */
- public static final String DRPC_CHILDOPTS = "drpc.childopts";
- public static final Object DRPC_CHILDOPTS_SCHEMA = String.class;
-
- /**
- * Class name of the HTTP credentials plugin for the UI.
- */
- public static final String UI_HTTP_CREDS_PLUGIN = "ui.http.creds.plugin";
- public static final Object UI_HTTP_CREDS_PLUGIN_SCHEMA = String.class;
-
- /**
- * Class name of the HTTP credentials plugin for DRPC.
- */
- public static final String DRPC_HTTP_CREDS_PLUGIN = "drpc.http.creds.plugin";
- public static final Object DRPC_HTTP_CREDS_PLUGIN_SCHEMA = String.class;
-
- /**
- * the metadata configured on the supervisor
- */
- public static final String SUPERVISOR_SCHEDULER_META = "supervisor.scheduler.meta";
- public static final Object SUPERVISOR_SCHEDULER_META_SCHEMA = Map.class;
- /**
- * A list of ports that can run workers on this supervisor. Each worker uses one port, and
- * the supervisor will only run one worker per port. Use this configuration to tune
- * how many workers run on each machine.
- */
- public static final String SUPERVISOR_SLOTS_PORTS = "supervisor.slots.ports";
- public static final Object SUPERVISOR_SLOTS_PORTS_SCHEMA = ConfigValidation.IntegersValidator;
-
- /**
- * A number representing the maximum number of workers any single topology can acquire.
- */
- public static final String NIMBUS_SLOTS_PER_TOPOLOGY = "nimbus.slots.perTopology";
- public static final Object NIMBUS_SLOTS_PER_TOPOLOGY_SCHEMA = Number.class;
-
- /**
- * A class implementing javax.servlet.Filter for DRPC HTTP requests
- */
- public static final String DRPC_HTTP_FILTER = "drpc.http.filter";
- public static final Object DRPC_HTTP_FILTER_SCHEMA = String.class;
-
- /**
- * Initialization parameters for the javax.servlet.Filter of the DRPC HTTP
- * service
- */
- public static final String DRPC_HTTP_FILTER_PARAMS = "drpc.http.filter.params";
- public static final Object DRPC_HTTP_FILTER_PARAMS_SCHEMA = Map.class;
-
- /**
- * A number representing the maximum number of executors any single topology can acquire.
- */
- public static final String NIMBUS_EXECUTORS_PER_TOPOLOGY = "nimbus.executors.perTopology";
- public static final Object NIMBUS_EXECUTORS_PER_TOPOLOGY_SCHEMA = Number.class;
-
- /**
- * This parameter is used by the storm-deploy project to configure the
- * jvm options for the supervisor daemon.
- */
- public static final String SUPERVISOR_CHILDOPTS = "supervisor.childopts";
- public static final Object SUPERVISOR_CHILDOPTS_SCHEMA = String.class;
-
- /**
- * How long a worker can go without heartbeating before the supervisor tries to
- * restart the worker process.
- */
- public static final String SUPERVISOR_WORKER_TIMEOUT_SECS = "supervisor.worker.timeout.secs";
- public static final Object SUPERVISOR_WORKER_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How long a worker can go without heartbeating during the initial launch before
- * the supervisor tries to restart the worker process. This value override
- * supervisor.worker.timeout.secs during launch because there is additional
- * overhead to starting and configuring the JVM on launch.
- */
- public static final String SUPERVISOR_WORKER_START_TIMEOUT_SECS = "supervisor.worker.start.timeout.secs";
- public static final Object SUPERVISOR_WORKER_START_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Whether or not the supervisor should launch workers assigned to it. Defaults
- * to true -- and you should probably never change this value. This configuration
- * is used in the Storm unit tests.
- */
- public static final String SUPERVISOR_ENABLE = "supervisor.enable";
- public static final Object SUPERVISOR_ENABLE_SCHEMA = Boolean.class;
-
- /**
- * how often the supervisor sends a heartbeat to the master.
- */
- public static final String SUPERVISOR_HEARTBEAT_FREQUENCY_SECS = "supervisor.heartbeat.frequency.secs";
- public static final Object SUPERVISOR_HEARTBEAT_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * How often the supervisor checks the worker heartbeats to see if any of them
- * need to be restarted.
- */
- public static final String SUPERVISOR_MONITOR_FREQUENCY_SECS = "supervisor.monitor.frequency.secs";
- public static final Object SUPERVISOR_MONITOR_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Should the supervior try to run the worker as the lauching user or not. Defaults to false.
- */
- public static final String SUPERVISOR_RUN_WORKER_AS_USER = "supervisor.run.worker.as.user";
- public static final Object SUPERVISOR_RUN_WORKER_AS_USER_SCHEMA = Boolean.class;
-
- /**
- * Full path to the worker-laucher executable that will be used to lauch workers when
- * SUPERVISOR_RUN_WORKER_AS_USER is set to true.
- */
- public static final String SUPERVISOR_WORKER_LAUNCHER = "supervisor.worker.launcher";
- public static final Object SUPERVISOR_WORKER_LAUNCHER_SCHEMA = String.class;
-
- /**
- * The jvm opts provided to workers launched by this supervisor. All "%ID%", "%WORKER-ID%", "%TOPOLOGY-ID%"
- * and "%WORKER-PORT%" substrings are replaced with:
- * %ID% -> port (for backward compatibility),
- * %WORKER-ID% -> worker-id,
- * %TOPOLOGY-ID% -> topology-id,
- * %WORKER-PORT% -> port.
- */
- public static final String WORKER_CHILDOPTS = "worker.childopts";
- public static final Object WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
-
- /**
- * The jvm opts provided to workers launched by this supervisor for GC. All "%ID%" substrings are replaced
- * with an identifier for this worker. Because the JVM complains about multiple GC opts the topology
- * can override this default value by setting topology.worker.gc.childopts.
- */
- public static final String WORKER_GC_CHILDOPTS = "worker.gc.childopts";
- public static final Object WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
-
- /**
- * control how many worker receiver threads we need per worker
- */
- public static final String WORKER_RECEIVER_THREAD_COUNT = "topology.worker.receiver.thread.count";
- public static final Object WORKER_RECEIVER_THREAD_COUNT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How often this worker should heartbeat to the supervisor.
- */
- public static final String WORKER_HEARTBEAT_FREQUENCY_SECS = "worker.heartbeat.frequency.secs";
- public static final Object WORKER_HEARTBEAT_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How often a task should heartbeat its status to the master.
- */
- public static final String TASK_HEARTBEAT_FREQUENCY_SECS = "task.heartbeat.frequency.secs";
- public static final Object TASK_HEARTBEAT_FREQUENCY_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * How often a task should sync its connections with other tasks (if a task is
- * reassigned, the other tasks sending messages to it need to refresh their connections).
- * In general though, when a reassignment happens other tasks will be notified
- * almost immediately. This configuration is here just in case that notification doesn't
- * come through.
- */
- public static final String TASK_REFRESH_POLL_SECS = "task.refresh.poll.secs";
- public static final Object TASK_REFRESH_POLL_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * How often a task should sync credentials, worst case.
- */
- public static final String TASK_CREDENTIALS_POLL_SECS = "task.credentials.poll.secs";
- public static final Object TASK_CREDENTIALS_POLL_SECS_SCHEMA = Number.class;
-
-
- /**
- * A list of users that are allowed to interact with the topology. To use this set
- * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
- */
- public static final String TOPOLOGY_USERS = "topology.users";
- public static final Object TOPOLOGY_USERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * A list of groups that are allowed to interact with the topology. To use this set
- * nimbus.authorizer to backtype.storm.security.auth.authorizer.SimpleACLAuthorizer
- */
- public static final String TOPOLOGY_GROUPS = "topology.groups";
- public static final Object TOPOLOGY_GROUPS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * True if Storm should timeout messages or not. Defaults to true. This is meant to be used
- * in unit tests to prevent tuples from being accidentally timed out during the test.
- */
- public static final String TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS = "topology.enable.message.timeouts";
- public static final Object TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS_SCHEMA = Boolean.class;
-
- /**
- * When set to true, Storm will log every message that's emitted.
- */
- public static final String TOPOLOGY_DEBUG = "topology.debug";
- public static final Object TOPOLOGY_DEBUG_SCHEMA = Boolean.class;
-
- /**
- * The serializer for communication between shell components and non-JVM
- * processes
- */
- public static final String TOPOLOGY_MULTILANG_SERIALIZER = "topology.multilang.serializer";
- public static final Object TOPOLOGY_MULTILANG_SERIALIZER_SCHEMA = String.class;
-
- /**
- * How many processes should be spawned around the cluster to execute this
- * topology. Each process will execute some number of tasks as threads within
- * them. This parameter should be used in conjunction with the parallelism hints
- * on each component in the topology to tune the performance of a topology.
- */
- public static final String TOPOLOGY_WORKERS = "topology.workers";
- public static final Object TOPOLOGY_WORKERS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How many instances to create for a spout/bolt. A task runs on a thread with zero or more
- * other tasks for the same spout/bolt. The number of tasks for a spout/bolt is always
- * the same throughout the lifetime of a topology, but the number of executors (threads) for
- * a spout/bolt can change over time. This allows a topology to scale to more or less resources
- * without redeploying the topology or violating the constraints of Storm (such as a fields grouping
- * guaranteeing that the same value goes to the same task).
- */
- public static final String TOPOLOGY_TASKS = "topology.tasks";
- public static final Object TOPOLOGY_TASKS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How many executors to spawn for ackers.
- *
- * <p>If this is set to 0, then Storm will immediately ack tuples as soon
- * as they come off the spout, effectively disabling reliability.</p>
- */
- public static final String TOPOLOGY_ACKER_EXECUTORS = "topology.acker.executors";
- public static final Object TOPOLOGY_ACKER_EXECUTORS_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * The maximum amount of time given to the topology to fully process a message
- * emitted by a spout. If the message is not acked within this time frame, Storm
- * will fail the message on the spout. Some spouts implementations will then replay
- * the message at a later time.
- */
- public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs";
- public static final Object TOPOLOGY_MESSAGE_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * A list of serialization registrations for Kryo ( http://code.google.com/p/kryo/ ),
- * the underlying serialization framework for Storm. A serialization can either
- * be the name of a class (in which case Kryo will automatically create a serializer for the class
- * that saves all the object's fields), or an implementation of com.esotericsoftware.kryo.Serializer.
- *
- * See Kryo's documentation for more information about writing custom serializers.
- */
- public static final String TOPOLOGY_KRYO_REGISTER = "topology.kryo.register";
- public static final Object TOPOLOGY_KRYO_REGISTER_SCHEMA = ConfigValidation.KryoRegValidator;
-
- /**
- * A list of classes that customize storm's kryo instance during start-up.
- * Each listed class name must implement IKryoDecorator. During start-up the
- * listed class is instantiated with 0 arguments, then its 'decorate' method
- * is called with storm's kryo instance as the only argument.
- */
- public static final String TOPOLOGY_KRYO_DECORATORS = "topology.kryo.decorators";
- public static final Object TOPOLOGY_KRYO_DECORATORS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * Class that specifies how to create a Kryo instance for serialization. Storm will then apply
- * topology.kryo.register and topology.kryo.decorators on top of this. The default implementation
- * implements topology.fall.back.on.java.serialization and turns references off.
- */
- public static final String TOPOLOGY_KRYO_FACTORY = "topology.kryo.factory";
- public static final Object TOPOLOGY_KRYO_FACTORY_SCHEMA = String.class;
-
-
- /**
- * Whether or not Storm should skip the loading of kryo registrations for which it
- * does not know the class or have the serializer implementation. Otherwise, the task will
- * fail to load and will throw an error at runtime. The use case of this is if you want to
- * declare your serializations on the storm.yaml files on the cluster rather than every single
- * time you submit a topology. Different applications may use different serializations and so
- * a single application may not have the code for the other serializers used by other apps.
- * By setting this config to true, Storm will ignore that it doesn't have those other serializations
- * rather than throw an error.
- */
- public static final String TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS= "topology.skip.missing.kryo.registrations";
- public static final Object TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS_SCHEMA = Boolean.class;
-
- /*
- * A list of classes implementing IMetricsConsumer (See storm.yaml.example for exact config format).
- * Each listed class will be routed all the metrics data generated by the storm metrics API.
- * Each listed class maps 1:1 to a system bolt named __metrics_ClassName#N, and it's parallelism is configurable.
- */
- public static final String TOPOLOGY_METRICS_CONSUMER_REGISTER = "topology.metrics.consumer.register";
- public static final Object TOPOLOGY_METRICS_CONSUMER_REGISTER_SCHEMA = ConfigValidation.MapsValidator;
-
-
- /**
- * The maximum parallelism allowed for a component in this topology. This configuration is
- * typically used in testing to limit the number of threads spawned in local mode.
- */
- public static final String TOPOLOGY_MAX_TASK_PARALLELISM="topology.max.task.parallelism";
- public static final Object TOPOLOGY_MAX_TASK_PARALLELISM_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * The maximum number of tuples that can be pending on a spout task at any given time.
- * This config applies to individual tasks, not to spouts or topologies as a whole.
- *
- * A pending tuple is one that has been emitted from a spout but has not been acked or failed yet.
- * Note that this config parameter has no effect for unreliable spouts that don't tag
- * their tuples with a message id.
- */
- public static final String TOPOLOGY_MAX_SPOUT_PENDING="topology.max.spout.pending";
- public static final Object TOPOLOGY_MAX_SPOUT_PENDING_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * A class that implements a strategy for what to do when a spout needs to wait. Waiting is
- * triggered in one of two conditions:
- *
- * 1. nextTuple emits no tuples
- * 2. The spout has hit maxSpoutPending and can't emit any more tuples
- */
- public static final String TOPOLOGY_SPOUT_WAIT_STRATEGY="topology.spout.wait.strategy";
- public static final Object TOPOLOGY_SPOUT_WAIT_STRATEGY_SCHEMA = String.class;
-
- /**
- * The amount of milliseconds the SleepEmptyEmitStrategy should sleep for.
- */
- public static final String TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS="topology.sleep.spout.wait.strategy.time.ms";
- public static final Object TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The maximum amount of time a component gives a source of state to synchronize before it requests
- * synchronization again.
- */
- public static final String TOPOLOGY_STATE_SYNCHRONIZATION_TIMEOUT_SECS="topology.state.synchronization.timeout.secs";
- public static final Object TOPOLOGY_STATE_SYNCHRONIZATION_TIMEOUT_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The percentage of tuples to sample to produce stats for a task.
- */
- public static final String TOPOLOGY_STATS_SAMPLE_RATE="topology.stats.sample.rate";
- public static final Object TOPOLOGY_STATS_SAMPLE_RATE_SCHEMA = ConfigValidation.DoubleValidator;
-
- /**
- * The time period that builtin metrics data in bucketed into.
- */
- public static final String TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS="topology.builtin.metrics.bucket.size.secs";
- public static final Object TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Whether or not to use Java serialization in a topology.
- */
- public static final String TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION="topology.fall.back.on.java.serialization";
- public static final Object TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION_SCHEMA = Boolean.class;
-
- /**
- * Topology-specific options for the worker child process. This is used in addition to WORKER_CHILDOPTS.
- */
- public static final String TOPOLOGY_WORKER_CHILDOPTS="topology.worker.childopts";
- public static final Object TOPOLOGY_WORKER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
-
- /**
- * Topology-specific options GC for the worker child process. This overrides WORKER_GC_CHILDOPTS.
- */
- public static final String TOPOLOGY_WORKER_GC_CHILDOPTS="topology.worker.gc.childopts";
- public static final Object TOPOLOGY_WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator;
-
- /**
- * Topology-specific classpath for the worker child process. This is combined to the usual classpath.
- */
- public static final String TOPOLOGY_CLASSPATH="topology.classpath";
- public static final Object TOPOLOGY_CLASSPATH_SCHEMA = ConfigValidation.StringOrStringListValidator;
-
- /**
- * Topology-specific environment variables for the worker child process.
- * This is added to the existing environment (that of the supervisor)
- */
- public static final String TOPOLOGY_ENVIRONMENT="topology.environment";
- public static final Object TOPOLOGY_ENVIRONMENT_SCHEMA = Map.class;
-
- /*
- * Topology-specific option to disable/enable bolt's outgoing overflow buffer.
- * Enabling this option ensures that the bolt can always clear the incoming messages,
- * preventing live-lock for the topology with cyclic flow.
- * The overflow buffer can fill degrading the performance gradually,
- * eventually running out of memory.
- */
- public static final String TOPOLOGY_BOLTS_OUTGOING_OVERFLOW_BUFFER_ENABLE="topology.bolts.outgoing.overflow.buffer.enable";
- public static final Object TOPOLOGY_BOLTS_OUTGOING_OVERFLOW_BUFFER_ENABLE_SCHEMA = Boolean.class;
-
- /**
- * This config is available for TransactionalSpouts, and contains the id ( a String) for
- * the transactional topology. This id is used to store the state of the transactional
- * topology in Zookeeper.
- */
- public static final String TOPOLOGY_TRANSACTIONAL_ID="topology.transactional.id";
- public static final Object TOPOLOGY_TRANSACTIONAL_ID_SCHEMA = String.class;
-
- /**
- * A list of task hooks that are automatically added to every spout and bolt in the topology. An example
- * of when you'd do this is to add a hook that integrates with your internal
- * monitoring system. These hooks are instantiated using the zero-arg constructor.
- */
- public static final String TOPOLOGY_AUTO_TASK_HOOKS="topology.auto.task.hooks";
- public static final Object TOPOLOGY_AUTO_TASK_HOOKS_SCHEMA = ConfigValidation.StringsValidator;
-
-
- /**
- * The size of the Disruptor receive queue for each executor. Must be a power of 2.
- */
- public static final String TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE="topology.executor.receive.buffer.size";
- public static final Object TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
-
- /**
- * The maximum number of messages to batch from the thread receiving off the network to the
- * executor queues. Must be a power of 2.
- */
- public static final String TOPOLOGY_RECEIVER_BUFFER_SIZE="topology.receiver.buffer.size";
- public static final Object TOPOLOGY_RECEIVER_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
-
- /**
- * The size of the Disruptor send queue for each executor. Must be a power of 2.
- */
- public static final String TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE="topology.executor.send.buffer.size";
- public static final Object TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE_SCHEMA = ConfigValidation.PowerOf2Validator;
-
- /**
- * The size of the Disruptor transfer queue for each worker.
- */
- public static final String TOPOLOGY_TRANSFER_BUFFER_SIZE="topology.transfer.buffer.size";
- public static final Object TOPOLOGY_TRANSFER_BUFFER_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How often a tick tuple from the "__system" component and "__tick" stream should be sent
- * to tasks. Meant to be used as a component-specific configuration.
- */
- public static final String TOPOLOGY_TICK_TUPLE_FREQ_SECS="topology.tick.tuple.freq.secs";
- public static final Object TOPOLOGY_TICK_TUPLE_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * Configure the wait strategy used for internal queuing. Can be used to tradeoff latency
- * vs. throughput
- */
- public static final String TOPOLOGY_DISRUPTOR_WAIT_STRATEGY="topology.disruptor.wait.strategy";
- public static final Object TOPOLOGY_DISRUPTOR_WAIT_STRATEGY_SCHEMA = String.class;
-
- /**
- * The size of the shared thread pool for worker tasks to make use of. The thread pool can be accessed
- * via the TopologyContext.
- */
- public static final String TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE="topology.worker.shared.thread.pool.size";
- public static final Object TOPOLOGY_WORKER_SHARED_THREAD_POOL_SIZE_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The interval in seconds to use for determining whether to throttle error reported to Zookeeper. For example,
- * an interval of 10 seconds with topology.max.error.report.per.interval set to 5 will only allow 5 errors to be
- * reported to Zookeeper per task for every 10 second interval of time.
- */
- public static final String TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS="topology.error.throttle.interval.secs";
- public static final Object TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * See doc for TOPOLOGY_ERROR_THROTTLE_INTERVAL_SECS
- */
- public static final String TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL="topology.max.error.report.per.interval";
- public static final Object TOPOLOGY_MAX_ERROR_REPORT_PER_INTERVAL_SCHEMA = ConfigValidation.IntegerValidator;
-
-
- /**
- * How often a batch can be emitted in a Trident topology.
- */
- public static final String TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS="topology.trident.batch.emit.interval.millis";
- public static final Object TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * Name of the topology. This config is automatically set by Storm when the topology is submitted.
- */
- public final static String TOPOLOGY_NAME="topology.name";
- public static final Object TOPOLOGY_NAME_SCHEMA = String.class;
-
- /**
- * The principal who submitted a topology
- */
- public final static String TOPOLOGY_SUBMITTER_PRINCIPAL = "topology.submitter.principal";
- public static final Object TOPOLOGY_SUBMITTER_PRINCIPAL_SCHEMA = String.class;
-
- /**
- * The local user name of the user who submitted a topology.
- */
- public static final String TOPOLOGY_SUBMITTER_USER = "topology.submitter.user";
- public static final Object TOPOLOGY_SUBMITTER_USER_SCHEMA = String.class;
-
- /**
- * Array of components that scheduler should try to place on separate hosts.
- */
- public static final String TOPOLOGY_SPREAD_COMPONENTS = "topology.spread.components";
- public static final Object TOPOLOGY_SPREAD_COMPONENTS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * A list of IAutoCredentials that the topology should load and use.
- */
- public static final String TOPOLOGY_AUTO_CREDENTIALS = "topology.auto-credentials";
- public static final Object TOPOLOGY_AUTO_CREDENTIALS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * Max pending tuples in one ShellBolt
- */
- public static final String TOPOLOGY_SHELLBOLT_MAX_PENDING="topology.shellbolt.max.pending";
- public static final Object TOPOLOGY_SHELLBOLT_MAX_PENDING_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The root directory in ZooKeeper for metadata about TransactionalSpouts.
- */
- public static final String TRANSACTIONAL_ZOOKEEPER_ROOT="transactional.zookeeper.root";
- public static final Object TRANSACTIONAL_ZOOKEEPER_ROOT_SCHEMA = String.class;
-
- /**
- * The list of zookeeper servers in which to keep the transactional state. If null (which is default),
- * will use storm.zookeeper.servers
- */
- public static final String TRANSACTIONAL_ZOOKEEPER_SERVERS="transactional.zookeeper.servers";
- public static final Object TRANSACTIONAL_ZOOKEEPER_SERVERS_SCHEMA = ConfigValidation.StringsValidator;
-
- /**
- * The port to use to connect to the transactional zookeeper servers. If null (which is default),
- * will use storm.zookeeper.port
- */
- public static final String TRANSACTIONAL_ZOOKEEPER_PORT="transactional.zookeeper.port";
- public static final Object TRANSACTIONAL_ZOOKEEPER_PORT_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The number of threads that should be used by the zeromq context in each worker process.
- */
- public static final String ZMQ_THREADS = "zmq.threads";
- public static final Object ZMQ_THREADS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * How long a connection should retry sending messages to a target host when
- * the connection is closed. This is an advanced configuration and can almost
- * certainly be ignored.
- */
- public static final String ZMQ_LINGER_MILLIS = "zmq.linger.millis";
- public static final Object ZMQ_LINGER_MILLIS_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * The high water for the ZeroMQ push sockets used for networking. Use this config to prevent buffer explosion
- * on the networking layer.
- */
- public static final String ZMQ_HWM = "zmq.hwm";
- public static final Object ZMQ_HWM_SCHEMA = ConfigValidation.IntegerValidator;
-
- /**
- * This value is passed to spawned JVMs (e.g., Nimbus, Supervisor, and Workers)
- * for the java.library.path value. java.library.path tells the JVM where
- * to look for native libraries. It is necessary to set this config correctly since
- * Storm uses the ZeroMQ and JZMQ native libs.
- */
- public static final String JAVA_LIBRARY_PATH = "java.library.path";
- public static final Object JAVA_LIBRARY_PATH_SCHEMA = String.class;
-
- /**
- * The path to use as the zookeeper dir when running a zookeeper server via
- * "storm dev-zookeeper". This zookeeper instance is only intended for development;
- * it is not a production grade zookeeper setup.
- */
- public static final String DEV_ZOOKEEPER_PATH = "dev.zookeeper.path";
- public static final Object DEV_ZOOKEEPER_PATH_SCHEMA = String.class;
-
- /**
- * A map from topology name to the number of machines that should be dedicated for that topology. Set storm.scheduler
- * to backtype.storm.scheduler.IsolationScheduler to make use of the isolation scheduler.
- */
- public static final String ISOLATION_SCHEDULER_MACHINES = "isolation.scheduler.machines";
- public static final Object ISOLATION_SCHEDULER_MACHINES_SCHEMA = ConfigValidation.MapOfStringToNumberValidator;
-
- /**
- * A map from the user name to the number of machines that should that user is allowed to use. Set storm.scheduler
- * to backtype.storm.scheduler.multitenant.MultitenantScheduler
- */
- public static final String MULTITENANT_SCHEDULER_USER_POOLS = "multitenant.scheduler.user.pools";
- public static final Object MULTITENANT_SCHEDULER_USER_POOLS_SCHEMA = ConfigValidation.MapOfStringToNumberValidator;
-
- /**
- * The number of machines that should be used by this topology to isolate it from all others. Set storm.scheduler
- * to backtype.storm.scheduler.multitenant.MultitenantScheduler
- */
- public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines";
- public static final Object TOPOLOGY_ISOLATED_MACHINES_SCHEMA = Number.class;
-
- /**
- * HDFS information, used to get the delegation token on behalf of the topology
- * submitter user and renew the tokens. see {@link backtype.storm.security.auth.hadoop.AutoHDFS}
- * kerberos principal name with realm should be provided.
- */
- public static final Object TOPOLOGY_HDFS_PRINCIPAL = "topology.hdfs.user";
- public static final Object TOPOLOGY_HDFS_PRINCIPAL_SCHEMA = String.class;
-
- /**
- * The HDFS URI to be used by AutoHDFS.java to grab the delegation token on topology
- * submitter user's behalf by the nimbus. If this is not provided the default URI provided
- * in the hdfs configuration files will be used.
- */
- public static final Object TOPOLOGY_HDFS_URI = "topology.hdfs.uri";
- public static final Object TOPOLOGY_HDFS_URI_SCHEMA = String.class;
-
- public static void setClasspath(Map conf, String cp) {
- conf.put(Config.TOPOLOGY_CLASSPATH, cp);
- }
-
- public void setClasspath(String cp) {
- setClasspath(this, cp);
- }
-
- public static void setEnvironment(Map conf, Map env) {
- conf.put(Config.TOPOLOGY_ENVIRONMENT, env);
- }
-
- public void setEnvironment(Map env) {
- setEnvironment(this, env);
- }
-
- public static void setDebug(Map conf, boolean isOn) {
- conf.put(Config.TOPOLOGY_DEBUG, isOn);
- }
-
- public void setDebug(boolean isOn) {
- setDebug(this, isOn);
- }
-
- public static void setNumWorkers(Map conf, int workers) {
- conf.put(Config.TOPOLOGY_WORKERS, workers);
- }
-
- public void setNumWorkers(int workers) {
- setNumWorkers(this, workers);
- }
-
- public static void setNumAckers(Map conf, int numExecutors) {
- conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, numExecutors);
- }
-
- public void setNumAckers(int numExecutors) {
- setNumAckers(this, numExecutors);
- }
-
- public static void setMessageTimeoutSecs(Map conf, int secs) {
- conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, secs);
- }
-
- public void setMessageTimeoutSecs(int secs) {
- setMessageTimeoutSecs(this, secs);
- }
-
- public static void registerSerialization(Map conf, Class klass) {
- getRegisteredSerializations(conf).add(klass.getName());
- }
-
- public void registerSerialization(Class klass) {
- registerSerialization(this, klass);
- }
-
- public static void registerSerialization(Map conf, Class klass, Class<? extends Serializer> serializerClass) {
- Map<String, String> register = new HashMap<String, String>();
- register.put(klass.getName(), serializerClass.getName());
- getRegisteredSerializations(conf).add(register);
- }
-
- public void registerSerialization(Class klass, Class<? extends Serializer> serializerClass) {
- registerSerialization(this, klass, serializerClass);
- }
-
- public static void registerMetricsConsumer(Map conf, Class klass, Object argument, long parallelismHint) {
- HashMap m = new HashMap();
- m.put("class", klass.getCanonicalName());
- m.put("parallelism.hint", parallelismHint);
- m.put("argument", argument);
-
- List l = (List)conf.get(TOPOLOGY_METRICS_CONSUMER_REGISTER);
- if (l == null) { l = new ArrayList(); }
- l.add(m);
- conf.put(TOPOLOGY_METRICS_CONSUMER_REGISTER, l);
- }
-
- public void registerMetricsConsumer(Class klass, Object argument, long parallelismHint) {
- registerMetricsConsumer(this, klass, argument, parallelismHint);
- }
-
- public static void registerMetricsConsumer(Map conf, Class klass, long parallelismHint) {
- registerMetricsConsumer(conf, klass, null, parallelismHint);
- }
-
- public void registerMetricsConsumer(Class klass, long parallelismHint) {
- registerMetricsConsumer(this, klass, parallelismHint);
- }
-
- public static void registerMetricsConsumer(Map conf, Class klass) {
- registerMetricsConsumer(conf, klass, null, 1L);
- }
-
- public void registerMetricsConsumer(Class klass) {
- registerMetricsConsumer(this, klass);
- }
-
- public static void registerDecorator(Map conf, Class<? extends IKryoDecorator> klass) {
- getRegisteredDecorators(conf).add(klass.getName());
- }
-
- public void registerDecorator(Class<? extends IKryoDecorator> klass) {
- registerDecorator(this, klass);
- }
-
- public static void setKryoFactory(Map conf, Class<? extends IKryoFactory> klass) {
- conf.put(Config.TOPOLOGY_KRYO_FACTORY, klass.getName());
- }
-
- public void setKryoFactory(Class<? extends IKryoFactory> klass) {
- setKryoFactory(this, klass);
- }
-
- public static void setSkipMissingKryoRegistrations(Map conf, boolean skip) {
- conf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, skip);
- }
-
- public void setSkipMissingKryoRegistrations(boolean skip) {
- setSkipMissingKryoRegistrations(this, skip);
- }
-
- public static void setMaxTaskParallelism(Map conf, int max) {
- conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, max);
- }
-
- public void setMaxTaskParallelism(int max) {
- setMaxTaskParallelism(this, max);
- }
-
- public static void setMaxSpoutPending(Map conf, int max) {
- conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, max);
- }
-
- public void setMaxSpoutPending(int max) {
- setMaxSpoutPending(this, max);
- }
-
- public static void setStatsSampleRate(Map conf, double rate) {
- conf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, rate);
- }
-
- public void setStatsSampleRate(double rate) {
- setStatsSampleRate(this, rate);
- }
-
- public static void setFallBackOnJavaSerialization(Map conf, boolean fallback) {
- conf.put(Config.TOPOLOGY_FALL_BACK_ON_JAVA_SERIALIZATION, fallback);
- }
-
- public void setFallBackOnJavaSerialization(boolean fallback) {
- setFallBackOnJavaSerialization(this, fallback);
- }
-
- private static List getRegisteredSerializations(Map conf) {
- List ret;
- if(!conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) {
- ret = new ArrayList();
- } else {
- ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_REGISTER));
- }
- conf.put(Config.TOPOLOGY_KRYO_REGISTER, ret);
- return ret;
- }
-
- private static List getRegisteredDecorators(Map conf) {
- List ret;
- if(!conf.containsKey(Config.TOPOLOGY_KRYO_DECORATORS)) {
- ret = new ArrayList();
- } else {
- ret = new ArrayList((List) conf.get(Config.TOPOLOGY_KRYO_DECORATORS));
- }
- conf.put(Config.TOPOLOGY_KRYO_DECORATORS, ret);
- return ret;
- }
-}
[12/18] storm git commit: Merge remote-tracking branch
'origin/master' into storm-596
Posted by bo...@apache.org.
Merge remote-tracking branch 'origin/master' into storm-596
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f2067b64
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f2067b64
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f2067b64
Branch: refs/heads/master
Commit: f2067b64fa4bb9bda8ec8647aa25d80560c3821a
Parents: 986ad60 232e4ac
Author: caofangkun <ca...@gmail.com>
Authored: Wed Apr 29 08:53:11 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Apr 29 08:53:11 2015 +0800
----------------------------------------------------------------------
.gitignore | 4 +
.travis.yml | 9 +
BYLAWS.md | 96 -
CHANGELOG.md | 123 +
DEVELOPER.md | 36 +-
LICENSE | 146 +-
README.markdown | 42 +-
SECURITY.md | 154 +-
STORM-UI-REST-API.md | 96 +-
bin/storm | 573 +-
bin/storm-config.cmd | 38 +-
bin/storm.cmd | 38 +-
bin/storm.py | 553 +
conf/defaults.yaml | 10 +-
conf/storm-env.sh | 24 +
dev-tools/github/__init__.py | 11 +
dev-tools/jira-github-join.py | 4 +-
dev-tools/storm-merge.py | 31 +
dev-tools/test-ns.py | 34 +
.../print-errors-from-clojure-test-reports.py | 58 +
dev-tools/travis/travis-build.sh | 50 +
docs/README.md | 9 +
docs/_includes/header.html | 13 +-
docs/_posts/2015-03-25-storm094-released.md | 24 +
docs/about/integrates.md | 2 +-
docs/about/multi-language.md | 6 +-
docs/about/simple-api.md | 2 +-
docs/css/main.scss | 3 +-
.../Acking-framework-implementation.md | 8 +-
docs/documentation/BYLAWS.md | 98 +
docs/documentation/Clojure-DSL.md | 10 +-
docs/documentation/Command-line-client.md | 2 +-
docs/documentation/Common-patterns.md | 22 +-
docs/documentation/Concepts.md | 57 +-
docs/documentation/Configuration.md | 6 +-
docs/documentation/Contributing-to-Storm.md | 2 +-
.../Creating-a-new-Storm-project.md | 6 +-
.../DSLs-and-multilang-adapters.md | 3 +-
...Defining-a-non-jvm-language-dsl-for-storm.md | 2 +-
docs/documentation/Distributed-RPC.md | 4 +-
docs/documentation/FAQ.md | 4 +-
.../Guaranteeing-message-processing.md | 6 +-
docs/documentation/Home.md | 2 +-
docs/documentation/Hooks.md | 6 +-
docs/documentation/Kestrel-and-Storm.md | 2 +-
docs/documentation/Lifecycle-of-a-topology.md | 70 +-
docs/documentation/Local-mode.md | 4 +-
docs/documentation/Maven.md | 50 +-
docs/documentation/Multilang-protocol.md | 71 +-
docs/documentation/Powered-By.md | 123 +-
...unning-topologies-on-a-production-cluster.md | 6 +-
.../Serialization-(prior-to-0.6.0).md | 6 +-
docs/documentation/Serialization.md | 2 +-
.../documentation/Setting-up-a-Storm-cluster.md | 11 +-
.../Setting-up-development-environment.md | 2 +-
docs/documentation/Structure-of-the-codebase.md | 96 +-
docs/documentation/Transactional-topologies.md | 24 +-
docs/documentation/Trident-API-Overview.md | 8 +-
docs/documentation/Trident-spouts.md | 8 +-
docs/documentation/Trident-state.md | 12 +-
docs/documentation/Tutorial.md | 12 +-
...nding-the-parallelism-of-a-Storm-topology.md | 16 +-
docs/downloads.html | 67 +-
docs/images/architecture.png | Bin 0 -> 69825 bytes
docs/images/architecture.svg | 1458 ++
docs/images/bolt.png | Bin 0 -> 24796 bytes
docs/images/bolt.svg | 743 +
docs/images/security.png | Bin 0 -> 72415 bytes
docs/images/security.svg | 1779 +++
docs/images/spout.png | Bin 0 -> 22911 bytes
docs/images/spout.svg | 833 ++
docs/images/storm.svg | 1326 ++
docs/images/topology.png | Bin 59837 -> 64740 bytes
docs/images/topology.svg | 1044 ++
docs/images/topology_dark.png | Bin 0 -> 49692 bytes
docs/images/topology_dark.svg | 1101 ++
examples/storm-starter/README.markdown | 8 +-
.../storm-starter/multilang/resources/storm.js | 373 -
.../storm-starter/multilang/resources/storm.py | 260 -
.../storm-starter/multilang/resources/storm.rb | 236 -
examples/storm-starter/pom.xml | 69 +-
.../jvm/storm/starter/BasicDRPCTopology.java | 3 +-
.../storm/starter/SkewedRollingTopWords.java | 134 +
.../storm/starter/bolt/AbstractRankerBolt.java | 4 +-
.../storm/starter/bolt/RollingCountAggBolt.java | 77 +
.../storm/starter/bolt/RollingCountBolt.java | 4 +-
.../jvm/storm/starter/util/TupleHelpers.java | 33 -
external/README.md | 18 +
external/storm-eventhubs/README.md | 41 +
external/storm-eventhubs/pom.xml | 122 +
.../storm/eventhubs/bolt/EventHubBolt.java | 81 +
.../client/ConnectionStringBuilder.java | 116 +
.../storm/eventhubs/client/Constants.java | 32 +
.../storm/eventhubs/client/EventHubClient.java | 92 +
.../eventhubs/client/EventHubConsumerGroup.java | 72 +
.../eventhubs/client/EventHubException.java | 37 +
.../eventhubs/client/EventHubReceiver.java | 139 +
.../eventhubs/client/EventHubSendClient.java | 70 +
.../storm/eventhubs/client/EventHubSender.java | 95 +
.../storm/eventhubs/client/SelectorFilter.java | 38 +
.../eventhubs/client/SelectorFilterWriter.java | 64 +
.../eventhubs/samples/AtMostOnceEventCount.java | 54 +
.../storm/eventhubs/samples/EventCount.java | 155 +
.../storm/eventhubs/samples/EventHubLoop.java | 51 +
.../samples/OpaqueTridentEventCount.java | 53 +
.../samples/TransactionalTridentEventCount.java | 81 +
.../eventhubs/samples/bolt/GlobalCountBolt.java | 83 +
.../samples/bolt/PartialCountBolt.java | 63 +
.../apache/storm/eventhubs/spout/EventData.java | 48 +
.../storm/eventhubs/spout/EventDataScheme.java | 55 +
.../eventhubs/spout/EventHubReceiverFilter.java | 56 +
.../eventhubs/spout/EventHubReceiverImpl.java | 150 +
.../storm/eventhubs/spout/EventHubSpout.java | 258 +
.../eventhubs/spout/EventHubSpoutConfig.java | 165 +
.../eventhubs/spout/EventHubSpoutException.java | 37 +
.../storm/eventhubs/spout/FieldConstants.java | 25 +
.../storm/eventhubs/spout/IEventDataScheme.java | 30 +
.../eventhubs/spout/IEventHubReceiver.java | 35 +
.../spout/IEventHubReceiverFactory.java | 30 +
.../spout/IEventHubReceiverFilter.java | 35 +
.../eventhubs/spout/IPartitionCoordinator.java | 27 +
.../eventhubs/spout/IPartitionManager.java | 37 +
.../spout/IPartitionManagerFactory.java | 33 +
.../storm/eventhubs/spout/IStateStore.java | 31 +
.../apache/storm/eventhubs/spout/MessageId.java | 56 +
.../storm/eventhubs/spout/PartitionManager.java | 101 +
.../eventhubs/spout/SimplePartitionManager.java | 136 +
.../spout/StaticPartitionCoordinator.java | 85 +
.../eventhubs/spout/ZookeeperStateStore.java | 95 +
.../storm/eventhubs/trident/Coordinator.java | 60 +
.../trident/ITridentPartitionManager.java | 35 +
.../ITridentPartitionManagerFactory.java | 26 +
.../trident/OpaqueTridentEventHubEmitter.java | 69 +
.../trident/OpaqueTridentEventHubSpout.java | 64 +
.../storm/eventhubs/trident/Partition.java | 39 +
.../storm/eventhubs/trident/Partitions.java | 41 +
.../TransactionalTridentEventHubEmitter.java | 167 +
.../TransactionalTridentEventHubSpout.java | 66 +
.../trident/TridentPartitionManager.java | 91 +
.../src/main/resources/config.properties | 27 +
.../eventhubs/spout/EventHubReceiverMock.java | 105 +
.../spout/EventHubSpoutCallerMock.java | 96 +
.../spout/PartitionManagerCallerMock.java | 105 +
.../spout/SpoutOutputCollectorMock.java | 61 +
.../storm/eventhubs/spout/StateStoreMock.java | 54 +
.../storm/eventhubs/spout/TestEventData.java | 47 +
.../eventhubs/spout/TestEventHubSpout.java | 70 +
.../eventhubs/spout/TestPartitionManager.java | 117 +
.../TestTransactionalTridentEmitter.java | 93 +
.../eventhubs/trident/TridentCollectorMock.java | 52 +
external/storm-hbase/README.md | 49 +-
external/storm-hbase/pom.xml | 19 +-
.../storm/hbase/bolt/AbstractHBaseBolt.java | 9 +-
.../org/apache/storm/hbase/bolt/HBaseBolt.java | 2 +-
.../storm/hbase/bolt/HBaseLookupBolt.java | 4 +-
.../apache/storm/hbase/security/AutoHBase.java | 243 +
.../storm/hbase/security/HBaseSecurityUtil.java | 32 +-
.../mapper/SimpleTridentHBaseMapMapper.java | 50 +
.../trident/mapper/TridentHBaseMapMapper.java | 40 +
.../hbase/trident/state/HBaseMapState.java | 49 +-
.../storm/hbase/trident/state/HBaseState.java | 13 +-
external/storm-hdfs/README.md | 45 +
external/storm-hdfs/pom.xml | 20 +-
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 2 +-
.../storm/hdfs/bolt/SequenceFileBolt.java | 2 +-
.../storm/hdfs/bolt/format/SequenceFormat.java | 5 +-
.../storm/hdfs/common/security/AutoHDFS.java | 281 +
.../hdfs/common/security/HdfsSecurityUtil.java | 30 +-
.../storm/hdfs/bolt/HdfsFileTopology.java | 6 +-
.../storm/hdfs/bolt/SequenceFileTopology.java | 4 +-
.../storm/hdfs/trident/TridentFileTopology.java | 2 +-
.../hdfs/trident/TridentSequenceTopology.java | 6 +-
external/storm-hive/README.md | 113 +
external/storm-hive/pom.xml | 143 +
.../org/apache/storm/hive/bolt/HiveBolt.java | 289 +
.../bolt/mapper/DelimitedRecordHiveMapper.java | 143 +
.../storm/hive/bolt/mapper/HiveMapper.java | 81 +
.../hive/bolt/mapper/JsonRecordHiveMapper.java | 132 +
.../apache/storm/hive/common/HiveOptions.java | 146 +
.../org/apache/storm/hive/common/HiveUtils.java | 76 +
.../apache/storm/hive/common/HiveWriter.java | 420 +
.../apache/storm/hive/trident/HiveState.java | 306 +
.../storm/hive/trident/HiveStateFactory.java | 31 +
.../apache/storm/hive/trident/HiveUpdater.java | 14 +
.../apache/storm/hive/bolt/HiveSetupUtil.java | 220 +
.../apache/storm/hive/bolt/HiveTopology.java | 150 +
.../hive/bolt/HiveTopologyPartitioned.java | 153 +
.../apache/storm/hive/bolt/TestHiveBolt.java | 330 +
.../storm/hive/common/TestHiveWriter.java | 193 +
.../storm/hive/trident/TridentHiveTopology.java | 190 +
external/storm-jdbc/LICENSE | 202 +
external/storm-jdbc/README.md | 264 +
external/storm-jdbc/pom.xml | 125 +
.../storm/jdbc/bolt/AbstractJdbcBolt.java | 57 +
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 95 +
.../apache/storm/jdbc/bolt/JdbcLookupBolt.java | 76 +
.../org/apache/storm/jdbc/common/Column.java | 112 +
.../apache/storm/jdbc/common/JdbcClient.java | 240 +
.../java/org/apache/storm/jdbc/common/Util.java | 75 +
.../storm/jdbc/mapper/JdbcLookupMapper.java | 26 +
.../apache/storm/jdbc/mapper/JdbcMapper.java | 33 +
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 46 +
.../storm/jdbc/mapper/SimpleJdbcMapper.java | 92 +
.../storm/jdbc/trident/state/JdbcQuery.java | 40 +
.../storm/jdbc/trident/state/JdbcState.java | 161 +
.../jdbc/trident/state/JdbcStateFactory.java | 40 +
.../storm/jdbc/trident/state/JdbcUpdater.java | 32 +
.../storm/jdbc/common/JdbcClientTest.java | 93 +
.../org/apache/storm/jdbc/common/UtilTest.java | 69 +
.../org/apache/storm/jdbc/spout/UserSpout.java | 90 +
.../jdbc/topology/AbstractUserTopology.java | 106 +
.../jdbc/topology/UserPersistanceTopology.java | 48 +
.../UserPersistanceTridentTopology.java | 61 +
external/storm-jdbc/src/test/sql/test.sql | 1 +
external/storm-kafka/README.md | 68 +-
external/storm-kafka/pom.xml | 23 +-
.../ExponentialBackoffMsgRetryManager.java | 167 +
.../jvm/storm/kafka/FailedMsgRetryManager.java | 26 +
.../src/jvm/storm/kafka/KafkaConfig.java | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 33 +-
.../src/jvm/storm/kafka/PartitionManager.java | 57 +-
.../src/jvm/storm/kafka/SpoutConfig.java | 8 +
.../kafka/TopicOffsetOutOfRangeException.java | 25 +
.../jvm/storm/kafka/UpdateOffsetException.java | 22 -
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 13 +-
.../FieldNameBasedTupleToKafkaMapper.java | 2 +-
.../kafka/trident/TridentKafkaEmitter.java | 47 +-
.../ExponentialBackoffMsgRetryManagerTest.java | 194 +
.../src/test/storm/kafka/KafkaUtilsTest.java | 8 +-
.../test/storm/kafka/bolt/KafkaBoltTest.java | 29 +-
external/storm-redis/LICENSE | 202 +
external/storm-redis/README.md | 241 +
external/storm-redis/pom.xml | 65 +
.../storm/redis/bolt/AbstractRedisBolt.java | 67 +
.../storm/redis/bolt/RedisLookupBolt.java | 112 +
.../apache/storm/redis/bolt/RedisStoreBolt.java | 100 +
.../redis/common/config/JedisClusterConfig.java | 82 +
.../redis/common/config/JedisPoolConfig.java | 97 +
.../common/container/JedisClusterContainer.java | 47 +
.../JedisCommandsContainerBuilder.java | 38 +
.../JedisCommandsInstanceContainer.java | 25 +
.../redis/common/container/JedisContainer.java | 60 +
.../common/mapper/RedisDataTypeDescription.java | 50 +
.../redis/common/mapper/RedisLookupMapper.java | 40 +
.../storm/redis/common/mapper/RedisMapper.java | 22 +
.../redis/common/mapper/RedisStoreMapper.java | 21 +
.../storm/redis/common/mapper/TupleMapper.java | 27 +
.../trident/state/RedisClusterMapState.java | 294 +
.../redis/trident/state/RedisClusterState.java | 80 +
.../trident/state/RedisClusterStateQuerier.java | 78 +
.../trident/state/RedisClusterStateUpdater.java | 76 +
.../redis/trident/state/RedisMapState.java | 316 +
.../storm/redis/trident/state/RedisState.java | 83 +
.../redis/trident/state/RedisStateQuerier.java | 70 +
.../redis/trident/state/RedisStateUpdater.java | 75 +
.../storm/redis/topology/LookupWordCount.java | 166 +
.../redis/topology/PersistentWordCount.java | 116 +
.../storm/redis/topology/WordCounter.java | 67 +
.../apache/storm/redis/topology/WordSpout.java | 88 +
.../storm/redis/trident/PrintFunction.java | 40 +
.../redis/trident/WordCountTridentRedis.java | 96 +
.../trident/WordCountTridentRedisCluster.java | 103 +
.../WordCountTridentRedisClusterMap.java | 99 +
.../redis/trident/WordCountTridentRedisMap.java | 92 +
.../redis/trident/WordCountTupleMapper.java | 16 +
pom.xml | 41 +-
.../maven-shade-clojure-transformer/pom.xml | 2 +-
storm-buildtools/storm-maven-plugins/pom.xml | 81 +
.../storm/maven/plugin/util/CommandExec.java | 89 +
.../plugin/versioninfo/VersionInfoMojo.java | 304 +
storm-core/pom.xml | 93 +-
storm-core/src/clj/backtype/storm/bootstrap.clj | 64 -
storm-core/src/clj/backtype/storm/clojure.clj | 2 +-
storm-core/src/clj/backtype/storm/cluster.clj | 97 +-
storm-core/src/clj/backtype/storm/config.clj | 8 +-
storm-core/src/clj/backtype/storm/converter.clj | 205 +
.../src/clj/backtype/storm/daemon/common.clj | 15 +-
.../src/clj/backtype/storm/daemon/drpc.clj | 27 +-
.../src/clj/backtype/storm/daemon/executor.clj | 39 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 94 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 193 +-
.../clj/backtype/storm/daemon/supervisor.clj | 280 +-
.../src/clj/backtype/storm/daemon/task.clj | 27 +-
.../src/clj/backtype/storm/daemon/worker.clj | 88 +-
.../src/clj/backtype/storm/local_state.clj | 99 +
.../src/clj/backtype/storm/messaging/local.clj | 2 +-
storm-core/src/clj/backtype/storm/stats.clj | 78 +-
storm-core/src/clj/backtype/storm/testing.clj | 25 +-
storm-core/src/clj/backtype/storm/thrift.clj | 34 +-
storm-core/src/clj/backtype/storm/tuple.clj | 4 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 255 +-
.../src/clj/backtype/storm/ui/helpers.clj | 86 +-
storm-core/src/clj/backtype/storm/util.clj | 51 +-
storm-core/src/dev/resources/storm.js | 373 -
storm-core/src/dev/resources/storm.py | 260 -
storm-core/src/dev/resources/storm.rb | 236 -
storm-core/src/genthrift.sh | 13 +-
storm-core/src/java_license_header.txt | 17 +
storm-core/src/jvm/backtype/storm/Config.java | 171 +-
.../jvm/backtype/storm/ConfigValidation.java | 8 +-
.../src/jvm/backtype/storm/StormSubmitter.java | 88 +-
.../storm/coordination/BatchBoltExecutor.java | 4 +-
.../coordination/BatchSubtopologyBuilder.java | 11 +
.../storm/drpc/DRPCInvocationsClient.java | 6 +
.../src/jvm/backtype/storm/drpc/DRPCSpout.java | 22 +-
.../storm/drpc/LinearDRPCInputDeclarer.java | 5 +-
.../storm/drpc/LinearDRPCTopologyBuilder.java | 13 +-
.../jvm/backtype/storm/drpc/ReturnResults.java | 8 +-
.../storm/generated/AlreadyAliveException.java | 149 +-
.../backtype/storm/generated/Assignment.java | 983 ++
.../storm/generated/AuthorizationException.java | 149 +-
.../src/jvm/backtype/storm/generated/Bolt.java | 194 +-
.../jvm/backtype/storm/generated/BoltStats.java | 908 +-
.../storm/generated/ClusterSummary.java | 339 +-
.../storm/generated/ClusterWorkerHeartbeat.java | 769 +
.../storm/generated/ComponentCommon.java | 426 +-
.../storm/generated/ComponentObject.java | 86 +-
.../backtype/storm/generated/Credentials.java | 220 +-
.../storm/generated/DRPCExecutionException.java | 149 +-
.../backtype/storm/generated/DRPCRequest.java | 185 +-
.../storm/generated/DistributedRPC.java | 529 +-
.../generated/DistributedRPCInvocations.java | 1199 +-
.../jvm/backtype/storm/generated/ErrorInfo.java | 300 +-
.../backtype/storm/generated/ExecutorInfo.java | 198 +-
.../storm/generated/ExecutorSpecificStats.java | 72 +-
.../backtype/storm/generated/ExecutorStats.java | 589 +-
.../storm/generated/ExecutorSummary.java | 371 +-
.../storm/generated/GetInfoOptions.java | 422 +
.../storm/generated/GlobalStreamId.java | 185 +-
.../jvm/backtype/storm/generated/Grouping.java | 163 +-
.../generated/InvalidTopologyException.java | 149 +-
.../backtype/storm/generated/JavaObject.java | 239 +-
.../backtype/storm/generated/JavaObjectArg.java | 108 +-
.../backtype/storm/generated/KillOptions.java | 176 +-
.../storm/generated/LSApprovedWorkers.java | 458 +
.../generated/LSSupervisorAssignments.java | 471 +
.../storm/generated/LSSupervisorId.java | 406 +
.../storm/generated/LSWorkerHeartbeat.java | 755 +
.../storm/generated/LocalAssignment.java | 561 +
.../storm/generated/LocalStateData.java | 471 +
.../jvm/backtype/storm/generated/Nimbus.java | 13125 ++++++++++++-----
.../jvm/backtype/storm/generated/NodeInfo.java | 556 +
.../storm/generated/NotAliveException.java | 149 +-
.../backtype/storm/generated/NullStruct.java | 112 +-
.../storm/generated/NumErrorsChoice.java | 65 +
.../storm/generated/RebalanceOptions.java | 348 +-
.../storm/generated/ShellComponent.java | 202 +-
.../jvm/backtype/storm/generated/SpoutSpec.java | 194 +-
.../backtype/storm/generated/SpoutStats.java | 614 +-
.../storm/generated/StateSpoutSpec.java | 194 +-
.../jvm/backtype/storm/generated/StormBase.java | 1211 ++
.../backtype/storm/generated/StormTopology.java | 410 +-
.../backtype/storm/generated/StreamInfo.java | 249 +-
.../backtype/storm/generated/SubmitOptions.java | 208 +-
.../storm/generated/SupervisorInfo.java | 1288 ++
.../storm/generated/SupervisorSummary.java | 424 +-
.../storm/generated/ThriftSerializedObject.java | 516 +
.../storm/generated/TopologyActionOptions.java | 387 +
.../backtype/storm/generated/TopologyInfo.java | 609 +-
.../storm/generated/TopologyInitialStatus.java | 3 +-
.../storm/generated/TopologyStatus.java | 68 +
.../storm/generated/TopologySummary.java | 486 +-
.../storm/grouping/PartialKeyGrouping.java | 82 +
.../storm/messaging/ConnectionWithStatus.java | 32 +
.../backtype/storm/messaging/netty/Client.java | 720 +-
.../backtype/storm/messaging/netty/Context.java | 33 +-
.../messaging/netty/SaslStormClientHandler.java | 5 +-
.../backtype/storm/messaging/netty/Server.java | 184 +-
.../netty/StormClientPipelineFactory.java | 5 +-
.../auth/DefaultHttpCredentialsPlugin.java | 19 +-
.../storm/security/auth/ITransportPlugin.java | 4 +-
.../storm/security/auth/ReqContext.java | 28 +-
.../security/auth/SaslTransportPlugin.java | 20 +-
.../security/auth/SimpleTransportPlugin.java | 10 +-
.../storm/security/auth/TBackoffConnect.java | 4 +-
.../storm/security/auth/ThriftClient.java | 12 +-
.../authorizer/ImpersonationAuthorizer.java | 154 +
.../auth/authorizer/SimpleACLAuthorizer.java | 55 +-
.../auth/digest/DigestSaslTransportPlugin.java | 6 +-
.../auth/digest/ServerCallbackHandler.java | 21 +-
.../storm/security/auth/hadoop/AutoHDFS.java | 262 -
.../kerberos/KerberosSaslTransportPlugin.java | 9 +-
.../auth/kerberos/ServerCallbackHandler.java | 38 +-
.../DefaultSerializationDelegate.java | 11 +-
.../GzipBridgeSerializationDelegate.java | 7 +-
.../GzipBridgeThriftSerializationDelegate.java | 64 +
.../GzipSerializationDelegate.java | 10 +-
.../GzipThriftSerializationDelegate.java | 57 +
.../serialization/SerializationDelegate.java | 2 +-
.../ThriftSerializationDelegate.java | 52 +
.../src/jvm/backtype/storm/task/IBolt.java | 4 +-
.../src/jvm/backtype/storm/task/ShellBolt.java | 5 +-
.../backtype/storm/task/TopologyContext.java | 104 +-
.../backtype/storm/topology/BoltDeclarer.java | 4 +
.../backtype/storm/topology/InputDeclarer.java | 131 +
.../storm/topology/TopologyBuilder.java | 17 +-
.../TransactionalTopologyBuilder.java | 13 +-
.../jvm/backtype/storm/utils/DRPCClient.java | 2 +-
.../storm/utils/ExtendedThreadPoolExecutor.java | 67 +
.../jvm/backtype/storm/utils/LocalState.java | 163 +-
.../src/jvm/backtype/storm/utils/Monitor.java | 8 +-
.../jvm/backtype/storm/utils/NimbusClient.java | 26 +-
.../jvm/backtype/storm/utils/ShellProcess.java | 11 +-
.../jvm/backtype/storm/utils/TupleUtils.java | 35 +
.../src/jvm/backtype/storm/utils/Utils.java | 250 +-
.../jvm/backtype/storm/utils/VersionInfo.java | 131 +
.../backtype/storm/utils/VersionedStore.java | 9 +-
storm-core/src/jvm/storm/trident/Stream.java | 2 +-
.../src/jvm/storm/trident/TridentTopology.java | 6 +-
.../trident/drpc/ReturnResultsReducer.java | 4 +-
.../trident/spout/RichSpoutBatchExecutor.java | 1 +
.../trident/topology/TridentBoltExecutor.java | 3 +-
.../topology/TridentTopologyBuilder.java | 13 +-
storm-core/src/multilang/js/storm.js | 366 -
storm-core/src/multilang/py/storm.py | 260 -
storm-core/src/multilang/rb/storm.rb | 236 -
.../src/native/worker-launcher/configure.ac | 2 +-
.../worker-launcher/impl/worker-launcher.c | 7 +-
storm-core/src/py/__init__.py | 2 +
storm-core/src/py/storm/DistributedRPC-remote | 35 +-
storm-core/src/py/storm/DistributedRPC.py | 40 +-
.../py/storm/DistributedRPCInvocations-remote | 43 +-
.../src/py/storm/DistributedRPCInvocations.py | 97 +-
storm-core/src/py/storm/Nimbus-remote | 114 +-
storm-core/src/py/storm/Nimbus.py | 850 +-
storm-core/src/py/storm/__init__.py | 2 +
storm-core/src/py/storm/constants.py | 8 +-
storm-core/src/py/storm/ttypes.py | 2032 ++-
storm-core/src/py_license_header.txt | 18 +
.../storm-core-version-info.properties | 24 +
storm-core/src/storm.thrift | 105 +
storm-core/src/ui/public/component.html | 167 +-
.../src/ui/public/css/bootstrap-1.4.0.css | 2469 ----
.../src/ui/public/css/bootstrap-3.3.1.min.css | 5 +
.../src/ui/public/css/dataTables.bootstrap.css | 319 +
.../public/css/jquery.dataTables.1.10.4.min.css | 1 +
.../src/ui/public/css/jsonFormatter.min.css | 1 +
storm-core/src/ui/public/css/style.css | 129 +-
.../src/ui/public/images/back_disabled.png | Bin 0 -> 1361 bytes
.../src/ui/public/images/back_enabled.png | Bin 0 -> 1379 bytes
.../src/ui/public/images/back_enabled_hover.png | Bin 0 -> 1375 bytes
.../src/ui/public/images/forward_disabled.png | Bin 0 -> 1363 bytes
.../src/ui/public/images/forward_enabled.png | Bin 0 -> 1380 bytes
.../ui/public/images/forward_enabled_hover.png | Bin 0 -> 1379 bytes
storm-core/src/ui/public/images/sort_asc.png | Bin 0 -> 1118 bytes
.../src/ui/public/images/sort_asc_disabled.png | Bin 0 -> 2916 bytes
storm-core/src/ui/public/images/sort_both.png | Bin 0 -> 1136 bytes
storm-core/src/ui/public/images/sort_desc.png | Bin 0 -> 1127 bytes
.../src/ui/public/images/sort_desc_disabled.png | Bin 0 -> 1045 bytes
storm-core/src/ui/public/index.html | 103 +-
.../src/ui/public/js/bootstrap-3.3.1.min.js | 7 +
storm-core/src/ui/public/js/bootstrap-twipsy.js | 310 -
.../ui/public/js/dataTables.bootstrap.min.js | 8 +
.../src/ui/public/js/jquery-1.11.1.min.js | 4 +
storm-core/src/ui/public/js/jquery-1.6.2.min.js | 18 -
.../public/js/jquery.dataTables.1.10.4.min.js | 157 +
.../src/ui/public/js/jquery.tablesorter.min.js | 5 -
.../src/ui/public/js/jsonFormatter.min.js | 2 +
storm-core/src/ui/public/js/script.js | 91 +-
.../public/templates/anti-forgery-template.html | 19 -
.../templates/component-page-template.html | 132 +-
.../public/templates/index-page-template.html | 58 +-
.../templates/topology-page-template.html | 94 +-
.../src/ui/public/templates/user-template.html | 2 +-
storm-core/src/ui/public/topology.html | 137 +-
.../test/clj/backtype/storm/clojure_test.clj | 9 +-
.../test/clj/backtype/storm/cluster_test.clj | 25 +-
.../test/clj/backtype/storm/drpc_test.clj | 8 +-
.../test/clj/backtype/storm/grouping_test.clj | 9 +-
.../clj/backtype/storm/integration_test.clj | 20 +-
.../clj/backtype/storm/local_state_test.clj | 40 +-
.../test/clj/backtype/storm/logviewer_test.clj | 47 +-
.../storm/messaging/netty_integration_test.clj | 5 +-
.../storm/messaging/netty_unit_test.clj | 73 +-
.../test/clj/backtype/storm/messaging_test.clj | 6 +-
.../test/clj/backtype/storm/metrics_test.clj | 8 +-
.../test/clj/backtype/storm/multilang_test.clj | 6 +-
.../test/clj/backtype/storm/nimbus_test.clj | 27 +-
.../scheduler/multitenant_scheduler_test.clj | 6 +-
.../test/clj/backtype/storm/scheduler_test.clj | 4 +-
.../storm/security/auth/AuthUtils_test.clj | 6 +-
.../auth/DefaultHttpCredentialsPlugin_test.clj | 16 +-
.../backtype/storm/security/auth/auth_test.clj | 167 +-
.../storm/security/auth/drpc_auth_test.clj | 7 +-
.../storm/security/auth/nimbus_auth_test.clj | 243 +-
.../clj/backtype/storm/subtopology_test.clj | 9 +-
.../test/clj/backtype/storm/supervisor_test.clj | 56 +-
.../test/clj/backtype/storm/tick_tuple_test.clj | 7 +-
.../clj/backtype/storm/transactional_test.clj | 20 +-
.../test/clj/backtype/storm/worker_test.clj | 37 +
.../storm/grouping/PartialKeyGroupingTest.java | 66 +
.../GzipBridgeSerializationDelegateTest.java | 6 +-
...ipBridgeThriftSerializationDelegateTest.java | 71 +
.../ThriftBridgeSerializationDelegateTest.java | 60 +
.../storm/utils/DisruptorQueueTest.java | 38 +-
storm-core/test/resources/logback-test.xml | 26 +
storm-dist/binary/pom.xml | 2 +-
storm-dist/binary/src/main/assembly/binary.xml | 82 +-
storm-dist/source/pom.xml | 2 +-
storm-multilang/javascript/pom.xml | 32 +
.../src/main/resources/resources/storm.js | 373 +
storm-multilang/python/pom.xml | 32 +
.../src/main/resources/resources/storm.py | 260 +
storm-multilang/ruby/pom.xml | 32 +
.../ruby/src/main/resources/resources/storm.rb | 236 +
505 files changed, 61174 insertions(+), 17085 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f2067b64/conf/defaults.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f2067b64/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/f2067b64/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
[03/18] storm git commit: Merge pull request #12 from apache/master
Posted by bo...@apache.org.
Merge pull request #12 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/f95938b3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/f95938b3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/f95938b3
Branch: refs/heads/master
Commit: f95938b3d973a9d2513d0e326188f77b9bf438c8
Parents: a7c3c27 87a6627
Author: caofangkun <ca...@gmail.com>
Authored: Mon Mar 16 14:31:04 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Mon Mar 16 14:31:04 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 1 +
docs/documentation/Powered-By.md | 12 ++++++-
.../backtype/storm/messaging/netty/Client.java | 6 +++-
.../backtype/storm/messaging/netty/Context.java | 33 ++++++++++++++------
4 files changed, 40 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[16/18] storm git commit: STORM-596:delete extra ')' in function
loader#launch-receive-thread!
Posted by bo...@apache.org.
STORM-596:delete extra ')' in function loader#launch-receive-thread!
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2b839cb4
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2b839cb4
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2b839cb4
Branch: refs/heads/master
Commit: 2b839cb4b29e13029e72c582ff96e7dd873680a6
Parents: 7ab8b96
Author: caofangkun <ca...@gmail.com>
Authored: Wed Apr 29 10:00:18 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Apr 29 10:00:18 2015 +0800
----------------------------------------------------------------------
pom.xml | 2 --
1 file changed, 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2b839cb4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ad5e73e..2e0c898 100644
--- a/pom.xml
+++ b/pom.xml
@@ -161,7 +161,6 @@
<module>storm-multilang/python</module>
<module>storm-multilang/ruby</module>
<module>storm-core</module>
- <!--
<module>examples/storm-starter</module>
<module>external/storm-kafka</module>
<module>external/storm-hdfs</module>
@@ -170,7 +169,6 @@
<module>external/storm-jdbc</module>
<module>external/storm-redis</module>
<module>external/storm-eventhubs</module>
- -->
</modules>
<scm>
[10/18] storm git commit: Merge pull request #19 from apache/master
Posted by bo...@apache.org.
Merge pull request #19 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bb8583ac
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bb8583ac
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bb8583ac
Branch: refs/heads/master
Commit: bb8583aca92fc563e0bd750d2783be59ff507173
Parents: a3602f2 84e8bc6
Author: caofangkun <ca...@gmail.com>
Authored: Tue Apr 21 14:27:13 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Tue Apr 21 14:27:13 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 6 ++++
LICENSE | 29 ++++++++++++++++++++
external/storm-kafka/README.md | 21 +++++++++++++-
.../src/jvm/storm/kafka/KafkaConfig.java | 2 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 5 +---
.../src/jvm/storm/kafka/PartitionManager.java | 4 +--
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 1 +
.../kafka/trident/TridentKafkaEmitter.java | 4 +--
.../src/test/storm/kafka/KafkaUtilsTest.java | 6 ++--
.../test/storm/kafka/bolt/KafkaBoltTest.java | 27 ++++++++++++++++++
.../src/clj/backtype/storm/daemon/executor.clj | 6 +++-
storm-core/src/clj/backtype/storm/ui/core.clj | 2 +-
.../src/ui/public/css/jsonFormatter.min.css | 1 +
storm-core/src/ui/public/css/style.css | 11 ++++++++
storm-core/src/ui/public/index.html | 3 ++
.../src/ui/public/js/jsonFormatter.min.js | 2 ++
storm-core/src/ui/public/js/script.js | 2 +-
storm-core/src/ui/public/topology.html | 3 ++
.../clj/backtype/storm/transactional_test.clj | 6 +++-
19 files changed, 124 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[17/18] storm git commit: Merge branch 'storm-596' of
https://github.com/caofangkun/apache-storm into STORM-596
Posted by bo...@apache.org.
Merge branch 'storm-596' of https://github.com/caofangkun/apache-storm into STORM-596
STORM-596: remove config topology.receiver.buffer.size
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/77bc461d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/77bc461d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/77bc461d
Branch: refs/heads/master
Commit: 77bc461da9ae8131026b3cc43c091572f80ffb2e
Parents: 972ab85 2b839cb
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri May 22 14:46:03 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri May 22 14:46:03 2015 -0500
----------------------------------------------------------------------
conf/defaults.yaml | 1 -
storm-core/src/clj/backtype/storm/daemon/worker.clj | 1 -
storm-core/src/clj/backtype/storm/messaging/loader.clj | 13 ++++++-------
storm-core/src/jvm/backtype/storm/Config.java | 7 -------
4 files changed, 6 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[05/18] storm git commit: Merge pull request #14 from apache/master
Posted by bo...@apache.org.
Merge pull request #14 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/57a691c8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/57a691c8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/57a691c8
Branch: refs/heads/master
Commit: 57a691c865f764cf331abd2e2e6d27bd320813f8
Parents: 146bd6b bb8d48d
Author: caofangkun <ca...@gmail.com>
Authored: Thu Mar 19 10:01:23 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Thu Mar 19 10:01:23 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 1 +
conf/defaults.yaml | 2 +-
storm-core/src/clj/backtype/storm/cluster.clj | 58 +-
storm-core/src/clj/backtype/storm/config.clj | 7 +-
storm-core/src/clj/backtype/storm/converter.clj | 201 +++
.../src/clj/backtype/storm/daemon/common.clj | 10 +-
.../src/clj/backtype/storm/daemon/executor.clj | 4 +-
.../src/clj/backtype/storm/daemon/nimbus.clj | 106 +-
.../src/clj/backtype/storm/daemon/worker.clj | 3 +-
storm-core/src/clj/backtype/storm/stats.clj | 78 +-
storm-core/src/clj/backtype/storm/thrift.clj | 17 +-
.../storm/coordination/BatchBoltExecutor.java | 4 +-
.../backtype/storm/generated/Assignment.java | 983 ++++++++++++++
.../storm/generated/ClusterWorkerHeartbeat.java | 673 ++++++++++
.../backtype/storm/generated/ExecutorStats.java | 105 +-
.../jvm/backtype/storm/generated/NodeInfo.java | 556 ++++++++
.../jvm/backtype/storm/generated/StormBase.java | 1211 ++++++++++++++++++
.../storm/generated/SupervisorInfo.java | 1182 +++++++++++++++++
.../storm/generated/TopologyActionOptions.java | 387 ++++++
.../storm/generated/TopologyStatus.java | 68 +
.../DefaultSerializationDelegate.java | 11 +-
.../GzipBridgeSerializationDelegate.java | 7 +-
.../GzipSerializationDelegate.java | 10 +-
.../serialization/SerializationDelegate.java | 2 +-
.../ThriftSerializationDelegate.java | 52 +
.../storm/topology/TopologyBuilder.java | 6 +-
.../jvm/backtype/storm/utils/LocalState.java | 4 +-
.../src/jvm/backtype/storm/utils/Utils.java | 82 +-
storm-core/src/jvm/storm/trident/Stream.java | 2 +-
.../src/jvm/storm/trident/TridentTopology.java | 6 +-
storm-core/src/py/storm/ttypes.py | 827 +++++++++++-
storm-core/src/storm.thrift | 51 +
.../test/clj/backtype/storm/cluster_test.clj | 25 +-
.../test/clj/backtype/storm/nimbus_test.clj | 6 +-
.../GzipBridgeSerializationDelegateTest.java | 6 +-
.../ThriftBridgeSerializationDelegateTest.java | 60 +
36 files changed, 6607 insertions(+), 206 deletions(-)
----------------------------------------------------------------------
[04/18] storm git commit: Merge pull request #13 from apache/master
Posted by bo...@apache.org.
Merge pull request #13 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/146bd6b7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/146bd6b7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/146bd6b7
Branch: refs/heads/master
Commit: 146bd6b7b18f03b338df78431654d7c54fa0e8a4
Parents: f95938b 1ea378d
Author: caofangkun <ca...@gmail.com>
Authored: Wed Mar 18 09:17:37 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Mar 18 09:17:37 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 5 ++
DEVELOPER.md | 7 ++
README.markdown | 1 +
SECURITY.md | 23 ++++++
external/storm-jdbc/README.md | 84 +++++++++++++-------
.../apache/storm/jdbc/bolt/JdbcInsertBolt.java | 36 +++++++--
.../org/apache/storm/jdbc/common/Column.java | 3 +-
.../apache/storm/jdbc/common/JdbcClient.java | 52 +++++++-----
.../jdbc/mapper/SimpleJdbcLookupMapper.java | 2 +-
.../storm/jdbc/trident/state/JdbcState.java | 18 ++++-
.../storm/jdbc/common/JdbcClientTest.java | 39 +++++----
.../jdbc/topology/UserPersistanceTopology.java | 2 +-
storm-core/pom.xml | 3 +-
storm-core/src/genthrift.sh | 13 ++-
storm-core/src/java_license_header.txt | 17 ++++
storm-core/src/jvm/backtype/storm/Config.java | 20 ++++-
.../backtype/storm/messaging/netty/Client.java | 4 +
.../auth/authorizer/SimpleACLAuthorizer.java | 42 +++++++---
storm-core/src/py/storm/Nimbus-remote | 0
storm-core/src/py_license_header.txt | 18 +++++
.../backtype/storm/security/auth/auth_test.clj | 16 ++++
21 files changed, 312 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
[14/18] storm git commit: STORM-596:delete extra ')' in function
loader#launch-receive-thread!
Posted by bo...@apache.org.
STORM-596:delete extra ')' in function loader#launch-receive-thread!
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/42bcf300
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/42bcf300
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/42bcf300
Branch: refs/heads/master
Commit: 42bcf300e1adfe16ba35a4727a7ae12723a25ca8
Parents: f2067b6
Author: caofangkun <ca...@gmail.com>
Authored: Wed Apr 29 09:01:44 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Apr 29 09:01:44 2015 +0800
----------------------------------------------------------------------
storm-core/src/clj/backtype/storm/daemon/worker.clj | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/42bcf300/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj
index c38a6d6..2943684 100644
--- a/storm-core/src/clj/backtype/storm/daemon/worker.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj
@@ -384,7 +384,7 @@
(:receiver-thread-count worker)
(:port worker)
(:transfer-local-fn worker))
- :kill-fn (fn [t] (exit-process! 11))))
+ :kill-fn (fn [t] (exit-process! 11)))
(defn- close-resources [worker]
(let [dr (:default-shared-resources worker)]
[15/18] storm git commit: STORM-596:delete extra ')' in function
loader#launch-receive-thread!
Posted by bo...@apache.org.
STORM-596:delete extra ')' in function loader#launch-receive-thread!
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7ab8b965
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7ab8b965
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7ab8b965
Branch: refs/heads/master
Commit: 7ab8b965ff27de917e6338f5b054946240d21c93
Parents: 42bcf30
Author: caofangkun <ca...@gmail.com>
Authored: Wed Apr 29 09:59:43 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Wed Apr 29 09:59:43 2015 +0800
----------------------------------------------------------------------
pom.xml | 2 ++
storm-core/src/clj/backtype/storm/daemon/worker.clj | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7ab8b965/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2e0c898..ad5e73e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -161,6 +161,7 @@
<module>storm-multilang/python</module>
<module>storm-multilang/ruby</module>
<module>storm-core</module>
+ <!--
<module>examples/storm-starter</module>
<module>external/storm-kafka</module>
<module>external/storm-hdfs</module>
@@ -169,6 +170,7 @@
<module>external/storm-jdbc</module>
<module>external/storm-redis</module>
<module>external/storm-eventhubs</module>
+ -->
</modules>
<scm>
http://git-wip-us.apache.org/repos/asf/storm/blob/7ab8b965/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/worker.clj b/storm-core/src/clj/backtype/storm/daemon/worker.clj
index 2943684..b5d80cc 100644
--- a/storm-core/src/clj/backtype/storm/daemon/worker.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/worker.clj
@@ -383,8 +383,8 @@
(:storm-id worker)
(:receiver-thread-count worker)
(:port worker)
- (:transfer-local-fn worker))
- :kill-fn (fn [t] (exit-process! 11)))
+ (:transfer-local-fn worker)
+ :kill-fn (fn [t] (exit-process! 11))))
(defn- close-resources [worker]
(let [dr (:default-shared-resources worker)]
[06/18] storm git commit: Merge pull request #15 from apache/master
Posted by bo...@apache.org.
Merge pull request #15 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2c5b4023
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2c5b4023
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2c5b4023
Branch: refs/heads/master
Commit: 2c5b402381dbffd3942c0b67cdb8013462b5552a
Parents: 57a691c dcee1e2
Author: caofangkun <ca...@gmail.com>
Authored: Tue Mar 24 10:44:47 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Tue Mar 24 10:44:47 2015 +0800
----------------------------------------------------------------------
BYLAWS.md | 95 -
CHANGELOG.md | 1 +
LICENSE | 117 +-
docs/_includes/header.html | 13 +-
docs/css/main.scss | 3 +-
docs/documentation/BYLAWS.md | 98 +
docs/documentation/Powered-By.md | 2 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 47 +-
.../src/clj/backtype/storm/ui/helpers.clj | 21 -
storm-core/src/ui/public/component.html | 167 +-
.../src/ui/public/css/bootstrap-1.4.0.css | 2469 ------------------
.../src/ui/public/css/bootstrap-3.3.1.min.css | 5 +
.../src/ui/public/css/dataTables.bootstrap.css | 319 +++
.../public/css/jquery.dataTables.1.10.4.min.css | 1 +
storm-core/src/ui/public/css/style.css | 94 +-
.../src/ui/public/images/back_disabled.png | Bin 0 -> 1361 bytes
.../src/ui/public/images/back_enabled.png | Bin 0 -> 1379 bytes
.../src/ui/public/images/back_enabled_hover.png | Bin 0 -> 1375 bytes
.../src/ui/public/images/forward_disabled.png | Bin 0 -> 1363 bytes
.../src/ui/public/images/forward_enabled.png | Bin 0 -> 1380 bytes
.../ui/public/images/forward_enabled_hover.png | Bin 0 -> 1379 bytes
storm-core/src/ui/public/images/sort_asc.png | Bin 0 -> 1118 bytes
.../src/ui/public/images/sort_asc_disabled.png | Bin 0 -> 2916 bytes
storm-core/src/ui/public/images/sort_both.png | Bin 0 -> 1136 bytes
storm-core/src/ui/public/images/sort_desc.png | Bin 0 -> 1127 bytes
.../src/ui/public/images/sort_desc_disabled.png | Bin 0 -> 1045 bytes
storm-core/src/ui/public/index.html | 99 +-
.../src/ui/public/js/bootstrap-3.3.1.min.js | 7 +
storm-core/src/ui/public/js/bootstrap-twipsy.js | 310 ---
.../ui/public/js/dataTables.bootstrap.min.js | 8 +
.../src/ui/public/js/jquery-1.11.1.min.js | 4 +
storm-core/src/ui/public/js/jquery-1.6.2.min.js | 18 -
.../public/js/jquery.dataTables.1.10.4.min.js | 157 ++
.../src/ui/public/js/jquery.tablesorter.min.js | 5 -
storm-core/src/ui/public/js/script.js | 86 +-
.../templates/component-page-template.html | 128 +-
.../public/templates/index-page-template.html | 52 +-
.../templates/topology-page-template.html | 90 +-
.../src/ui/public/templates/user-template.html | 2 +-
storm-core/src/ui/public/topology.html | 132 +-
40 files changed, 1191 insertions(+), 3359 deletions(-)
----------------------------------------------------------------------
[18/18] storm git commit: Added STORM-596 to Changelog
Posted by bo...@apache.org.
Added STORM-596 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2195fd60
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2195fd60
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2195fd60
Branch: refs/heads/master
Commit: 2195fd606c101ed949e12e29b0d9e2bf9024978e
Parents: 77bc461
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri May 22 14:54:12 2015 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri May 22 14:54:12 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/2195fd60/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08a66cc..53d6486 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 0.11.0
+ * STORM-596: remove config topology.receiver.buffer.size
* STORM-808: allow null to be parsed as null
* STORM-816: maven-gpg-plugin does not work with gpg 2.1
* STORM-827: Allow AutoTGT to work with storm-hdfs too.
[07/18] storm git commit: Merge pull request #16 from apache/master
Posted by bo...@apache.org.
Merge pull request #16 from apache/master
Merge from apache/storm to caofangkun/apache-storm
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/2bc9447c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/2bc9447c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/2bc9447c
Branch: refs/heads/master
Commit: 2bc9447c78fc6945d71db3772bdb49edad153f07
Parents: 2c5b402 330e135
Author: caofangkun <ca...@gmail.com>
Authored: Mon Mar 30 10:46:28 2015 +0800
Committer: caofangkun <ca...@gmail.com>
Committed: Mon Mar 30 10:46:28 2015 +0800
----------------------------------------------------------------------
CHANGELOG.md | 6 +
README.markdown | 1 +
docs/_posts/2015-03-25-storm094-released.md | 24 ++++
docs/downloads.html | 64 +++++++----
examples/storm-starter/pom.xml | 2 +-
external/storm-hbase/pom.xml | 2 +-
external/storm-hdfs/pom.xml | 2 +-
external/storm-hive/pom.xml | 2 +-
external/storm-jdbc/pom.xml | 2 +-
external/storm-kafka/pom.xml | 2 +-
external/storm-redis/pom.xml | 2 +-
.../storm/redis/bolt/AbstractRedisBolt.java | 8 +-
.../storm/redis/bolt/RedisLookupBolt.java | 112 ++++++++++++++++++
.../apache/storm/redis/bolt/RedisStoreBolt.java | 100 ++++++++++++++++
.../redis/common/config/JedisClusterConfig.java | 82 +++++++++++++
.../redis/common/config/JedisPoolConfig.java | 97 ++++++++++++++++
.../common/container/JedisClusterContainer.java | 47 ++++++++
.../JedisCommandsContainerBuilder.java | 38 ++++++
.../JedisCommandsInstanceContainer.java | 25 ++++
.../redis/common/container/JedisContainer.java | 65 +++++++++++
.../common/mapper/RedisDataTypeDescription.java | 50 ++++++++
.../redis/common/mapper/RedisLookupMapper.java | 40 +++++++
.../storm/redis/common/mapper/RedisMapper.java | 22 ++++
.../redis/common/mapper/RedisStoreMapper.java | 21 ++++
.../storm/redis/common/mapper/TupleMapper.java | 27 +++++
.../trident/mapper/TridentTupleMapper.java | 27 -----
.../trident/state/RedisClusterMapState.java | 2 +-
.../redis/trident/state/RedisClusterState.java | 2 +-
.../trident/state/RedisClusterStateQuerier.java | 10 +-
.../trident/state/RedisClusterStateUpdater.java | 10 +-
.../redis/trident/state/RedisMapState.java | 21 +---
.../storm/redis/trident/state/RedisState.java | 2 +-
.../redis/trident/state/RedisStateQuerier.java | 10 +-
.../state/RedisStateSetCountQuerier.java | 74 ------------
.../trident/state/RedisStateSetUpdater.java | 80 -------------
.../redis/trident/state/RedisStateUpdater.java | 10 +-
.../redis/util/config/JedisClusterConfig.java | 82 -------------
.../redis/util/config/JedisPoolConfig.java | 97 ----------------
.../util/container/JedisClusterContainer.java | 47 --------
.../JedisCommandsContainerBuilder.java | 38 ------
.../JedisCommandsInstanceContainer.java | 25 ----
.../redis/util/container/JedisContainer.java | 65 -----------
.../storm/redis/topology/LookupWordCount.java | 115 +++++++++++++------
.../redis/topology/PersistentWordCount.java | 46 +++++++-
.../storm/redis/topology/WordCounter.java | 19 ++-
.../redis/trident/WordCountTridentRedis.java | 7 +-
.../trident/WordCountTridentRedisCluster.java | 6 +-
.../WordCountTridentRedisClusterMap.java | 8 +-
.../redis/trident/WordCountTridentRedisMap.java | 9 +-
.../redis/trident/WordCountTupleMapper.java | 10 +-
pom.xml | 2 +-
.../maven-shade-clojure-transformer/pom.xml | 2 +-
storm-buildtools/storm-maven-plugins/pom.xml | 2 +-
storm-core/pom.xml | 2 +-
storm-core/src/ui/public/css/style.css | 62 ++++++++++
storm-dist/binary/pom.xml | 2 +-
storm-dist/source/pom.xml | 2 +-
57 files changed, 1056 insertions(+), 683 deletions(-)
----------------------------------------------------------------------