You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2015/08/24 15:52:01 UTC
[16/50] [abbrv] storm git commit: Merge remote-tracking branch
'upstream/master' into STORM-166
Merge remote-tracking branch 'upstream/master' into STORM-166
Conflicts:
storm-core/src/jvm/backtype/storm/Config.java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/58667be9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/58667be9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/58667be9
Branch: refs/heads/master
Commit: 58667be951623848406f699f82cafa79a40cb496
Parents: 5658bbb e71e2a3
Author: Parth Brahmbhatt <br...@gmail.com>
Authored: Thu Jan 8 11:59:22 2015 -0500
Committer: Parth Brahmbhatt <br...@gmail.com>
Committed: Thu Jan 8 11:59:22 2015 -0500
----------------------------------------------------------------------
CHANGELOG.md | 14 +
DEVELOPER.md | 5 +-
README.markdown | 18 +-
SECURITY.md | 25 +-
bin/storm | 59 ++--
conf/defaults.yaml | 1 +
docs/about/integrates.md | 2 +-
docs/about/multi-language.md | 6 +-
docs/about/simple-api.md | 2 +-
.../Acking-framework-implementation.md | 8 +-
docs/documentation/Clojure-DSL.md | 6 +-
docs/documentation/Common-patterns.md | 2 +-
docs/documentation/Configuration.md | 4 +-
.../Creating-a-new-Storm-project.md | 6 +-
.../DSLs-and-multilang-adapters.md | 3 +-
...Defining-a-non-jvm-language-dsl-for-storm.md | 2 +-
docs/documentation/Distributed-RPC.md | 2 +-
docs/documentation/FAQ.md | 4 +-
docs/documentation/Kestrel-and-Storm.md | 2 +-
docs/documentation/Lifecycle-of-a-topology.md | 70 ++---
docs/documentation/Maven.md | 50 +---
docs/documentation/Multilang-protocol.md | 30 +-
.../Serialization-(prior-to-0.6.0).md | 2 +-
.../documentation/Setting-up-a-Storm-cluster.md | 4 +-
.../Setting-up-development-environment.md | 2 +-
docs/documentation/Structure-of-the-codebase.md | 88 +++---
docs/documentation/Transactional-topologies.md | 12 +-
docs/documentation/Trident-API-Overview.md | 4 +-
docs/documentation/Trident-spouts.md | 8 +-
docs/documentation/Trident-state.md | 12 +-
docs/documentation/Tutorial.md | 4 +-
docs/downloads.html | 2 +-
external/storm-hbase/README.md | 49 +++-
external/storm-hbase/pom.xml | 17 +-
.../storm/hbase/bolt/AbstractHBaseBolt.java | 9 +-
.../apache/storm/hbase/security/AutoHBase.java | 243 ++++++++++++++++
.../storm/hbase/security/HBaseSecurityUtil.java | 32 ++-
.../storm/hbase/trident/state/HBaseState.java | 9 +-
external/storm-hdfs/README.md | 45 +++
.../storm/hdfs/bolt/format/SequenceFormat.java | 5 +-
.../storm/hdfs/common/security/AutoHDFS.java | 281 +++++++++++++++++++
.../hdfs/common/security/HdfsSecurityUtil.java | 30 +-
.../src/jvm/storm/kafka/KafkaUtils.java | 12 +-
.../src/jvm/storm/kafka/PartitionManager.java | 6 +-
.../kafka/TopicOffsetOutOfRangeException.java | 25 ++
.../jvm/storm/kafka/UpdateOffsetException.java | 22 --
.../FieldNameBasedTupleToKafkaMapper.java | 2 +-
.../kafka/trident/TridentKafkaEmitter.java | 43 ++-
.../src/test/storm/kafka/KafkaUtilsTest.java | 2 +-
.../src/clj/backtype/storm/daemon/logviewer.clj | 36 ++-
.../src/clj/backtype/storm/daemon/nimbus.clj | 7 +-
.../clj/backtype/storm/daemon/supervisor.clj | 22 +-
storm-core/src/clj/backtype/storm/ui/core.clj | 1 +
storm-core/src/jvm/backtype/storm/Config.java | 38 +--
.../storm/security/auth/hadoop/AutoHDFS.java | 262 -----------------
.../jvm/backtype/storm/utils/ShellProcess.java | 11 +-
.../src/native/worker-launcher/configure.ac | 2 +-
.../worker-launcher/impl/worker-launcher.c | 7 +-
.../test/clj/backtype/storm/logviewer_test.clj | 47 +++-
.../backtype/storm/security/auth/auth_test.clj | 8 +-
.../test/clj/backtype/storm/supervisor_test.clj | 6 +-
61 files changed, 1108 insertions(+), 630 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/conf/defaults.yaml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index c96dd88,9d2f313..3fcf7eb
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -296,9 -294,12 +296,13 @@@
(log-error t "Error when processing event")
(exit-process! 20 "Error when processing an event")
))
+ :event-timer (mk-timer :kill-fn (fn [t]
+ (log-error t "Error when processing event")
+ (exit-process! 20 "Error when processing an event")
+ ))
:assignment-versions (atom {})
:sync-retry (atom 0)
+ :bt-tracker (mk-bt-tracker conf)
})
(defn sync-processes [supervisor]
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/Config.java
index 4678177,3a241ba..df330a0
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@@ -1234,54 -1246,6 +1240,36 @@@ public class Config extends HashMap<Str
public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines";
public static final Object TOPOLOGY_ISOLATED_MACHINES_SCHEMA = Number.class;
+ /**
- * HDFS information, used to get the delegation token on behalf of the topology
- * submitter user and renew the tokens. see {@link backtype.storm.security.auth.hadoop.AutoHDFS}
- * kerberos principal name with realm should be provided.
- */
- public static final Object TOPOLOGY_HDFS_PRINCIPAL = "topology.hdfs.user";
- public static final Object TOPOLOGY_HDFS_PRINCIPAL_SCHEMA = String.class;
-
- /**
- * The HDFS URI to be used by AutoHDFS.java to grab the delegation token on topology
- * submitter user's behalf by the nimbus. If this is not provided the default URI provided
- * in the hdfs configuration files will be used.
- */
- public static final Object TOPOLOGY_HDFS_URI = "topology.hdfs.uri";
- public static final Object TOPOLOGY_HDFS_URI_SCHEMA = String.class;
-
- /**
+ * Which implementation of {@link backtype.storm.codedistributor.ICodeDistributor} should be used by storm for code
+ * distribution.
+ */
+ public static final String STORM_CODE_DISTRIBUTOR_CLASS = "storm.codedistributor.class";
+ public static final Object STORM_CODE_DISTRIBUTOR_CLASS_SCHEMA = String.class;
+
+ /**
+ * Minimum number of nimbus hosts where the code must be replicated before leader nimbus
+ * is allowed to perform topology activation tasks like setting up heartbeats/assignments
+ * and marking the topology as active. default is 0.
+ */
+ public static final String NIMBUS_MIN_REPLICATION_COUNT = "nimbus.min.replication.count";
+ public static final Object NIMBUS_MIN_REPLICATION_COUNT_SCHEMA = Number.class;
+
+ /**
+ * Maximum wait time for the nimbus host replication to achieve the nimbus.min.replication.count.
+ * Once this time is elapsed nimbus will go ahead and perform topology activation tasks even
+ * if required nimbus.min.replication.count is not achieved. The default is 0 seconds, a value of
+ * -1 indicates to wait for ever.
+ */
+ public static final String NIMBUS_MAX_REPLICATION_WAIT_TIME_SEC = "nimbus.max.replication.wait.time.sec";
+ public static final Object NIMBUS_MAX_REPLICATION_WAIT_TIME_SEC_SCHEMA = Number.class;
+
+ /**
- * How often nimbus should wake the cleanup thread to clean the inbox.
- * @see NIMBUS_INBOX_JAR_EXPIRATION_SECS
++ * How often nimbus's background thread to sync code for missing topologies should run.
+ */
+ public static final String NIMBUS_CODE_SYNC_FREQ_SECS = "nimbus.code.sync.freq.secs";
+ public static final Object NIMBUS_CODE_SYNC_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
+
-
public static void setClasspath(Map conf, String cp) {
conf.put(Config.TOPOLOGY_CLASSPATH, cp);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------