You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/11/13 20:37:13 UTC
[18/50] storm git commit: Merge branch 'master' into security-upemerge
Merge branch 'master' into security-upemerge
Conflicts:
README.markdown
storm-core/src/ui/public/templates/index-page-template.html
storm-core/src/ui/public/templates/topology-page-template.html
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d20c101f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d20c101f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d20c101f
Branch: refs/heads/master
Commit: d20c101fa73c413b8d9fe1065a0291125e18bc96
Parents: a55d792 7a58c2d
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 18 14:25:07 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 18 14:25:07 2014 -0500
----------------------------------------------------------------------
CHANGELOG.md | 12 +
README.markdown | 3 +
bin/storm | 6 +-
external/storm-hdfs/README.md | 343 +++++++++++++++++++
external/storm-hdfs/pom.xml | 68 ++++
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 139 ++++++++
.../org/apache/storm/hdfs/bolt/HdfsBolt.java | 131 +++++++
.../storm/hdfs/bolt/SequenceFileBolt.java | 149 ++++++++
.../hdfs/bolt/format/DefaultFileNameFormat.java | 86 +++++
.../hdfs/bolt/format/DefaultSequenceFormat.java | 52 +++
.../hdfs/bolt/format/DelimitedRecordFormat.java | 88 +++++
.../storm/hdfs/bolt/format/FileNameFormat.java | 42 +++
.../storm/hdfs/bolt/format/RecordFormat.java | 32 ++
.../storm/hdfs/bolt/format/SequenceFormat.java | 40 +++
.../hdfs/bolt/rotation/FileRotationPolicy.java | 51 +++
.../bolt/rotation/FileSizeRotationPolicy.java | 81 +++++
.../hdfs/bolt/rotation/NoRotationPolicy.java | 35 ++
.../hdfs/bolt/rotation/TimedRotationPolicy.java | 55 +++
.../storm/hdfs/bolt/sync/CountSyncPolicy.java | 46 +++
.../apache/storm/hdfs/bolt/sync/SyncPolicy.java | 46 +++
.../hdfs/common/rotation/MoveFileAction.java | 27 ++
.../hdfs/common/rotation/RotationAction.java | 12 +
.../hdfs/common/security/HdfsSecurityUtil.java | 48 +++
.../apache/storm/hdfs/trident/HdfsState.java | 317 +++++++++++++++++
.../storm/hdfs/trident/HdfsStateFactory.java | 29 ++
.../apache/storm/hdfs/trident/HdfsUpdater.java | 14 +
.../trident/format/DefaultFileNameFormat.java | 83 +++++
.../trident/format/DefaultSequenceFormat.java | 54 +++
.../trident/format/DelimitedRecordFormat.java | 83 +++++
.../hdfs/trident/format/FileNameFormat.java | 40 +++
.../storm/hdfs/trident/format/RecordFormat.java | 32 ++
.../hdfs/trident/format/SequenceFormat.java | 40 +++
.../trident/rotation/FileRotationPolicy.java | 50 +++
.../rotation/FileSizeRotationPolicy.java | 81 +++++
.../hdfs/trident/rotation/NoRotationPolicy.java | 35 ++
.../trident/rotation/TimedRotationPolicy.java | 54 +++
.../hdfs/trident/sync/CountSyncPolicy.java | 46 +++
.../storm/hdfs/trident/sync/SyncPolicy.java | 46 +++
.../storm/hdfs/bolt/HdfsFileTopology.java | 198 +++++++++++
.../storm/hdfs/bolt/SequenceFileTopology.java | 193 +++++++++++
.../storm/hdfs/trident/FixedBatchSpout.java | 80 +++++
.../storm/hdfs/trident/TridentFileTopology.java | 73 ++++
.../hdfs/trident/TridentSequenceTopology.java | 71 ++++
external/storm-kafka/README.md | 197 ++++++++++-
.../jvm/storm/kafka/DynamicBrokersReader.java | 5 +
.../src/jvm/storm/kafka/bolt/KafkaBolt.java | 49 ++-
.../FieldNameBasedTupleToKafkaMapper.java | 48 +++
.../kafka/bolt/mapper/TupleToKafkaMapper.java | 32 ++
.../bolt/selector/DefaultTopicSelector.java | 34 ++
.../kafka/bolt/selector/KafkaTopicSelector.java | 26 ++
.../storm/kafka/trident/TridentKafkaState.java | 99 ++++++
.../kafka/trident/TridentKafkaStateFactory.java | 57 +++
.../kafka/trident/TridentKafkaUpdater.java | 31 ++
.../FieldNameBasedTupleToKafkaMapper.java | 41 +++
.../mapper/TridentTupleToKafkaMapper.java | 28 ++
.../trident/selector/DefaultTopicSelector.java | 34 ++
.../trident/selector/KafkaTopicSelector.java | 26 ++
.../src/test/storm/kafka/TestUtils.java | 62 ++++
.../src/test/storm/kafka/TridentKafkaTest.java | 89 +++++
.../test/storm/kafka/TridentKafkaTopology.java | 100 ++++++
logback/cluster.xml | 12 +-
pom.xml | 5 +-
storm-core/pom.xml | 2 +-
.../clj/backtype/storm/daemon/supervisor.clj | 9 +-
storm-core/src/clj/backtype/storm/disruptor.clj | 11 +-
storm-core/src/clj/backtype/storm/util.clj | 2 +-
.../jvm/backtype/storm/multilang/ShellMsg.java | 17 +
.../jvm/backtype/storm/spout/ShellSpout.java | 3 +
.../src/jvm/backtype/storm/task/ShellBolt.java | 6 +
.../backtype/storm/utils/DisruptorQueue.java | 56 ++-
storm-core/src/ui/public/css/style.css | 67 ++++
.../public/templates/index-page-template.html | 2 +-
.../templates/topology-page-template.html | 2 +-
.../test/clj/backtype/storm/supervisor_test.clj | 1 +
.../storm/utils/DisruptorQueueTest.java | 25 +-
storm-dist/binary/src/main/assembly/binary.xml | 14 +
76 files changed, 4327 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index 586f171,d5c3b23..27f897b
--- a/README.markdown
+++ b/README.markdown
@@@ -159,8 -159,11 +159,11 @@@ under the License
* Jo Liss ([@joliss](https://github.com/joliss))
* averykhoo ([@averykhoo](https://github.com/averykhoo))
* Curtis Allen ([@curtisallen](https://github.com/curtisallen))
+* Raghavendra Nandagopal ([@RaghavendraNandagopal](https://github.com/RaghavendraNandagopal))
+ * Boris Aksenov ([@miofthena](https://github.com/miofthena))
+ * Itai Frenkel ([@itaifrenkel](https://github.com/itaifrenkel))
+ * Naresh Kosgi ([@NareshKosgi](https://github.com/NareshKosgi))
-
## Acknowledgements
YourKit is kindly supporting open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of innovative and intelligent tools for profiling Java and .NET applications. Take a look at YourKit's leading software products: [YourKit Java Profiler](http://www.yourkit.com/java/profiler/index.jsp) and [YourKit .NET Profiler](http://www.yourkit.com/.net/profiler/index.jsp).
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/bin/storm
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index d8ff6b5,6473ed1..4d83497
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -14,9 -14,9 +14,10 @@@
;; See the License for the specific language governing permissions and
;; limitations under the License.
(ns backtype.storm.daemon.supervisor
+ (:import [java.io OutputStreamWriter BufferedWriter IOException])
(:import [backtype.storm.scheduler ISupervisor]
- [java.net JarURLConnection])
+ [java.net JarURLConnection]
+ [java.net URI])
(:use [backtype.storm bootstrap])
(:use [backtype.storm.daemon common])
(:require [backtype.storm.daemon [worker :as worker]])
@@@ -606,8 -491,8 +607,9 @@@
(defmethod launch-worker
:distributed [supervisor storm-id port worker-id]
(let [conf (:conf supervisor)
+ run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
storm-home (System/getProperty "storm.home")
+ storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home "/logs"))
stormroot (supervisor-stormdist-root conf storm-id)
jlp (jlp stormroot conf)
stormjar (supervisor-stormjar-path stormroot)
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index bd78c5c,45906e5..8f1c29b
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -48,13 -38,11 +48,13 @@@
<tr>
<td><a href="/topology.html?id={{id}}">{{name}}</a></td>
<td>{{id}}</td>
+ <td>{{owner}}</td>
<td>{{status}}</td>
<td>{{uptime}}</td>
- <td>{{tasksTotal}}</td>
<td>{{workersTotal}}</td>
<td>{{executorsTotal}}</td>
+ <td>{{schedulerInfo}}</td>
+ <td>{{tasksTotal}}</td>
</tr>
{{/topologies}}
</tbody>
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/topology-page-template.html
index c249053,2b93ed6..ddf3c17
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@@ -31,13 -21,11 +31,13 @@@
<tr>
<td>{{name}}</td>
<td>{{id}}</td>
+ <td>{{owner}}</td>
<td>{{status}}</td>
<td>{{uptime}}</td>
- <td>{{tasksTotal}}</td>
<td>{{workersTotal}}</td>
<td>{{executorsTotal}}</td>
+ <td>{{schedulerInfo}}</td>
+ <td>{{tasksTotal}}</td>
</tr>
</tbody>
</table>
http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 6b40060,f6fc285..3eb5de6
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,8 -253,9 +259,9 @@@
opts
topo-opts
["-Djava.library.path="
- (str "-Dlogfile.name=worker-" mock-port ".log")
+ (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
"-Dstorm.home="
+ "-Dstorm.log.dir=/logs"
"-Dlogback.configurationFile=/logback/cluster.xml"
(str "-Dstorm.id=" mock-storm-id)
(str "-Dworker.id=" mock-worker-id)