You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2014/11/13 20:37:13 UTC

[18/50] storm git commit: Merge branch 'master' into security-upemerge

Merge branch 'master' into security-upemerge

Conflicts:
	README.markdown
	storm-core/src/ui/public/templates/index-page-template.html
	storm-core/src/ui/public/templates/topology-page-template.html


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d20c101f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d20c101f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d20c101f

Branch: refs/heads/master
Commit: d20c101fa73c413b8d9fe1065a0291125e18bc96
Parents: a55d792 7a58c2d
Author: Robert (Bobby) Evans <bo...@apache.org>
Authored: Mon Aug 18 14:25:07 2014 -0500
Committer: Robert (Bobby) Evans <bo...@apache.org>
Committed: Mon Aug 18 14:25:07 2014 -0500

----------------------------------------------------------------------
 CHANGELOG.md                                    |  12 +
 README.markdown                                 |   3 +
 bin/storm                                       |   6 +-
 external/storm-hdfs/README.md                   | 343 +++++++++++++++++++
 external/storm-hdfs/pom.xml                     |  68 ++++
 .../storm/hdfs/bolt/AbstractHdfsBolt.java       | 139 ++++++++
 .../org/apache/storm/hdfs/bolt/HdfsBolt.java    | 131 +++++++
 .../storm/hdfs/bolt/SequenceFileBolt.java       | 149 ++++++++
 .../hdfs/bolt/format/DefaultFileNameFormat.java |  86 +++++
 .../hdfs/bolt/format/DefaultSequenceFormat.java |  52 +++
 .../hdfs/bolt/format/DelimitedRecordFormat.java |  88 +++++
 .../storm/hdfs/bolt/format/FileNameFormat.java  |  42 +++
 .../storm/hdfs/bolt/format/RecordFormat.java    |  32 ++
 .../storm/hdfs/bolt/format/SequenceFormat.java  |  40 +++
 .../hdfs/bolt/rotation/FileRotationPolicy.java  |  51 +++
 .../bolt/rotation/FileSizeRotationPolicy.java   |  81 +++++
 .../hdfs/bolt/rotation/NoRotationPolicy.java    |  35 ++
 .../hdfs/bolt/rotation/TimedRotationPolicy.java |  55 +++
 .../storm/hdfs/bolt/sync/CountSyncPolicy.java   |  46 +++
 .../apache/storm/hdfs/bolt/sync/SyncPolicy.java |  46 +++
 .../hdfs/common/rotation/MoveFileAction.java    |  27 ++
 .../hdfs/common/rotation/RotationAction.java    |  12 +
 .../hdfs/common/security/HdfsSecurityUtil.java  |  48 +++
 .../apache/storm/hdfs/trident/HdfsState.java    | 317 +++++++++++++++++
 .../storm/hdfs/trident/HdfsStateFactory.java    |  29 ++
 .../apache/storm/hdfs/trident/HdfsUpdater.java  |  14 +
 .../trident/format/DefaultFileNameFormat.java   |  83 +++++
 .../trident/format/DefaultSequenceFormat.java   |  54 +++
 .../trident/format/DelimitedRecordFormat.java   |  83 +++++
 .../hdfs/trident/format/FileNameFormat.java     |  40 +++
 .../storm/hdfs/trident/format/RecordFormat.java |  32 ++
 .../hdfs/trident/format/SequenceFormat.java     |  40 +++
 .../trident/rotation/FileRotationPolicy.java    |  50 +++
 .../rotation/FileSizeRotationPolicy.java        |  81 +++++
 .../hdfs/trident/rotation/NoRotationPolicy.java |  35 ++
 .../trident/rotation/TimedRotationPolicy.java   |  54 +++
 .../hdfs/trident/sync/CountSyncPolicy.java      |  46 +++
 .../storm/hdfs/trident/sync/SyncPolicy.java     |  46 +++
 .../storm/hdfs/bolt/HdfsFileTopology.java       | 198 +++++++++++
 .../storm/hdfs/bolt/SequenceFileTopology.java   | 193 +++++++++++
 .../storm/hdfs/trident/FixedBatchSpout.java     |  80 +++++
 .../storm/hdfs/trident/TridentFileTopology.java |  73 ++++
 .../hdfs/trident/TridentSequenceTopology.java   |  71 ++++
 external/storm-kafka/README.md                  | 197 ++++++++++-
 .../jvm/storm/kafka/DynamicBrokersReader.java   |   5 +
 .../src/jvm/storm/kafka/bolt/KafkaBolt.java     |  49 ++-
 .../FieldNameBasedTupleToKafkaMapper.java       |  48 +++
 .../kafka/bolt/mapper/TupleToKafkaMapper.java   |  32 ++
 .../bolt/selector/DefaultTopicSelector.java     |  34 ++
 .../kafka/bolt/selector/KafkaTopicSelector.java |  26 ++
 .../storm/kafka/trident/TridentKafkaState.java  |  99 ++++++
 .../kafka/trident/TridentKafkaStateFactory.java |  57 +++
 .../kafka/trident/TridentKafkaUpdater.java      |  31 ++
 .../FieldNameBasedTupleToKafkaMapper.java       |  41 +++
 .../mapper/TridentTupleToKafkaMapper.java       |  28 ++
 .../trident/selector/DefaultTopicSelector.java  |  34 ++
 .../trident/selector/KafkaTopicSelector.java    |  26 ++
 .../src/test/storm/kafka/TestUtils.java         |  62 ++++
 .../src/test/storm/kafka/TridentKafkaTest.java  |  89 +++++
 .../test/storm/kafka/TridentKafkaTopology.java  | 100 ++++++
 logback/cluster.xml                             |  12 +-
 pom.xml                                         |   5 +-
 storm-core/pom.xml                              |   2 +-
 .../clj/backtype/storm/daemon/supervisor.clj    |   9 +-
 storm-core/src/clj/backtype/storm/disruptor.clj |  11 +-
 storm-core/src/clj/backtype/storm/util.clj      |   2 +-
 .../jvm/backtype/storm/multilang/ShellMsg.java  |  17 +
 .../jvm/backtype/storm/spout/ShellSpout.java    |   3 +
 .../src/jvm/backtype/storm/task/ShellBolt.java  |   6 +
 .../backtype/storm/utils/DisruptorQueue.java    |  56 ++-
 storm-core/src/ui/public/css/style.css          |  67 ++++
 .../public/templates/index-page-template.html   |   2 +-
 .../templates/topology-page-template.html       |   2 +-
 .../test/clj/backtype/storm/supervisor_test.clj |   1 +
 .../storm/utils/DisruptorQueueTest.java         |  25 +-
 storm-dist/binary/src/main/assembly/binary.xml  |  14 +
 76 files changed, 4327 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/README.markdown
----------------------------------------------------------------------
diff --cc README.markdown
index 586f171,d5c3b23..27f897b
--- a/README.markdown
+++ b/README.markdown
@@@ -159,8 -159,11 +159,11 @@@ under the License
  * Jo Liss ([@joliss](https://github.com/joliss))
  * averykhoo ([@averykhoo](https://github.com/averykhoo))
  * Curtis Allen ([@curtisallen](https://github.com/curtisallen))
 +* Raghavendra Nandagopal ([@RaghavendraNandagopal](https://github.com/RaghavendraNandagopal)) 
+ * Boris Aksenov ([@miofthena](https://github.com/miofthena))
+ * Itai Frenkel ([@itaifrenkel](https://github.com/itaifrenkel))
+ * Naresh Kosgi ([@NareshKosgi](https://github.com/NareshKosgi))
  
 -
  ## Acknowledgements
  
  YourKit is kindly supporting open source projects with its full-featured Java Profiler. YourKit, LLC is the creator of innovative and intelligent tools for profiling Java and .NET applications. Take a look at YourKit's leading software products: [YourKit Java Profiler](http://www.yourkit.com/java/profiler/index.jsp) and [YourKit .NET Profiler](http://www.yourkit.com/.net/profiler/index.jsp).

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/bin/storm
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index d8ff6b5,6473ed1..4d83497
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -14,9 -14,9 +14,10 @@@
  ;; See the License for the specific language governing permissions and
  ;; limitations under the License.
  (ns backtype.storm.daemon.supervisor
 +  (:import [java.io OutputStreamWriter BufferedWriter IOException])
    (:import [backtype.storm.scheduler ISupervisor]
-            [java.net JarURLConnection])
+            [java.net JarURLConnection]
+            [java.net URI])
    (:use [backtype.storm bootstrap])
    (:use [backtype.storm.daemon common])
    (:require [backtype.storm.daemon [worker :as worker]])
@@@ -606,8 -491,8 +607,9 @@@
  (defmethod launch-worker
      :distributed [supervisor storm-id port worker-id]
      (let [conf (:conf supervisor)
 +          run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
            storm-home (System/getProperty "storm.home")
+           storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home "/logs"))
            stormroot (supervisor-stormdist-root conf storm-id)
            jlp (jlp stormroot conf)
            stormjar (supervisor-stormjar-path stormroot)

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/clj/backtype/storm/util.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/css/style.css
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index bd78c5c,45906e5..8f1c29b
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -48,13 -38,11 +48,13 @@@
  <tr>
    <td><a href="/topology.html?id={{id}}">{{name}}</a></td>
    <td>{{id}}</td>
 +  <td>{{owner}}</td>
    <td>{{status}}</td>
    <td>{{uptime}}</td>
-   <td>{{tasksTotal}}</td>
    <td>{{workersTotal}}</td>
    <td>{{executorsTotal}}</td>
 +  <td>{{schedulerInfo}}</td>
+   <td>{{tasksTotal}}</td>
  </tr>
  {{/topologies}}
  </tbody>

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/topology-page-template.html
index c249053,2b93ed6..ddf3c17
--- a/storm-core/src/ui/public/templates/topology-page-template.html
+++ b/storm-core/src/ui/public/templates/topology-page-template.html
@@@ -31,13 -21,11 +31,13 @@@
        <tr>
          <td>{{name}}</td>
          <td>{{id}}</td>
 +        <td>{{owner}}</td>
          <td>{{status}}</td>
          <td>{{uptime}}</td>
-         <td>{{tasksTotal}}</td>
          <td>{{workersTotal}}</td>
          <td>{{executorsTotal}}</td>
 +        <td>{{schedulerInfo}}</td>
+         <td>{{tasksTotal}}</td>
        </tr>
      </tbody>
    </table>

http://git-wip-us.apache.org/repos/asf/storm/blob/d20c101f/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 6b40060,f6fc285..3eb5de6
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,8 -253,9 +259,9 @@@
                                 opts
                                 topo-opts
                                 ["-Djava.library.path="
 -                                (str "-Dlogfile.name=worker-" mock-port ".log")
 +                                (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port ".log")
                                  "-Dstorm.home="
+                                 "-Dstorm.log.dir=/logs"
                                  "-Dlogback.configurationFile=/logback/cluster.xml"
                                  (str "-Dstorm.id=" mock-storm-id)
                                  (str "-Dworker.id=" mock-worker-id)