You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/02/12 21:06:49 UTC
[1/4] storm git commit: STORM-1248: port
backtype.storm.messaging.loader to java
Repository: storm
Updated Branches:
refs/heads/master 9ddd29ff2 -> 12ceb0975
STORM-1248: port backtype.storm.messaging.loader to java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/36aa7b07
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/36aa7b07
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/36aa7b07
Branch: refs/heads/master
Commit: 36aa7b07344fe6b0caf46b3592d1754891ff9597
Parents: 3339dae
Author: Abhishek Agarwal <ab...@inmobi.com>
Authored: Fri Feb 12 00:33:56 2016 +0530
Committer: Abhishek Agarwal <ab...@inmobi.com>
Committed: Fri Feb 12 00:33:56 2016 +0530
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/worker.clj | 13 ++++----
.../clj/org/apache/storm/messaging/loader.clj | 34 --------------------
.../clj/org/apache/storm/messaging/local.clj | 23 -------------
storm-core/src/clj/org/apache/storm/testing.clj | 8 +++--
4 files changed, 11 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/36aa7b07/storm-core/src/clj/org/apache/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/worker.clj b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
index 48934f6..0a2a6d6 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/worker.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
@@ -21,14 +21,13 @@
(:require [org.apache.storm.daemon [executor :as executor]])
(:require [org.apache.storm [disruptor :as disruptor] [cluster :as cluster]])
(:require [clojure.set :as set])
- (:require [org.apache.storm.messaging.loader :as msg-loader])
(:import [java.util.concurrent Executors]
[org.apache.storm.hooks IWorkerHook BaseWorkerHook])
(:import [java.util ArrayList HashMap])
(:import [org.apache.storm.utils Utils ConfigUtils TransferDrainer ThriftTopologyUtils WorkerBackpressureThread DisruptorQueue])
(:import [org.apache.storm.grouping LoadMapping])
(:import [org.apache.storm.messaging TransportFactory])
- (:import [org.apache.storm.messaging TaskMessage IContext IConnection ConnectionWithStatus ConnectionWithStatus$Status])
+ (:import [org.apache.storm.messaging TaskMessage IContext IConnection ConnectionWithStatus ConnectionWithStatus$Status DeserializingConnectionCallback])
(:import [org.apache.storm.daemon Shutdownable])
(:import [org.apache.storm.serialization KryoTupleSerializer])
(:import [org.apache.storm.generated StormTopology])
@@ -461,11 +460,11 @@
)))))
(defn register-callbacks [worker]
- (log-message "Registering IConnectionCallbacks for " (:assignment-id worker) ":" (:port worker))
- (msg-loader/register-callback (:transfer-local-fn worker)
- (:receiver worker)
- (:storm-conf worker)
- (worker-context worker)))
+ (let [transfer-local-fn (:transfer-local-fn worker) ^IConnection socket (:receiver worker)]
+ (log-message "Registering IConnectionCallbacks for " (:assignment-id worker) ":" (:port worker))
+ (.registerRecv socket (DeserializingConnectionCallback. (:storm-conf worker)
+ (worker-context worker)
+ transfer-local-fn))))
(defn- close-resources [worker]
(let [dr (:default-shared-resources worker)]
http://git-wip-us.apache.org/repos/asf/storm/blob/36aa7b07/storm-core/src/clj/org/apache/storm/messaging/loader.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/messaging/loader.clj b/storm-core/src/clj/org/apache/storm/messaging/loader.clj
deleted file mode 100644
index b190ab0..0000000
--- a/storm-core/src/clj/org/apache/storm/messaging/loader.clj
+++ /dev/null
@@ -1,34 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.messaging.loader
- (:import [org.apache.storm.messaging IConnection DeserializingConnectionCallback])
- (:require [org.apache.storm.messaging [local :as local]]))
-
-(defn mk-local-context []
- (local/mk-context))
-
-(defn- mk-connection-callback
- "make an IConnectionCallback"
- [transfer-local-fn storm-conf worker-context]
- (DeserializingConnectionCallback. storm-conf
- worker-context
- (fn [batch]
- (transfer-local-fn batch))))
-
-(defn register-callback
- "register the local-transfer-fn with the server"
- [transfer-local-fn ^IConnection socket storm-conf worker-context]
- (.registerRecv socket (mk-connection-callback transfer-local-fn storm-conf worker-context)))
http://git-wip-us.apache.org/repos/asf/storm/blob/36aa7b07/storm-core/src/clj/org/apache/storm/messaging/local.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/messaging/local.clj b/storm-core/src/clj/org/apache/storm/messaging/local.clj
deleted file mode 100644
index 32fbb34..0000000
--- a/storm-core/src/clj/org/apache/storm/messaging/local.clj
+++ /dev/null
@@ -1,23 +0,0 @@
-;; Licensed to the Apache Software Foundation (ASF) under one
-;; or more contributor license agreements. See the NOTICE file
-;; distributed with this work for additional information
-;; regarding copyright ownership. The ASF licenses this file
-;; to you under the Apache License, Version 2.0 (the
-;; "License"); you may not use this file except in compliance
-;; with the License. You may obtain a copy of the License at
-;;
-;; http://www.apache.org/licenses/LICENSE-2.0
-;;
-;; Unless required by applicable law or agreed to in writing, software
-;; distributed under the License is distributed on an "AS IS" BASIS,
-;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-;; See the License for the specific language governing permissions and
-;; limitations under the License.
-(ns org.apache.storm.messaging.local
- (:import [org.apache.storm.messaging IContext])
- (:import [org.apache.storm.messaging.local Context]))
-
-(defn mk-context []
- (let [context (Context.)]
- (.prepare ^IContext context nil)
- context))
http://git-wip-us.apache.org/repos/asf/storm/blob/36aa7b07/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/testing.clj b/storm-core/src/clj/org/apache/storm/testing.clj
index cc78659..12828d6 100644
--- a/storm-core/src/clj/org/apache/storm/testing.clj
+++ b/storm-core/src/clj/org/apache/storm/testing.clj
@@ -44,9 +44,9 @@
(:import [org.apache.storm.transactional.partitioned PartitionedTransactionalSpoutExecutor])
(:import [org.apache.storm.tuple Tuple])
(:import [org.apache.storm.generated StormTopology])
- (:import [org.apache.storm.task TopologyContext])
+ (:import [org.apache.storm.task TopologyContext]
+ (org.apache.storm.messaging IContext))
(:require [org.apache.storm [zookeeper :as zk]])
- (:require [org.apache.storm.messaging.loader :as msg-loader])
(:require [org.apache.storm.daemon.acker :as acker])
(:use [org.apache.storm cluster util thrift config log local-state]))
@@ -117,7 +117,9 @@
(defn mk-shared-context [conf]
(if-not (conf STORM-LOCAL-MODE-ZMQ)
- (msg-loader/mk-local-context)))
+ (let [context (org.apache.storm.messaging.local.Context.)]
+ (.prepare ^IContext context nil)
+ context)))
(defn start-nimbus-daemon [conf nimbus]
(let [server (ThriftServer. conf (Nimbus$Processor. nimbus)
[4/4] storm git commit: Added STORM-1248 to Changelog
Posted by bo...@apache.org.
Added STORM-1248 to Changelog
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/12ceb097
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/12ceb097
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/12ceb097
Branch: refs/heads/master
Commit: 12ceb09758e57e699436514e3fc69994da1415b8
Parents: 3db43db
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Feb 12 14:06:21 2016 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Feb 12 14:06:21 2016 -0600
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/12ceb097/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9dbc9ae..147f854 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1248: port backtype.storm.messaging.loader to java
* STORM-1538: Exception being thrown after Utils conversion to java
* STORM-1242: migrate backtype.storm.command.config-value to java
* STORM-1226: Port backtype.storm.util to java
[2/4] storm git commit: Merge master branch into STORM-1248
Posted by bo...@apache.org.
Merge master branch into STORM-1248
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/55d49687
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/55d49687
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/55d49687
Branch: refs/heads/master
Commit: 55d496871a81c25aea20aab655ee3e6bb90dcfa2
Parents: 36aa7b0 d041183
Author: Abhishek Agarwal <ab...@inmobi.com>
Authored: Fri Feb 12 01:01:28 2016 +0530
Committer: Abhishek Agarwal <ab...@inmobi.com>
Committed: Fri Feb 12 01:01:28 2016 +0530
----------------------------------------------------------------------
CHANGELOG.md | 1 +
external/sql/storm-sql-core/pom.xml | 9 +
pom.xml | 7 +-
storm-core/pom.xml | 9 +
.../src/clj/org/apache/storm/LocalCluster.clj | 4 +-
storm-core/src/clj/org/apache/storm/clojure.clj | 8 +-
storm-core/src/clj/org/apache/storm/cluster.clj | 27 +-
.../cluster_state/zookeeper_state_factory.clj | 11 +-
.../clj/org/apache/storm/command/blobstore.clj | 11 +-
.../org/apache/storm/command/dev_zookeeper.clj | 6 +-
.../clj/org/apache/storm/command/get_errors.clj | 12 +-
.../apache/storm/command/shell_submission.clj | 4 +-
storm-core/src/clj/org/apache/storm/config.clj | 18 +-
.../src/clj/org/apache/storm/converter.clj | 14 +-
.../src/clj/org/apache/storm/daemon/acker.clj | 21 +-
.../src/clj/org/apache/storm/daemon/common.clj | 29 +-
.../src/clj/org/apache/storm/daemon/drpc.clj | 23 +-
.../clj/org/apache/storm/daemon/executor.clj | 528 +++++-----
.../clj/org/apache/storm/daemon/logviewer.clj | 68 +-
.../src/clj/org/apache/storm/daemon/nimbus.clj | 170 ++--
.../clj/org/apache/storm/daemon/supervisor.clj | 200 ++--
.../src/clj/org/apache/storm/daemon/task.clj | 2 +-
.../src/clj/org/apache/storm/daemon/worker.clj | 67 +-
.../src/clj/org/apache/storm/disruptor.clj | 10 +-
storm-core/src/clj/org/apache/storm/event.clj | 2 +-
.../src/clj/org/apache/storm/local_state.clj | 9 +-
.../org/apache/storm/pacemaker/pacemaker.clj | 7 +-
.../storm/pacemaker/pacemaker_state_factory.clj | 24 +-
.../clj/org/apache/storm/process_simulator.clj | 4 +-
.../apache/storm/scheduler/DefaultScheduler.clj | 7 +-
.../apache/storm/scheduler/EvenScheduler.clj | 23 +-
.../storm/scheduler/IsolationScheduler.clj | 29 +-
storm-core/src/clj/org/apache/storm/stats.clj | 82 +-
storm-core/src/clj/org/apache/storm/testing.clj | 83 +-
storm-core/src/clj/org/apache/storm/thrift.clj | 6 +-
storm-core/src/clj/org/apache/storm/timer.clj | 12 +-
.../clj/org/apache/storm/trident/testing.clj | 9 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 97 +-
.../src/clj/org/apache/storm/ui/helpers.clj | 14 +-
storm-core/src/clj/org/apache/storm/util.clj | 923 +----------------
.../src/clj/org/apache/storm/zookeeper.clj | 1 -
.../storm/logging/ThriftAccessLogger.java | 13 +-
.../serialization/SerializationFactory.java | 17 +-
.../staticmocking/MockedConfigUtils.java | 31 -
.../jvm/org/apache/storm/utils/ConfigUtils.java | 20 +-
.../jvm/org/apache/storm/utils/Container.java | 11 +-
.../jvm/org/apache/storm/utils/IPredicate.java | 27 +
.../org/apache/storm/utils/NimbusClient.java | 2 +-
.../jvm/org/apache/storm/utils/TestUtils.java | 34 -
.../src/jvm/org/apache/storm/utils/Time.java | 26 +-
.../src/jvm/org/apache/storm/utils/Utils.java | 989 +++++++++++++++++--
.../storm/validation/ConfigValidation.java | 2 +-
.../org/apache/storm/zookeeper/Zookeeper.java | 7 +
.../org/apache/storm/integration_test.clj | 100 +-
.../org/apache/storm/testing4j_test.clj | 37 +-
.../apache/storm/trident/integration_test.clj | 15 +-
.../test/clj/org/apache/storm/cluster_test.clj | 20 +-
.../test/clj/org/apache/storm/drpc_test.clj | 23 +-
.../clj/org/apache/storm/logviewer_test.clj | 267 ++---
.../storm/messaging/netty_integration_test.clj | 2 +-
.../test/clj/org/apache/storm/nimbus_test.clj | 131 ++-
.../scheduler/resource_aware_scheduler_test.clj | 21 +-
.../apache/storm/security/auth/auth_test.clj | 11 +-
.../authorizer/DRPCSimpleACLAuthorizer_test.clj | 2 +-
.../BlowfishTupleSerializer_test.clj | 1 -
.../clj/org/apache/storm/serialization_test.clj | 23 +-
.../clj/org/apache/storm/supervisor_test.clj | 645 ++++++------
.../clj/org/apache/storm/transactional_test.clj | 18 +
.../clj/org/apache/storm/trident/state_test.clj | 5 +-
.../clj/org/apache/storm/trident/tuple_test.clj | 15 +-
.../test/clj/org/apache/storm/utils_test.clj | 16 +-
.../test/clj/org/apache/storm/worker_test.clj | 1 -
.../staticmocking/ConfigUtilsInstaller.java | 38 +
.../utils/staticmocking/UtilsInstaller.java | 38 +
.../storm/utils/staticmocking/package-info.java | 95 ++
75 files changed, 2999 insertions(+), 2295 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/55d49687/storm-core/src/clj/org/apache/storm/daemon/worker.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/org/apache/storm/daemon/worker.clj
index 0a2a6d6,b2bdcdb..fe8cfae
--- a/storm-core/src/clj/org/apache/storm/daemon/worker.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/worker.clj
@@@ -21,13 -21,17 +21,16 @@@
(:require [org.apache.storm.daemon [executor :as executor]])
(:require [org.apache.storm [disruptor :as disruptor] [cluster :as cluster]])
(:require [clojure.set :as set])
- (:require [org.apache.storm.messaging.loader :as msg-loader])
(:import [java.util.concurrent Executors]
- [org.apache.storm.hooks IWorkerHook BaseWorkerHook])
- (:import [java.util ArrayList HashMap])
- (:import [org.apache.storm.utils Utils ConfigUtils TransferDrainer ThriftTopologyUtils WorkerBackpressureThread DisruptorQueue])
+ [org.apache.storm.hooks IWorkerHook BaseWorkerHook]
+ [uk.org.lidalia.sysoutslf4j.context SysOutOverSLF4J])
+ (:import [java.util ArrayList HashMap]
+ [java.util.concurrent.locks ReentrantReadWriteLock])
+ (:import [org.apache.commons.io FileUtils])
+ (:import [org.apache.storm.utils Utils ConfigUtils TransferDrainer ThriftTopologyUtils WorkerBackpressureThread DisruptorQueue Time])
(:import [org.apache.storm.grouping LoadMapping])
(:import [org.apache.storm.messaging TransportFactory])
- (:import [org.apache.storm.messaging TaskMessage IContext IConnection ConnectionWithStatus ConnectionWithStatus$Status])
+ (:import [org.apache.storm.messaging TaskMessage IContext IConnection ConnectionWithStatus ConnectionWithStatus$Status DeserializingConnectionCallback])
(:import [org.apache.storm.daemon Shutdownable])
(:import [org.apache.storm.serialization KryoTupleSerializer])
(:import [org.apache.storm.generated StormTopology])
@@@ -460,11 -487,11 +486,12 @@@
)))))
(defn register-callbacks [worker]
- (let [transfer-local-fn (:transfer-local-fn worker) ^IConnection socket (:receiver worker)]
- (log-message "Registering IConnectionCallbacks for " (:assignment-id worker) ":" (:port worker))
- (msg-loader/register-callback (:transfer-local-fn worker)
- (:receiver worker)
- (:storm-conf worker)
- (worker-context worker)))
++ (let [transfer-local-fn (:transfer-local-fn worker)
++ ^IConnection socket (:receiver worker)]
+ (log-message "Registering IConnectionCallbacks for " (:assignment-id worker) ":" (:port worker))
+ (.registerRecv socket (DeserializingConnectionCallback. (:storm-conf worker)
+ (worker-context worker)
+ transfer-local-fn))))
(defn- close-resources [worker]
(let [dr (:default-shared-resources worker)]
http://git-wip-us.apache.org/repos/asf/storm/blob/55d49687/storm-core/src/clj/org/apache/storm/testing.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/org/apache/storm/testing.clj
index 12828d6,9a487af..c872742
--- a/storm-core/src/clj/org/apache/storm/testing.clj
+++ b/storm-core/src/clj/org/apache/storm/testing.clj
@@@ -45,8 -46,9 +46,9 @@@
(:import [org.apache.storm.tuple Tuple])
(:import [org.apache.storm.generated StormTopology])
(:import [org.apache.storm.task TopologyContext]
- (org.apache.storm.messaging IContext))
++ (org.apache.storm.messaging IContext)
+ [org.json.simple JSONValue])
(:require [org.apache.storm [zookeeper :as zk]])
- (:require [org.apache.storm.messaging.loader :as msg-loader])
(:require [org.apache.storm.daemon.acker :as acker])
(:use [org.apache.storm cluster util thrift config log local-state]))
[3/4] storm git commit: Merge branch 'thrift' of
https://github.com/abhishekagarwal87/storm into STORM-1248
Posted by bo...@apache.org.
Merge branch 'thrift' of https://github.com/abhishekagarwal87/storm into STORM-1248
STORM-1248: port backtype.storm.messaging.loader to java
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3db43db5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3db43db5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3db43db5
Branch: refs/heads/master
Commit: 3db43db5aad666fbb4b5f0849ab10a6cdad68896
Parents: 9ddd29f 55d4968
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Fri Feb 12 13:56:08 2016 -0600
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Feb 12 13:56:08 2016 -0600
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/worker.clj | 14 ++++----
.../clj/org/apache/storm/messaging/loader.clj | 34 --------------------
.../clj/org/apache/storm/messaging/local.clj | 23 -------------
storm-core/src/clj/org/apache/storm/testing.clj | 6 ++--
4 files changed, 11 insertions(+), 66 deletions(-)
----------------------------------------------------------------------