You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by ka...@apache.org on 2017/01/04 08:58:32 UTC

[1/5] storm git commit: Provide Socket time out for nimbus thrift client

Repository: storm
Updated Branches:
  refs/heads/1.x-branch bd93b7f7b -> 3129bfa37


Provide Socket time out for nimbus thrift client


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/209e2cd5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/209e2cd5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/209e2cd5

Branch: refs/heads/1.x-branch
Commit: 209e2cd504f1e7078451b919ae7b765bcf2fe4a2
Parents: bd93b7f
Author: Sanket <sc...@untilservice-lm>
Authored: Wed Dec 21 15:08:12 2016 -0600
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Jan 4 17:35:10 2017 +0900

----------------------------------------------------------------------
 conf/defaults.yaml                              |  1 +
 storm-core/src/jvm/org/apache/storm/Config.java |  7 +++++
 .../security/auth/SaslTransportPlugin.java      | 28 ++++++++++++--------
 .../security/auth/ThriftConnectionType.java     | 19 +++++++++----
 4 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/209e2cd5/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 041a628..a8a9ce6 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -40,6 +40,7 @@ storm.exhibitor.poll.uripath: "/exhibitor/v1/cluster/list"
 storm.cluster.mode: "distributed" # can be distributed or local
 storm.local.mode.zmq: false
 storm.thrift.transport: "org.apache.storm.security.auth.SimpleTransportPlugin"
+storm.thrift.socket.timeout.ms: 600000
 storm.principal.tolocal: "org.apache.storm.security.auth.DefaultPrincipalToLocal"
 storm.group.mapping.service: "org.apache.storm.security.auth.ShellBasedGroupsMapping"
 storm.group.mapping.service.params: null

http://git-wip-us.apache.org/repos/asf/storm/blob/209e2cd5/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/Config.java b/storm-core/src/jvm/org/apache/storm/Config.java
index 41b3f67..1b9725e 100644
--- a/storm-core/src/jvm/org/apache/storm/Config.java
+++ b/storm-core/src/jvm/org/apache/storm/Config.java
@@ -484,6 +484,13 @@ public class Config extends HashMap<String, Object> {
     public static final String NIMBUS_THRIFT_TRANSPORT_PLUGIN = "nimbus.thrift.transport";
 
     /**
+     * How long before a Thrift Client socket hangs before timeout
+     * and restart the socket.
+     */
+    @isInteger
+    public static final String STORM_THRIFT_SOCKET_TIMEOUT_MS = "storm.thrift.socket.timeout.ms";
+
+    /**
      * The host that the master server is running on, added only for backward compatibility,
      * the usage deprecated in favor of nimbus.seeds config.
      */

http://git-wip-us.apache.org/repos/asf/storm/blob/209e2cd5/storm-core/src/jvm/org/apache/storm/security/auth/SaslTransportPlugin.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/security/auth/SaslTransportPlugin.java b/storm-core/src/jvm/org/apache/storm/security/auth/SaslTransportPlugin.java
index cc840dd..cad2b30 100644
--- a/storm-core/src/jvm/org/apache/storm/security/auth/SaslTransportPlugin.java
+++ b/storm-core/src/jvm/org/apache/storm/security/auth/SaslTransportPlugin.java
@@ -64,8 +64,14 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
     @Override
     public TServer getServer(TProcessor processor) throws IOException, TTransportException {
         int port = type.getPort(storm_conf);
+        Integer socketTimeout = type.getSocketTimeOut(storm_conf);
         TTransportFactory serverTransportFactory = getServerTransportFactory();
-        TServerSocket serverTransport = new TServerSocket(port);
+        TServerSocket serverTransport = null;
+        if (socketTimeout != null) {
+            serverTransport = new TServerSocket(port, socketTimeout);
+        } else {
+            serverTransport = new TServerSocket(port);
+        }
         int numWorkerThreads = type.getNumThreads(storm_conf);
         Integer queueSize = type.getQueueSize(storm_conf);
 
@@ -96,12 +102,12 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
     protected abstract TTransportFactory getServerTransportFactory() throws IOException;
 
 
-    /**                                                                                                                                                                             
-     * Processor that pulls the SaslServer object out of the transport, and                                                                                                         
-     * assumes the remote user's UGI before calling through to the original                                                                                                         
-     * processor.                                                                                                                                                                   
-     *                                                                                                                                                                              
-     * This is used on the server side to set the UGI for each specific call.                                                                                                       
+    /**
+     * Processor that pulls the SaslServer object out of the transport, and
+     * assumes the remote user's UGI before calling through to the original
+     * processor.
+     *
+     * This is used on the server side to set the UGI for each specific call.
      */
     private static class TUGIWrapProcessor implements TProcessor {
         final TProcessor wrapped;
@@ -111,7 +117,7 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
         }
 
         public boolean process(final TProtocol inProt, final TProtocol outProt) throws TException {
-            //populating request context 
+            //populating request context
             ReqContext req_context = ReqContext.context();
 
             TTransport trans = inProt.getTransport();
@@ -127,7 +133,7 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
             Socket socket = tsocket.getSocket();
             req_context.setRemoteAddress(socket.getInetAddress());
 
-            //remote subject 
+            //remote subject
             SaslServer saslServer = saslTrans.getSaslServer();
             String authId = saslServer.getAuthorizationID();
             Subject remoteUser = new Subject();
@@ -146,8 +152,8 @@ public abstract class SaslTransportPlugin implements ITransportPlugin {
             this.name =  name;
         }
 
-        /**                                                                                                                                                                                
-         * Get the full name of the user.                                                                                                                                                  
+        /**
+         * Get the full name of the user.
          */
         public String getName() {
             return name;

http://git-wip-us.apache.org/repos/asf/storm/blob/209e2cd5/storm-core/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java b/storm-core/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java
index f627956..bc7c966 100644
--- a/storm-core/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java
+++ b/storm-core/src/jvm/org/apache/storm/security/auth/ThriftConnectionType.java
@@ -27,25 +27,27 @@ import java.util.Map;
  */
 public enum ThriftConnectionType {
     NIMBUS(Config.NIMBUS_THRIFT_TRANSPORT_PLUGIN, Config.NIMBUS_THRIFT_PORT, Config.NIMBUS_QUEUE_SIZE,
-         Config.NIMBUS_THRIFT_THREADS, Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE),
+         Config.NIMBUS_THRIFT_THREADS, Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Config.STORM_THRIFT_SOCKET_TIMEOUT_MS),
     DRPC(Config.DRPC_THRIFT_TRANSPORT_PLUGIN, Config.DRPC_PORT, Config.DRPC_QUEUE_SIZE,
-         Config.DRPC_WORKER_THREADS, Config.DRPC_MAX_BUFFER_SIZE),
+         Config.DRPC_WORKER_THREADS, Config.DRPC_MAX_BUFFER_SIZE, null),
     DRPC_INVOCATIONS(Config.DRPC_INVOCATIONS_THRIFT_TRANSPORT_PLUGIN, Config.DRPC_INVOCATIONS_PORT, null,
-         Config.DRPC_INVOCATIONS_THREADS, Config.DRPC_MAX_BUFFER_SIZE);
+         Config.DRPC_INVOCATIONS_THREADS, Config.DRPC_MAX_BUFFER_SIZE, null);
 
     private final String _transConf;
     private final String _portConf;
     private final String _qConf;
     private final String _threadsConf;
     private final String _buffConf;
+    private final String _socketTimeoutConf;
 
     ThriftConnectionType(String transConf, String portConf, String qConf,
-                         String threadsConf, String buffConf) {
+                         String threadsConf, String buffConf, String socketTimeoutConf) {
         _transConf = transConf;
         _portConf = portConf;
         _qConf = qConf;
         _threadsConf = threadsConf;
         _buffConf = buffConf;
+        _socketTimeoutConf = socketTimeoutConf;
     }
 
     public String getTransportPlugin(Map conf) {
@@ -67,7 +69,7 @@ public enum ThriftConnectionType {
         return (Integer)conf.get(_qConf);
     }
 
-    public int getNumThreads(Map conf) { 
+    public int getNumThreads(Map conf) {
         return Utils.getInt(conf.get(_threadsConf));
     }
 
@@ -80,4 +82,11 @@ public enum ThriftConnectionType {
     public int getMaxBufferSize(Map conf) {
         return Utils.getInt(conf.get(_buffConf));
     }
+
+    public Integer getSocketTimeOut(Map conf) {
+        if (_socketTimeoutConf == null) {
+            return null;
+        }
+        return Utils.getInt(conf.get(_socketTimeoutConf));
+    }
 }


[5/5] storm git commit: STORM-2254: CHANGELOG

Posted by ka...@apache.org.
STORM-2254: CHANGELOG


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3129bfa3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3129bfa3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3129bfa3

Branch: refs/heads/1.x-branch
Commit: 3129bfa37a51bce9923b17f32499cadbd9e5addb
Parents: 204dd87
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Jan 4 17:57:30 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Jan 4 17:57:30 2017 +0900

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/3129bfa3/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5205662..5fb170f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 1.1.0
+ * STORM-2254: Provide Socket time out for nimbus thrift client
  * STORM-2200: [Storm SQL] Drop Aggregate & Join support on Trident mode
  * STORM-2266: Close NimbusClient instances appropriately
  * STORM-2203: Add a getAll method to KeyValueState interface


[3/5] storm git commit: realized tumbling window test is failing from way before my PR and has nothing to do with mine so switching it back to 10 minutes which should be sufficient

Posted by ka...@apache.org.
realized tumbling window test is failing from way before my PR and has nothing to do with mine so switching it back to 10 minutes which should be sufficient


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b432d7dc
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b432d7dc
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b432d7dc

Branch: refs/heads/1.x-branch
Commit: b432d7dcafce646dd28e9107381b5838db98bdf7
Parents: 6184e56
Author: Sanket <sc...@untilservice-lm>
Authored: Thu Dec 22 09:45:42 2016 -0600
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Jan 4 17:38:29 2017 +0900

----------------------------------------------------------------------
 conf/defaults.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b432d7dc/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index e92d25e..a8a9ce6 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -40,7 +40,7 @@ storm.exhibitor.poll.uripath: "/exhibitor/v1/cluster/list"
 storm.cluster.mode: "distributed" # can be distributed or local
 storm.local.mode.zmq: false
 storm.thrift.transport: "org.apache.storm.security.auth.SimpleTransportPlugin"
-storm.thrift.socket.timeout.ms: 1200000
+storm.thrift.socket.timeout.ms: 600000
 storm.principal.tolocal: "org.apache.storm.security.auth.DefaultPrincipalToLocal"
 storm.group.mapping.service: "org.apache.storm.security.auth.ShellBasedGroupsMapping"
 storm.group.mapping.service.params: null


[2/5] storm git commit: increasing timeout as tumbling window test is taking more than 10 minutes to run?

Posted by ka...@apache.org.
increasing timeout as tumbling window test is taking more than 10 minutes to run?


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6184e560
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6184e560
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6184e560

Branch: refs/heads/1.x-branch
Commit: 6184e560d49542f157c397e371fe187f527c6c3f
Parents: 209e2cd
Author: Sanket <sc...@untilservice-lm>
Authored: Wed Dec 21 16:29:02 2016 -0600
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Jan 4 17:35:22 2017 +0900

----------------------------------------------------------------------
 conf/defaults.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/6184e560/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index a8a9ce6..e92d25e 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -40,7 +40,7 @@ storm.exhibitor.poll.uripath: "/exhibitor/v1/cluster/list"
 storm.cluster.mode: "distributed" # can be distributed or local
 storm.local.mode.zmq: false
 storm.thrift.transport: "org.apache.storm.security.auth.SimpleTransportPlugin"
-storm.thrift.socket.timeout.ms: 600000
+storm.thrift.socket.timeout.ms: 1200000
 storm.principal.tolocal: "org.apache.storm.security.auth.DefaultPrincipalToLocal"
 storm.group.mapping.service: "org.apache.storm.security.auth.ShellBasedGroupsMapping"
 storm.group.mapping.service.params: null


[4/5] storm git commit: Merge branch 'STORM-2254-1.x-merge' into 1.x-branch

Posted by ka...@apache.org.
Merge branch 'STORM-2254-1.x-merge' into 1.x-branch


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/204dd876
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/204dd876
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/204dd876

Branch: refs/heads/1.x-branch
Commit: 204dd8765fdebf7093c2dadc62567bf9a924d89f
Parents: bd93b7f b432d7d
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Jan 4 17:44:12 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Jan 4 17:44:12 2017 +0900

----------------------------------------------------------------------
 conf/defaults.yaml                              |  1 +
 storm-core/src/jvm/org/apache/storm/Config.java |  7 +++++
 .../security/auth/SaslTransportPlugin.java      | 28 ++++++++++++--------
 .../security/auth/ThriftConnectionType.java     | 19 +++++++++----
 4 files changed, 39 insertions(+), 16 deletions(-)
----------------------------------------------------------------------