You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/03/24 19:47:34 UTC

[1/3] storm git commit: STORM-1648: drpc spout reconnect on failure

Repository: storm
Updated Branches:
  refs/heads/master df3867fb2 -> 92d302481


STORM-1648: drpc spout reconnect on failure


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a8498888
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a8498888
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a8498888

Branch: refs/heads/master
Commit: a8498888705d975c2e50a4e289d01fcb57d0adc6
Parents: 4e6cb58
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Tue Mar 22 15:11:27 2016 -0500
Committer: Alessandro Bellina <ab...@yahoo-inc.com>
Committed: Tue Mar 22 16:57:18 2016 -0500

----------------------------------------------------------------------
 .../jvm/org/apache/storm/drpc/DRPCSpout.java    | 51 +++++++++++----
 .../test/clj/org/apache/storm/drpc_test.clj     | 69 ++++++++++++++++++--
 2 files changed, 100 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/a8498888/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java b/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
index 791fc91..45a82db 100644
--- a/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
@@ -106,16 +106,25 @@ public class DRPCSpout extends BaseRichSpout {
         }
     }
 
-    private void reconnect(final DRPCInvocationsClient c) {
+    private void reconnectAsync(final DRPCInvocationsClient client) {
         _futures.add(_backround.submit(new Callable<Void>() {
             @Override
             public Void call() throws Exception {
-                c.reconnectClient();
+                client.reconnectClient();
                 return null;
             }
         }));
     }
 
+    private void reconnectSync(DRPCInvocationsClient client) {
+        try {
+            LOG.info("reconnecting... ");
+            client.reconnectClient(); //Blocking call
+        } catch (TException e2) {
+            LOG.error("Failed to connect to DRPC server", e2);
+        }
+    }
+
     private void checkFutures() {
         Iterator<Future<Void>> i = _futures.iterator();
         while (i.hasNext()) {
@@ -172,7 +181,7 @@ public class DRPCSpout extends BaseRichSpout {
     public void nextTuple() {
         boolean gotRequest = false;
         if(_local_drpc_id==null) {
-            int size;
+            int size = 0;
             synchronized (_clients) {
                 size = _clients.size(); //This will only ever grow, so no need to worry about falling off the end
             }
@@ -183,7 +192,7 @@ public class DRPCSpout extends BaseRichSpout {
                 }
                 if (!client.isConnected()) {
                     LOG.warn("DRPCInvocationsClient [{}:{}] is not connected.", client.getHost(), client.getPort());
-                    reconnect(client);
+                    reconnectAsync(client);
                     continue;
                 }
                 try {
@@ -198,10 +207,10 @@ public class DRPCSpout extends BaseRichSpout {
                         break;
                     }
                 } catch (AuthorizationException aze) {
-                    reconnect(client);
+                    reconnectAsync(client);
                     LOG.error("Not authorized to fetch DRPC result from DRPC server", aze);
                 } catch (TException e) {
-                    reconnect(client);
+                    reconnectAsync(client);
                     LOG.error("Failed to fetch DRPC result from DRPC server", e);
                 } catch (Exception e) {
                     LOG.error("Failed to fetch DRPC result from DRPC server", e);
@@ -219,7 +228,8 @@ public class DRPCSpout extends BaseRichSpout {
                         returnInfo.put("host", _local_drpc_id);
                         returnInfo.put("port", 0);
                         gotRequest = true;
-                        _collector.emit(new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)), new DRPCMessageId(req.get_request_id(), 0));
+                        _collector.emit(new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)), 
+                                        new DRPCMessageId(req.get_request_id(), 0));
                     }
                 } catch (AuthorizationException aze) {
                     throw new RuntimeException(aze);
@@ -242,17 +252,30 @@ public class DRPCSpout extends BaseRichSpout {
         DRPCMessageId did = (DRPCMessageId) msgId;
         DistributedRPCInvocations.Iface client;
         
-        if(_local_drpc_id == null) {
+        if (_local_drpc_id == null) {
             client = _clients.get(did.index);
         } else {
             client = (DistributedRPCInvocations.Iface) ServiceRegistry.getService(_local_drpc_id);
         }
-        try {
-            client.failRequest(did.id);
-        } catch (AuthorizationException aze) {
-            LOG.error("Not authorized to failREquest from DRPC server", aze);
-        } catch (TException e) {
-            LOG.error("Failed to fail request", e);
+
+        int retryCnt = 0;
+        int maxRetries = 3;
+
+        while (retryCnt < maxRetries) {
+            retryCnt++;
+            try {
+                client.failRequest(did.id);
+                break;
+            } catch (AuthorizationException aze) {
+                LOG.error("Not authorized to failRequest from DRPC server", aze);
+                throw new RuntimeException(aze);
+            } catch (TException tex) {
+                if (retryCnt >= maxRetries) {
+                    LOG.error("Failed to fail request", tex);
+                    break;
+                }
+                reconnectSync((DRPCInvocationsClient)client);
+            }
         }
     }
 

http://git-wip-us.apache.org/repos/asf/storm/blob/a8498888/storm-core/test/clj/org/apache/storm/drpc_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/drpc_test.clj b/storm-core/test/clj/org/apache/storm/drpc_test.clj
index a20872e..827f536 100644
--- a/storm-core/test/clj/org/apache/storm/drpc_test.clj
+++ b/storm-core/test/clj/org/apache/storm/drpc_test.clj
@@ -16,20 +16,21 @@
 (ns org.apache.storm.drpc-test
   (:use [clojure test])
   (:import [org.apache.storm.drpc ReturnResults DRPCSpout
-            LinearDRPCTopologyBuilder]
-           [org.apache.storm.utils ConfigUtils Utils])
+            LinearDRPCTopologyBuilder DRPCInvocationsClient]
+           [org.apache.storm.utils ConfigUtils Utils ServiceRegistry])
   (:import [org.apache.storm.topology FailedException])
   (:import [org.apache.storm.coordination CoordinatedBolt$FinishedCallback])
-  (:import [org.apache.storm LocalDRPC LocalCluster])
+  (:import [org.apache.storm ILocalDRPC LocalDRPC LocalCluster])
   (:import [org.apache.storm.tuple Fields])
   (:import [org.mockito Mockito])
   (:import [org.mockito.exceptions.base MockitoAssertionError])
-  (:import [org.apache.storm.utils ConfigUtils]
-           [org.apache.storm.utils.staticmocking ConfigUtilsInstaller])
-  (:import [org.apache.storm.generated DRPCExecutionException])
+  (:import [org.apache.storm.utils.staticmocking ConfigUtilsInstaller])
+  (:import [org.apache.storm.spout SpoutOutputCollector])
+  (:import [org.apache.storm.generated DRPCExecutionException DRPCRequest])
   (:import [java.util.concurrent ConcurrentLinkedQueue])
   (:import [org.apache.storm Thrift])
   (:import [org.apache.storm.daemon DrpcServer])
+  (:import [org.mockito ArgumentCaptor Mockito Matchers])
   (:use [org.apache.storm config testing])
   (:use [org.apache.storm.internal clojure])
   (:use [org.apache.storm.daemon common drpc])
@@ -255,3 +256,59 @@
       (is (thrown? DRPCExecutionException
             (.execute drpc-handler "ArbitraryDRPCFunctionName" "no-args"))))))
 
+(deftest test-drpc-attempts-two-reconnects-in-fail-request
+  (let [handler (Mockito/mock DRPCInvocationsClient 
+                              (.extraInterfaces (Mockito/withSettings) (into-array Class [ILocalDRPC])))
+        ;; used to capture the msgId on .emit
+        output-collector (Mockito/mock SpoutOutputCollector)
+        captor (ArgumentCaptor/forClass Object)
+        ;; pick up service-id from registry
+        service-id (ServiceRegistry/registerService handler)]
+
+    ;; mock getServiceId s.t. DRPCSpout uses our handler
+    (. (Mockito/when (.getServiceId handler)) thenReturn service-id)
+    ;; mock fetchRequest s.t. DRPCSpout has a request to process on nextTuple
+    (. (Mockito/when (.fetchRequest handler (Matchers/anyString))) thenReturn (DRPCRequest. "square 2" "bar"))
+    ;; mock failRequest s.t. DRPCSpout attempts retry on .fail
+    (.failRequest (.when (Mockito/doThrow (DRPCExecutionException.)) handler) (Matchers/anyString))
+
+    (let [spout (DRPCSpout. "test" handler)]
+      ;; tell the spout to use the mock collector
+      (.open spout nil nil output-collector) 
+      ;; pick up a msgId to fail
+      (.nextTuple spout) 
+      (.emit (Mockito/verify output-collector) (Mockito/anyList) (.capture captor))
+      ;; fail the msg
+      (.fail spout (.getValue captor))
+      ;; attempt 2 reconnects 
+      (.reconnectClient (Mockito/verify handler (Mockito/times 2)))
+      (.failRequest (Mockito/verify handler (Mockito/times 3)) (Matchers/anyString)))))
+
+(deftest test-drpc-stops-retrying-after-successful-reconnect
+  (let [handler (Mockito/mock DRPCInvocationsClient 
+                              (.extraInterfaces (Mockito/withSettings) (into-array Class [ILocalDRPC])))
+        ;; used to capture the msgId on .emit
+        output-collector (Mockito/mock SpoutOutputCollector)
+        captor (ArgumentCaptor/forClass Object)
+        ;; pick up service-id from registry
+        service-id (ServiceRegistry/registerService handler)]
+
+    ;; mock getServiceId s.t. DRPCSpout uses our handler
+    (. (Mockito/when (.getServiceId handler)) thenReturn service-id)
+    ;; mock fetchRequest s.t. DRPCSpout has a request to process on nextTuple
+    (. (Mockito/when (.fetchRequest handler (Matchers/anyString))) thenReturn (DRPCRequest. "square 2" "bar"))
+    ;; mock failRequest s.t. DRPCSpout attempts retry on .fail the first time only, but succeed the second time
+    (.failRequest (.when (.doNothing
+                         (Mockito/doThrow (DRPCExecutionException.))) handler) (Matchers/anyString))
+
+    (let [spout (DRPCSpout. "test" handler)]
+      ;; tell the spout to use the mock collector
+      (.open spout nil nil output-collector) 
+      ;; pick up a msgId to fail
+      (.nextTuple spout) 
+      (.emit (Mockito/verify output-collector) (Mockito/anyList) (.capture captor))
+      ;; fail the msg
+      (.fail spout (.getValue captor))
+      ;; reconnect once after a failure
+      (.reconnectClient (Mockito/verify handler (Mockito/times 1)))
+      (.failRequest (Mockito/verify handler (Mockito/times 2)) (Matchers/anyString)))))


[3/3] storm git commit: Added STORM-1648 to Changelog.

Posted by bo...@apache.org.
Added STORM-1648 to Changelog.


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/92d30248
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/92d30248
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/92d30248

Branch: refs/heads/master
Commit: 92d302481e05cdc20215ac4f96ae24c2ba16d70e
Parents: 6fea520
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Mar 24 13:46:47 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Mar 24 13:46:47 2016 -0500

----------------------------------------------------------------------
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/92d30248/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a81117..fa52b20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ## 2.0.0
+ * STORM-1648: drpc spout reconnect on failure
  * STORM-1631: Storm CGroup bugs
  * STORM-1616: Add RAS API for Trident
  * STORM-1623: nimbus.clj's minor bug


[2/3] storm git commit: Merge branch 'STORM-1648_drpcspout_reconnect_in_fail' of https://github.com/abellina/storm into STORM-1648

Posted by bo...@apache.org.
Merge branch 'STORM-1648_drpcspout_reconnect_in_fail' of https://github.com/abellina/storm into STORM-1648

STORM-1648: drpc spout reconnect on failure


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6fea5207
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6fea5207
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6fea5207

Branch: refs/heads/master
Commit: 6fea5207381a958e2c0a08e5ab79d6ffc4896782
Parents: df3867f a849888
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Mar 24 13:35:25 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Mar 24 13:35:25 2016 -0500

----------------------------------------------------------------------
 .../jvm/org/apache/storm/drpc/DRPCSpout.java    | 51 +++++++++++----
 .../test/clj/org/apache/storm/drpc_test.clj     | 69 ++++++++++++++++++--
 2 files changed, 100 insertions(+), 20 deletions(-)
----------------------------------------------------------------------