You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by bo...@apache.org on 2016/03/24 19:47:34 UTC
[1/3] storm git commit: STORM-1648: drpc spout reconnect on failure
Repository: storm
Updated Branches:
refs/heads/master df3867fb2 -> 92d302481
STORM-1648: drpc spout reconnect on failure
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a8498888
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a8498888
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a8498888
Branch: refs/heads/master
Commit: a8498888705d975c2e50a4e289d01fcb57d0adc6
Parents: 4e6cb58
Author: Alessandro Bellina <ab...@yahoo-inc.com>
Authored: Tue Mar 22 15:11:27 2016 -0500
Committer: Alessandro Bellina <ab...@yahoo-inc.com>
Committed: Tue Mar 22 16:57:18 2016 -0500
----------------------------------------------------------------------
.../jvm/org/apache/storm/drpc/DRPCSpout.java | 51 +++++++++++----
.../test/clj/org/apache/storm/drpc_test.clj | 69 ++++++++++++++++++--
2 files changed, 100 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/a8498888/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java b/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
index 791fc91..45a82db 100644
--- a/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/drpc/DRPCSpout.java
@@ -106,16 +106,25 @@ public class DRPCSpout extends BaseRichSpout {
}
}
- private void reconnect(final DRPCInvocationsClient c) {
+ private void reconnectAsync(final DRPCInvocationsClient client) {
_futures.add(_backround.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
- c.reconnectClient();
+ client.reconnectClient();
return null;
}
}));
}
+ private void reconnectSync(DRPCInvocationsClient client) {
+ try {
+ LOG.info("reconnecting... ");
+ client.reconnectClient(); //Blocking call
+ } catch (TException e2) {
+ LOG.error("Failed to connect to DRPC server", e2);
+ }
+ }
+
private void checkFutures() {
Iterator<Future<Void>> i = _futures.iterator();
while (i.hasNext()) {
@@ -172,7 +181,7 @@ public class DRPCSpout extends BaseRichSpout {
public void nextTuple() {
boolean gotRequest = false;
if(_local_drpc_id==null) {
- int size;
+ int size = 0;
synchronized (_clients) {
size = _clients.size(); //This will only ever grow, so no need to worry about falling off the end
}
@@ -183,7 +192,7 @@ public class DRPCSpout extends BaseRichSpout {
}
if (!client.isConnected()) {
LOG.warn("DRPCInvocationsClient [{}:{}] is not connected.", client.getHost(), client.getPort());
- reconnect(client);
+ reconnectAsync(client);
continue;
}
try {
@@ -198,10 +207,10 @@ public class DRPCSpout extends BaseRichSpout {
break;
}
} catch (AuthorizationException aze) {
- reconnect(client);
+ reconnectAsync(client);
LOG.error("Not authorized to fetch DRPC result from DRPC server", aze);
} catch (TException e) {
- reconnect(client);
+ reconnectAsync(client);
LOG.error("Failed to fetch DRPC result from DRPC server", e);
} catch (Exception e) {
LOG.error("Failed to fetch DRPC result from DRPC server", e);
@@ -219,7 +228,8 @@ public class DRPCSpout extends BaseRichSpout {
returnInfo.put("host", _local_drpc_id);
returnInfo.put("port", 0);
gotRequest = true;
- _collector.emit(new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)), new DRPCMessageId(req.get_request_id(), 0));
+ _collector.emit(new Values(req.get_func_args(), JSONValue.toJSONString(returnInfo)),
+ new DRPCMessageId(req.get_request_id(), 0));
}
} catch (AuthorizationException aze) {
throw new RuntimeException(aze);
@@ -242,17 +252,30 @@ public class DRPCSpout extends BaseRichSpout {
DRPCMessageId did = (DRPCMessageId) msgId;
DistributedRPCInvocations.Iface client;
- if(_local_drpc_id == null) {
+ if (_local_drpc_id == null) {
client = _clients.get(did.index);
} else {
client = (DistributedRPCInvocations.Iface) ServiceRegistry.getService(_local_drpc_id);
}
- try {
- client.failRequest(did.id);
- } catch (AuthorizationException aze) {
- LOG.error("Not authorized to failREquest from DRPC server", aze);
- } catch (TException e) {
- LOG.error("Failed to fail request", e);
+
+ int retryCnt = 0;
+ int maxRetries = 3;
+
+ while (retryCnt < maxRetries) {
+ retryCnt++;
+ try {
+ client.failRequest(did.id);
+ break;
+ } catch (AuthorizationException aze) {
+ LOG.error("Not authorized to failRequest from DRPC server", aze);
+ throw new RuntimeException(aze);
+ } catch (TException tex) {
+ if (retryCnt >= maxRetries) {
+ LOG.error("Failed to fail request", tex);
+ break;
+ }
+ reconnectSync((DRPCInvocationsClient)client);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/a8498888/storm-core/test/clj/org/apache/storm/drpc_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/drpc_test.clj b/storm-core/test/clj/org/apache/storm/drpc_test.clj
index a20872e..827f536 100644
--- a/storm-core/test/clj/org/apache/storm/drpc_test.clj
+++ b/storm-core/test/clj/org/apache/storm/drpc_test.clj
@@ -16,20 +16,21 @@
(ns org.apache.storm.drpc-test
(:use [clojure test])
(:import [org.apache.storm.drpc ReturnResults DRPCSpout
- LinearDRPCTopologyBuilder]
- [org.apache.storm.utils ConfigUtils Utils])
+ LinearDRPCTopologyBuilder DRPCInvocationsClient]
+ [org.apache.storm.utils ConfigUtils Utils ServiceRegistry])
(:import [org.apache.storm.topology FailedException])
(:import [org.apache.storm.coordination CoordinatedBolt$FinishedCallback])
- (:import [org.apache.storm LocalDRPC LocalCluster])
+ (:import [org.apache.storm ILocalDRPC LocalDRPC LocalCluster])
(:import [org.apache.storm.tuple Fields])
(:import [org.mockito Mockito])
(:import [org.mockito.exceptions.base MockitoAssertionError])
- (:import [org.apache.storm.utils ConfigUtils]
- [org.apache.storm.utils.staticmocking ConfigUtilsInstaller])
- (:import [org.apache.storm.generated DRPCExecutionException])
+ (:import [org.apache.storm.utils.staticmocking ConfigUtilsInstaller])
+ (:import [org.apache.storm.spout SpoutOutputCollector])
+ (:import [org.apache.storm.generated DRPCExecutionException DRPCRequest])
(:import [java.util.concurrent ConcurrentLinkedQueue])
(:import [org.apache.storm Thrift])
(:import [org.apache.storm.daemon DrpcServer])
+ (:import [org.mockito ArgumentCaptor Mockito Matchers])
(:use [org.apache.storm config testing])
(:use [org.apache.storm.internal clojure])
(:use [org.apache.storm.daemon common drpc])
@@ -255,3 +256,59 @@
(is (thrown? DRPCExecutionException
(.execute drpc-handler "ArbitraryDRPCFunctionName" "no-args"))))))
+(deftest test-drpc-attempts-two-reconnects-in-fail-request
+ (let [handler (Mockito/mock DRPCInvocationsClient
+ (.extraInterfaces (Mockito/withSettings) (into-array Class [ILocalDRPC])))
+ ;; used to capture the msgId on .emit
+ output-collector (Mockito/mock SpoutOutputCollector)
+ captor (ArgumentCaptor/forClass Object)
+ ;; pick up service-id from registry
+ service-id (ServiceRegistry/registerService handler)]
+
+ ;; mock getServiceId s.t. DRPCSpout uses our handler
+ (. (Mockito/when (.getServiceId handler)) thenReturn service-id)
+ ;; mock fetchRequest s.t. DRPCSpout has a request to process on nextTuple
+ (. (Mockito/when (.fetchRequest handler (Matchers/anyString))) thenReturn (DRPCRequest. "square 2" "bar"))
+ ;; mock failRequest s.t. DRPCSpout attempts retry on .fail
+ (.failRequest (.when (Mockito/doThrow (DRPCExecutionException.)) handler) (Matchers/anyString))
+
+ (let [spout (DRPCSpout. "test" handler)]
+ ;; tell the spout to use the mock collector
+ (.open spout nil nil output-collector)
+ ;; pick up a msgId to fail
+ (.nextTuple spout)
+ (.emit (Mockito/verify output-collector) (Mockito/anyList) (.capture captor))
+ ;; fail the msg
+ (.fail spout (.getValue captor))
+ ;; attempt 2 reconnects
+ (.reconnectClient (Mockito/verify handler (Mockito/times 2)))
+ (.failRequest (Mockito/verify handler (Mockito/times 3)) (Matchers/anyString)))))
+
+(deftest test-drpc-stops-retrying-after-successful-reconnect
+ (let [handler (Mockito/mock DRPCInvocationsClient
+ (.extraInterfaces (Mockito/withSettings) (into-array Class [ILocalDRPC])))
+ ;; used to capture the msgId on .emit
+ output-collector (Mockito/mock SpoutOutputCollector)
+ captor (ArgumentCaptor/forClass Object)
+ ;; pick up service-id from registry
+ service-id (ServiceRegistry/registerService handler)]
+
+ ;; mock getServiceId s.t. DRPCSpout uses our handler
+ (. (Mockito/when (.getServiceId handler)) thenReturn service-id)
+ ;; mock fetchRequest s.t. DRPCSpout has a request to process on nextTuple
+ (. (Mockito/when (.fetchRequest handler (Matchers/anyString))) thenReturn (DRPCRequest. "square 2" "bar"))
+ ;; mock failRequest s.t. DRPCSpout attempts retry on .fail the first time only, but succeed the second time
+ (.failRequest (.when (.doNothing
+ (Mockito/doThrow (DRPCExecutionException.))) handler) (Matchers/anyString))
+
+ (let [spout (DRPCSpout. "test" handler)]
+ ;; tell the spout to use the mock collector
+ (.open spout nil nil output-collector)
+ ;; pick up a msgId to fail
+ (.nextTuple spout)
+ (.emit (Mockito/verify output-collector) (Mockito/anyList) (.capture captor))
+ ;; fail the msg
+ (.fail spout (.getValue captor))
+ ;; reconnect once after a failure
+ (.reconnectClient (Mockito/verify handler (Mockito/times 1)))
+ (.failRequest (Mockito/verify handler (Mockito/times 2)) (Matchers/anyString)))))
[3/3] storm git commit: Added STORM-1648 to Changelog.
Posted by bo...@apache.org.
Added STORM-1648 to Changelog.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/92d30248
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/92d30248
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/92d30248
Branch: refs/heads/master
Commit: 92d302481e05cdc20215ac4f96ae24c2ba16d70e
Parents: 6fea520
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Mar 24 13:46:47 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Mar 24 13:46:47 2016 -0500
----------------------------------------------------------------------
CHANGELOG.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/92d30248/CHANGELOG.md
----------------------------------------------------------------------
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0a81117..fa52b20 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
## 2.0.0
+ * STORM-1648: drpc spout reconnect on failure
* STORM-1631: Storm CGroup bugs
* STORM-1616: Add RAS API for Trident
* STORM-1623: nimbus.clj's minor bug
[2/3] storm git commit: Merge branch
'STORM-1648_drpcspout_reconnect_in_fail' of https://github.com/abellina/storm
into STORM-1648
Posted by bo...@apache.org.
Merge branch 'STORM-1648_drpcspout_reconnect_in_fail' of https://github.com/abellina/storm into STORM-1648
STORM-1648: drpc spout reconnect on failure
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6fea5207
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6fea5207
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6fea5207
Branch: refs/heads/master
Commit: 6fea5207381a958e2c0a08e5ab79d6ffc4896782
Parents: df3867f a849888
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Mar 24 13:35:25 2016 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Thu Mar 24 13:35:25 2016 -0500
----------------------------------------------------------------------
.../jvm/org/apache/storm/drpc/DRPCSpout.java | 51 +++++++++++----
.../test/clj/org/apache/storm/drpc_test.clj | 69 ++++++++++++++++++--
2 files changed, 100 insertions(+), 20 deletions(-)
----------------------------------------------------------------------