You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2016/09/26 13:09:40 UTC
asterixdb git commit: Timeout NC thread dump requests after 60 seconds
Repository: asterixdb
Updated Branches:
refs/heads/master 25c43f1e0 -> 63fdfaeb6
Timeout NC thread dump requests after 60 seconds
Change-Id: If4840c78a6f6a2916ee682a9061df62a50bedc8a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1213
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/63fdfaeb
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/63fdfaeb
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/63fdfaeb
Branch: refs/heads/master
Commit: 63fdfaeb6a1de808a72946463561d2de679f18ae
Parents: 25c43f1
Author: Michael Blow <mb...@apache.org>
Authored: Mon Sep 26 00:43:37 2016 -0400
Committer: Michael Blow <mb...@apache.org>
Committed: Mon Sep 26 06:08:41 2016 -0700
----------------------------------------------------------------------
.../control/cc/work/GetThreadDumpWork.java | 26 ++++++++++++++++++++
.../cc/work/NotifyThreadDumpResponse.java | 11 ++++++++-
.../control/nc/work/NodeThreadDumpWork.java | 1 -
3 files changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/63fdfaeb/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
index be53232..7931cf8 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
@@ -20,6 +20,10 @@ package org.apache.hyracks.control.cc.work;
import java.lang.management.ManagementFactory;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.cc.NodeControllerState;
@@ -27,6 +31,9 @@ import org.apache.hyracks.control.common.work.IResultCallback;
import org.apache.hyracks.control.common.work.ThreadDumpWork;
public class GetThreadDumpWork extends ThreadDumpWork {
+ private static final Logger LOGGER = Logger.getLogger(ThreadDumpWork.class.getName());
+ public static final int TIMEOUT_SECS = 60;
+
private final ClusterControllerService ccs;
private final String nodeId;
private final IResultCallback<String> callback;
@@ -55,8 +62,27 @@ public class GetThreadDumpWork extends ThreadDumpWork {
try {
ncState.getNodeController().takeThreadDump(run.getRequestId());
} catch (Exception e) {
+ ccs.removeThreadDumpRun(run.getRequestId());
callback.setException(e);
}
+ final long requestTime = System.currentTimeMillis();
+ ccs.getExecutor().execute(() -> {
+ try {
+ final long queueTime = System.currentTimeMillis() - requestTime;
+ final long sleepTime = TimeUnit.SECONDS.toMillis(TIMEOUT_SECS) - queueTime;
+ if (sleepTime > 0) {
+ Thread.sleep(sleepTime);
+ }
+ if (ccs.removeThreadDumpRun(run.getRequestId()) != null) {
+ LOGGER.log(Level.WARNING, "Timed out thread dump request " + run.getRequestId()
+ + " for node " + nodeId);
+ callback.setException(new TimeoutException("Thread dump request for node " + nodeId
+ + " timed out after " + TIMEOUT_SECS + " seconds."));
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ });
}
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/63fdfaeb/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
index bbdf211..2dae4b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/NotifyThreadDumpResponse.java
@@ -18,10 +18,13 @@
*/
package org.apache.hyracks.control.cc.work;
+import java.util.logging.Logger;
+
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.common.work.AbstractWork;
public class NotifyThreadDumpResponse extends AbstractWork {
+ private static final Logger LOGGER = Logger.getLogger(NotifyThreadDumpResponse.class.getName());
private final ClusterControllerService ccs;
@@ -36,6 +39,12 @@ public class NotifyThreadDumpResponse extends AbstractWork {
@Override
public void run() {
- ccs.removeThreadDumpRun(requestId).notifyThreadDumpReceived(threadDumpJSON);
+ LOGGER.fine("Delivering thread dump response: " + requestId);
+ final GetThreadDumpWork.ThreadDumpRun threadDumpRun = ccs.removeThreadDumpRun(requestId);
+ if (threadDumpRun == null) {
+ LOGGER.warning("Thread dump run " + requestId + " not found; discarding reply: " + threadDumpJSON);
+ } else {
+ threadDumpRun.notifyThreadDumpReceived(threadDumpJSON);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/63fdfaeb/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
index 1fc4690..85233b2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NodeThreadDumpWork.java
@@ -33,7 +33,6 @@ public class NodeThreadDumpWork extends ThreadDumpWork {
@Override
protected void doRun() throws Exception {
final String result = takeDump(ncs.getThreadMXBean());
-
ncs.getClusterController().notifyThreadDump(
ncs.getApplicationContext().getNodeId(), requestId, result);
}