You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2017/08/26 02:12:23 UTC
asterixdb git commit: [NO ISSUE] Refactoring / cleanup of HTTP
cluster state / diagnostics APIs
Repository: asterixdb
Updated Branches:
refs/heads/master a3c47764a -> b3637a6a2
[NO ISSUE] Refactoring / cleanup of HTTP cluster state / diagnostics APIs
Change-Id: If47ec45bf88a39d63421903080fee3ddc0f1e42b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1969
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b3637a6a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b3637a6a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b3637a6a
Branch: refs/heads/master
Commit: b3637a6a2902033661eff93da6906cb97020e67d
Parents: a3c4776
Author: Michael Blow <mb...@apache.org>
Authored: Fri Aug 25 00:14:22 2017 -0400
Committer: Michael Blow <mb...@apache.org>
Committed: Fri Aug 25 19:12:02 2017 -0700
----------------------------------------------------------------------
.../api/http/server/DiagnosticsApiServlet.java | 80 ++++++++++++--------
.../server/NodeControllerDetailsApiServlet.java | 5 +-
.../hyracks/bootstrap/CCApplication.java | 2 +-
.../asterix/test/runtime/LangExecutionUtil.java | 6 +-
.../api/diagnostics_1/diagnostics_1.1.regexadm | 9 ++-
.../control/cc/work/GetThreadDumpWork.java | 2 +-
.../control/common/utils/ThreadDumpHelper.java | 25 +++---
.../hyracks/control/nc/task/ThreadDumpTask.java | 2 +-
8 files changed, 76 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
index 9d2415d..b64b6f6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/DiagnosticsApiServlet.java
@@ -34,25 +34,32 @@ import java.util.concurrent.Future;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.runtime.utils.ClusterStateManager;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
+import org.apache.hyracks.util.JSONUtil;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
import io.netty.handler.codec.http.HttpResponseStatus;
public class DiagnosticsApiServlet extends NodeControllerDetailsApiServlet {
private static final Logger LOGGER = Logger.getLogger(DiagnosticsApiServlet.class.getName());
- private final ICcApplicationContext appCtx;
+ protected final ObjectMapper om;
+ protected final IHyracksClientConnection hcc;
+ protected final ExecutorService executor;
- public DiagnosticsApiServlet(ConcurrentMap<String, Object> ctx, String[] paths, ICcApplicationContext appCtx) {
+ public DiagnosticsApiServlet(ConcurrentMap<String, Object> ctx, String[] paths) {
super(ctx, paths);
- this.appCtx = appCtx;
+ this.om = new ObjectMapper();
+ this.hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
+ this.executor = (ExecutorService) ctx.get(ServletConstants.EXECUTOR_SERVICE_ATTR);
}
@Override
@@ -60,15 +67,13 @@ public class DiagnosticsApiServlet extends NodeControllerDetailsApiServlet {
HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, HttpUtil.Encoding.UTF8);
PrintWriter responseWriter = response.writer();
ObjectNode json;
- ObjectMapper om = new ObjectMapper();
response.setStatus(HttpResponseStatus.OK);
om.enable(SerializationFeature.INDENT_OUTPUT);
try {
if (!"".equals(localPath(request))) {
throw new IllegalArgumentException();
}
- json = getClusterDiagnosticsJSON();
- responseWriter.write(om.writerWithDefaultPrettyPrinter().writeValueAsString(json));
+ responseWriter.write(JSONUtil.convertNode(getClusterDiagnosticsJSON()));
} catch (IllegalStateException e) { // NOSONAR - exception not logged or rethrown
response.setStatus(HttpResponseStatus.SERVICE_UNAVAILABLE);
} catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
@@ -81,44 +86,53 @@ public class DiagnosticsApiServlet extends NodeControllerDetailsApiServlet {
responseWriter.flush();
}
- private ObjectNode getClusterDiagnosticsJSON() throws Exception {
- ObjectMapper om = new ObjectMapper();
- IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
- ExecutorService executor = (ExecutorService) ctx.get(ServletConstants.EXECUTOR_SERVICE_ATTR);
- Map<String, Future<ObjectNode>> ccFutureData = new HashMap<>();
- ccFutureData.put("threaddump",
- executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getThreadDump(null)))));
- ccFutureData.put("config",
- executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getNodeDetailsJSON(null, false, true)))));
- ccFutureData.put("stats",
- executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getNodeDetailsJSON(null, true, false)))));
+ protected ObjectNode getClusterDiagnosticsJSON() throws Exception {
+ Map<String, Future<JsonNode>> ccFutureData;
+ ccFutureData = getCcDiagosticsFutures();
- Map<String, Map<String, Future<ObjectNode>>> ncDataMap = new HashMap<>();
- for (String nc : appCtx.getMetadataProperties().getNodeNames()) {
- Map<String, Future<ObjectNode>> ncData = new HashMap<>();
- ncData.put("threaddump", executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getThreadDump(nc)))));
- ncData.put("config", executor
- .submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getNodeDetailsJSON(nc, false, true)))));
- ncData.put("stats", executor.submit(() -> fixupKeys(processNodeStats(hcc, nc))));
- ncDataMap.put(nc, ncData);
+ Map<String, Map<String, Future<JsonNode>>> ncDataMap = new HashMap<>();
+ for (String nc : ClusterStateManager.INSTANCE.getParticipantNodes()) {
+ ncDataMap.put(nc, getNcDiagnosticFutures(nc));
}
ObjectNode result = om.createObjectNode();
result.putPOJO("cc", resolveFutures(ccFutureData));
List<Map<String, ?>> ncList = new ArrayList<>();
- for (Map.Entry<String, Map<String, Future<ObjectNode>>> entry : ncDataMap.entrySet()) {
- final Map<String, Object> ncMap = resolveFutures(entry.getValue());
- ncMap.put("node_id", entry.getKey());
+ for (Map.Entry<String, Map<String, Future<JsonNode>>> entry : ncDataMap.entrySet()) {
+ final Map<String, JsonNode> ncMap = resolveFutures(entry.getValue());
+ ncMap.put("node_id", new TextNode(entry.getKey()));
ncList.add(ncMap);
}
result.putPOJO("ncs", ncList);
- result.putPOJO("date", new Date());
+ result.put("date", String.valueOf(new Date()));
return result;
}
- private Map<String, Object> resolveFutures(Map<String, Future<ObjectNode>> futureMap)
+ protected Map<String, Future<JsonNode>> getNcDiagnosticFutures(String nc) {
+ Map<String, Future<JsonNode>> ncData;
+ ncData = new HashMap<>();
+ ncData.put("threaddump", executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getThreadDump(nc)))));
+ ncData.put("config",
+ executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getNodeDetailsJSON(nc, false, true)))));
+ ncData.put("stats", executor.submit(() -> fixupKeys(processNodeStats(hcc, nc))));
+ return ncData;
+ }
+
+ protected Map<String, Future<JsonNode>> getCcDiagosticsFutures() {
+ Map<String, Future<JsonNode>> ccFutureData;
+ ccFutureData = new HashMap<>();
+ ccFutureData.put("threaddump",
+ executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getThreadDump(null)))));
+ ccFutureData.put("config",
+ executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getNodeDetailsJSON(null, false, true)))));
+ ccFutureData.put("stats",
+ executor.submit(() -> fixupKeys((ObjectNode) om.readTree(hcc.getNodeDetailsJSON(null, true, false)))));
+ return ccFutureData;
+ }
+
+ protected Map<String, JsonNode> resolveFutures(Map<String, Future<JsonNode>> futureMap)
throws ExecutionException, InterruptedException {
- Map<String, Object> result = new HashMap<>();
- for (Map.Entry<String, Future<ObjectNode>> entry : futureMap.entrySet()) {
+ Map<String, JsonNode> result = new HashMap<>();
+ for (Map.Entry<String, Future<JsonNode>> entry : futureMap.entrySet()) {
result.put(entry.getKey(), entry.getValue().get());
}
return result;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
index 6291869..01c59f3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NodeControllerDetailsApiServlet.java
@@ -180,8 +180,9 @@ public class NodeControllerDetailsApiServlet extends ClusterApiServlet {
for (int i = 0; i < gcNames.size(); i++) {
ObjectNode gc = om.createObjectNode();
gc.set("name", gcNames.get(i));
- gc.set("collection-time", ((ArrayNode) gcCollectionTimes.get(i)).get(index));
- gc.set("collection-count", ((ArrayNode) gcCollectionCounts.get(i)).get(index));
+ gc.set("collection-time", gcCollectionTimes.get(i).get(index));
+ gc.set("collection-count", gcCollectionCounts.get(i).get(index));
+ fixupKeys(gc);
gcs.add(gc);
}
json.set("gcs", gcs);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 9040ad1..b08c1e2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -313,7 +313,7 @@ public class CCApplication extends BaseCCApplication {
case Servlets.CLUSTER_STATE_CC_DETAIL:
return new ClusterControllerDetailsApiServlet(ctx, paths);
case Servlets.DIAGNOSTICS:
- return new DiagnosticsApiServlet(ctx, paths, appCtx);
+ return new DiagnosticsApiServlet(ctx, paths);
case Servlets.ACTIVE_STATS:
return new ActiveStatsApiServlet(ctx, paths, appCtx);
default:
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index be84bc3..c5b9d11 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -19,7 +19,7 @@
package org.apache.asterix.test.runtime;
-import static org.apache.hyracks.control.common.utils.ThreadDumpHelper.takeDumpJSON;
+import static org.apache.hyracks.control.common.utils.ThreadDumpHelper.takeDumpJSONString;
import java.io.BufferedReader;
import java.io.File;
@@ -190,7 +190,7 @@ public class LangExecutionUtil {
}
private static void checkThreadLeaks() throws IOException {
- String threadDump = ThreadDumpHelper.takeDumpJSON(ManagementFactory.getThreadMXBean());
+ String threadDump = ThreadDumpHelper.takeDumpJSONString(ManagementFactory.getThreadMXBean());
// Currently we only do sanity check for threads used in the execution engine.
// Later we should check if there are leaked storage threads as well.
if (threadDump.contains("Operator") || threadDump.contains("SuperActivity")
@@ -215,7 +215,7 @@ public class LangExecutionUtil {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
int runFileCount = Integer.parseInt(reader.readLine().trim());
if (runFileCount != 0) {
- System.out.print(takeDumpJSON(ManagementFactory.getThreadMXBean()));
+ System.out.print(takeDumpJSONString(ManagementFactory.getThreadMXBean()));
outputLeakedOpenFiles(processId);
throw new AssertionError("There are " + runFileCount + " leaked run files.");
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm
index b2dafd9..b0d6638 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/diagnostics_1/diagnostics_1.1.regexadm
@@ -1,7 +1,10 @@
.*"cc" : \{.*
+ "config" : \{.*
+ \},.*
"stats" : \{.*
\},.*
"threaddump" : \{.*
- \},.*
- "config" : \{.*
- \}.*"ncs".*"node_id" : "asterix_nc1".*"threaddump".*"node_id" : "asterix_nc2".*"threaddump".*
\ No newline at end of file
+ \}.*
+ },
+ "date" : "[SMTWF][^"]* [^"]*[0-9]{4}",
+ "ncs".*"node_id" : "asterix_nc1".*"threaddump".*"node_id" : "asterix_nc2".*"threaddump".*
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
index d7c7be1..b5388c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
@@ -55,7 +55,7 @@ public class GetThreadDumpWork extends AbstractWork {
if (nodeId == null) {
// null nodeId means the request is for the cluster controller
try {
- callback.setValue(ThreadDumpHelper.takeDumpJSON(ManagementFactory.getThreadMXBean()));
+ callback.setValue(ThreadDumpHelper.takeDumpJSONString(ManagementFactory.getThreadMXBean()));
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Exception taking CC thread dump", e);
callback.setException(e);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java
index 1d6dbcd..62c6586 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/ThreadDumpHelper.java
@@ -18,12 +18,6 @@
*/
package org.apache.hyracks.control.common.utils;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
import java.io.IOException;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
@@ -33,12 +27,23 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class ThreadDumpHelper {
+ private static final ObjectMapper om = new ObjectMapper();
private ThreadDumpHelper() {
+ om.enable(SerializationFeature.INDENT_OUTPUT);
}
- public static String takeDumpJSON(ThreadMXBean threadMXBean) throws IOException {
+ public static String takeDumpJSONString(ThreadMXBean threadMXBean) throws IOException {
+ ObjectNode json = takeDumpJSON(threadMXBean);
+ return om.writerWithDefaultPrettyPrinter().writeValueAsString(json);
+ }
+
+ public static ObjectNode takeDumpJSON(ThreadMXBean threadMXBean) {
ThreadInfo[] threadInfos = threadMXBean.dumpAllThreads(true, true);
List<Map<String, Object>> threads = new ArrayList<>();
@@ -73,9 +78,8 @@ public class ThreadDumpHelper {
}
threads.add(threadMap);
}
- ObjectMapper om = new ObjectMapper();
ObjectNode json = om.createObjectNode();
- json.put("date", new Date().toString());
+ json.put("date", String.valueOf(new Date()));
json.putPOJO("threads", threads);
long[] deadlockedThreads = threadMXBean.findDeadlockedThreads();
@@ -86,7 +90,6 @@ public class ThreadDumpHelper {
if (monitorDeadlockedThreads != null && monitorDeadlockedThreads.length > 0) {
json.putPOJO("monitor_deadlocked_thread_ids", monitorDeadlockedThreads);
}
- om.enable(SerializationFeature.INDENT_OUTPUT);
- return om.writerWithDefaultPrettyPrinter().writeValueAsString(json);
+ return json;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b3637a6a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
index 5ebb99a..abde87f 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/task/ThreadDumpTask.java
@@ -38,7 +38,7 @@ public class ThreadDumpTask implements Runnable {
public void run() {
String result;
try {
- result = ThreadDumpHelper.takeDumpJSON(ncs.getThreadMXBean());
+ result = ThreadDumpHelper.takeDumpJSONString(ncs.getThreadMXBean());
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Exception taking thread dump", e);
result = null;