You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2018/02/04 00:51:46 UTC
[43/50] [abbrv] kylin git commit: KYLIN-2880 Enhance BadQueryDetector
to include query id
KYLIN-2880 Enhance BadQueryDetector to include query id
Signed-off-by: lidongsjtu <li...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6a2ee80c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6a2ee80c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6a2ee80c
Branch: refs/heads/sync
Commit: 6a2ee80c9ec19be1d7e72ed8b587fbc48a195867
Parents: c704f7c
Author: Zhong <nj...@apache.org>
Authored: Tue Jan 30 14:13:18 2018 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Tue Jan 30 21:45:18 2018 +0800
----------------------------------------------------------------------
.../kylin/metadata/badquery/BadQueryEntry.java | 14 +++++++++-
.../badquery/BadQueryHistoryManagerTest.java | 15 ++++++----
.../kylin/rest/service/BadQueryDetector.java | 29 +++++++++++---------
.../apache/kylin/rest/service/QueryService.java | 6 ++--
.../rest/service/BadQueryDetectorTest.java | 6 ++--
.../rest/service/KylinQueryTimeoutTest.java | 4 ++-
6 files changed, 49 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
index 1e9311f..56b27ca 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
@@ -47,8 +47,11 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
private String thread;
@JsonProperty("user")
private String user;
+ @JsonProperty("query_id")
+ private String queryId;
- public BadQueryEntry(String sql, String adj, long startTime, float runningSec, String server, String thread, String user) {
+ public BadQueryEntry(String sql, String adj, long startTime, float runningSec, String server, String thread,
+ String user, String queryId) {
this.updateRandomUuid();
this.adj = adj;
this.sql = sql;
@@ -57,12 +60,21 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
this.server = server;
this.thread = thread;
this.user = user;
+ this.queryId = queryId;
}
public BadQueryEntry() {
}
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
public String getUser() {
return user;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
index c4a8229..5001cd2 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.NavigableSet;
+import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.JsonUtil;
@@ -65,7 +66,8 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
public void testAddEntryToProject() throws IOException {
KylinConfig kylinConfig = getTestConfig();
BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
- BadQueryEntry entry = new BadQueryEntry("sql", "adj", 1459362239992L, 100, "server", "t-0", "user");
+ BadQueryEntry entry = new BadQueryEntry("sql", "adj", 1459362239992L, 100, "server", "t-0", "user",
+ UUID.randomUUID().toString());
BadQueryHistory history = manager.upsertEntryToProject(entry, "default");
NavigableSet<BadQueryEntry> entries = history.getEntries();
assertEquals(4, entries.size());
@@ -81,7 +83,8 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
assertEquals("t-0", newEntry.getThread());
for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) {
- BadQueryEntry tmp = new BadQueryEntry("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "user");
+ BadQueryEntry tmp = new BadQueryEntry("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "user",
+ UUID.randomUUID().toString());
history = manager.upsertEntryToProject(tmp, "default");
}
assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size());
@@ -92,14 +95,16 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
KylinConfig kylinConfig = getTestConfig();
BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
- manager.upsertEntryToProject(new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user"),
+ String queryId = UUID.randomUUID().toString();
+ manager.upsertEntryToProject(
+ new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user", queryId),
"default");
BadQueryHistory history = manager.upsertEntryToProject(
- new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user"), "default");
+ new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId), "default");
NavigableSet<BadQueryEntry> entries = history.getEntries();
BadQueryEntry newEntry = entries
- .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user"));
+ .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId));
System.out.println(newEntry);
assertEquals("adj2", newEntry.getAdj());
assertEquals("server2", newEntry.getServer());
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 4f7bccf..51f49a7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -102,15 +102,15 @@ public class BadQueryDetector extends Thread {
for (Notifier notifier : notifiers) {
try {
notifier.badQueryFound(adj, runningSec, //
- e.startTime, e.sqlRequest.getProject(), e.sqlRequest.getSql(), e.user, e.thread);
+ e.startTime, e.sqlRequest.getProject(), e.sqlRequest.getSql(), e.user, e.thread, e.queryId);
} catch (Exception ex) {
logger.error("", ex);
}
}
}
- public void queryStart(Thread thread, SQLRequest sqlRequest, String user) {
- runningQueries.put(thread, new Entry(sqlRequest, user, thread));
+ public void queryStart(Thread thread, SQLRequest sqlRequest, String user, String queryId) {
+ runningQueries.put(thread, new Entry(sqlRequest, user, thread, queryId));
}
public void queryEnd(Thread thread) {
@@ -155,7 +155,7 @@ public class BadQueryDetector extends Thread {
if (runningSec >= alertRunningSec) {
notify(BadQueryEntry.ADJ_SLOW, e);
- dumpStackTrace(e.thread);
+ dumpStackTrace(e.thread, e.queryId);
} else {
break; // entries are sorted by startTime
}
@@ -175,12 +175,13 @@ public class BadQueryDetector extends Thread {
}
// log the stack trace of bad query thread for further analysis
- private void dumpStackTrace(Thread t) {
+ private void dumpStackTrace(Thread t, String queryId) {
int maxStackTraceDepth = kylinConfig.getBadQueryStackTraceDepth();
int current = 0;
StackTraceElement[] stackTrace = t.getStackTrace();
- StringBuilder buf = new StringBuilder("Problematic thread 0x" + Long.toHexString(t.getId()));
+ StringBuilder buf = new StringBuilder(
+ "Problematic thread 0x" + Long.toHexString(t.getId()) + " " + t.getName() + ", query id: " + queryId);
buf.append("\n");
for (StackTraceElement e : stackTrace) {
if (++current > maxStackTraceDepth) {
@@ -193,15 +194,15 @@ public class BadQueryDetector extends Thread {
public interface Notifier {
void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user,
- Thread t);
+ Thread t, String queryId);
}
private class LoggerNotifier implements Notifier {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user,
- Thread t) {
- logger.info("{} query has been running {} seconds (project:{}, thread: 0x{}, user:{}) -- {}", adj,
- runningSec, project, Long.toHexString(t.getId()), user, sql);
+ Thread t, String queryId) {
+ logger.info("{} query has been running {} seconds (project:{}, thread: 0x{}, user:{}, query id:{}) -- {}",
+ adj, runningSec, project, Long.toHexString(t.getId()), user, queryId, sql);
}
}
@@ -220,10 +221,10 @@ public class BadQueryDetector extends Thread {
@Override
public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user,
- Thread t) {
+ Thread t, String queryId) {
try {
BadQueryEntry entry = new BadQueryEntry(sql, adj, startTime, runningSec, serverHostname, t.getName(),
- user);
+ user, queryId);
badQueryManager.upsertEntryToProject(entry, project);
} catch (IOException e) {
logger.error("Error in bad query persistence.", e);
@@ -236,12 +237,14 @@ public class BadQueryDetector extends Thread {
final long startTime;
final Thread thread;
final String user;
+ final String queryId;
- Entry(SQLRequest sqlRequest, String user, Thread thread) {
+ Entry(SQLRequest sqlRequest, String user, Thread thread, String queryId) {
this.sqlRequest = sqlRequest;
this.startTime = System.currentTimeMillis();
this.thread = thread;
this.user = user;
+ this.queryId = queryId;
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 56fab34..841cad9 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -182,11 +182,11 @@ public class QueryService extends BasicService {
return getMetadata(getCubeManager(), project);
}
- public SQLResponse query(SQLRequest sqlRequest) throws Exception {
+ public SQLResponse query(SQLRequest sqlRequest, String queryId) throws Exception {
SQLResponse ret = null;
try {
final String user = SecurityContextHolder.getContext().getAuthentication().getName();
- badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, user);
+ badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, user, queryId);
ret = queryWithSqlMassage(sqlRequest);
return ret;
@@ -492,7 +492,7 @@ public class QueryService extends BasicService {
try {
final boolean isSelect = QueryUtil.isSelectStatement(sqlRequest.getSql());
if (isSelect) {
- sqlResponse = query(sqlRequest);
+ sqlResponse = query(sqlRequest, queryContext.getQueryId());
Trace.addTimelineAnnotation("query almost done");
} else if (kylinConfig.isPushDownEnabled() && kylinConfig.isPushDownUpdateEnabled()) {
sqlResponse = update(sqlRequest);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index d61dfbe..61e52b7 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
+import java.util.UUID;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.badquery.BadQueryEntry;
@@ -53,7 +54,8 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec, 1000);
badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
@Override
- public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql, String user, Thread t) {
+ public void badQueryFound(String adj, float runningSec, long startTime, String project, String sql,
+ String user, Thread t, String queryId) {
alerts.add(new String[] { adj, sql });
}
});
@@ -64,7 +66,7 @@ public class BadQueryDetectorTest extends LocalFileMetadataTestCase {
SQLRequest sqlRequest = new SQLRequest();
sqlRequest.setSql(mockSql);
- badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, "user");
+ badQueryDetector.queryStart(Thread.currentThread(), sqlRequest, "user", UUID.randomUUID().toString());
// make sure bad query check happens twice
Thread.sleep((alertRunningSec * 2 + 1) * 1000);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6a2ee80c/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
index 6a61b2f..38b3270 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/KylinQueryTimeoutTest.java
@@ -17,7 +17,9 @@
*/
package org.apache.kylin.rest.service;
+
import java.sql.SQLException;
+import java.util.UUID;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.exceptions.KylinTimeoutException;
@@ -69,7 +71,7 @@ public class KylinQueryTimeoutTest extends LocalFileMetadataTestCase {
SQLRequest request = new SQLRequest();
request.setProject("default");
request.setSql("select count(*) from STREAMING_TABLE");
- detector.queryStart(Thread.currentThread(), request, "ADMIN");
+ detector.queryStart(Thread.currentThread(), request, "ADMIN", UUID.randomUUID().toString());
try {
QueryACLTestUtil.mockQuery("default", "select * from STREAMING_TABLE");
} finally{