You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/10/26 10:07:58 UTC
[12/45] incubator-kylin git commit: KYLIN-967 kill long running query
when low memory detected
KYLIN-967 kill long running query when low memory detected
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/c611d7ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/c611d7ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/c611d7ee
Branch: refs/heads/master
Commit: c611d7eefe78e8b84705b6b0e3d4af29e1686b9e
Parents: bad57cb
Author: shaofengshi <sh...@apache.org>
Authored: Fri Sep 25 16:54:43 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Sep 25 23:31:31 2015 +0800
----------------------------------------------------------------------
.../kylin/rest/service/BadQueryDetector.java | 37 ++++++++++++++------
.../rest/service/BadQueryDetectorTest.java | 6 ++--
2 files changed, 29 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c611d7ee/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 5f5b247..d3315c0 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -20,6 +20,7 @@ package org.apache.kylin.rest.service;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.rest.request.SQLRequest;
@@ -36,19 +37,21 @@ public class BadQueryDetector extends Thread {
private final long detectionInterval;
private final int alertMB;
private final int alertRunningSec;
+ private final int killRunningSec;
private ArrayList<Notifier> notifiers = new ArrayList<Notifier>();
public BadQueryDetector() {
- this(60 * 1000, 100, 60); // 1 minute, 100 MB, 60 seconds
+ this(60 * 1000, 100, 60, 5 * 60); // 1 minute, 100 MB, 60 seconds, 5 minutes
}
- public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec) {
+ public BadQueryDetector(long detectionInterval, int alertMB, int alertRunningSec, int killRunningSec) {
super("BadQueryDetector");
this.setDaemon(true);
this.detectionInterval = detectionInterval;
this.alertMB = alertMB;
this.alertRunningSec = alertRunningSec;
+ this.killRunningSec = killRunningSec;
this.notifiers.add(new Notifier() {
@Override
@@ -121,15 +124,6 @@ public class BadQueryDetector extends Thread {
ArrayList<Entry> entries = new ArrayList<Entry>(runningQueries.values());
Collections.sort(entries);
- // report if low memory
- if (getSystemAvailMB() < alertMB) {
- logger.info("System free memory less than " + alertMB + " MB. " + entries.size() + " queries running.");
- for (int i = 0; i < entries.size(); i++) {
- Entry e = entries.get(i);
- notify("Low mem", (int) ((now - e.startTime) / 1000), e.sqlRequest.getSql());
- }
- }
-
// report if query running long
for (Entry e : entries) {
int runningSec = (int) ((now - e.startTime) / 1000);
@@ -139,6 +133,27 @@ public class BadQueryDetector extends Thread {
break; // entries are sorted by startTime
}
}
+
+ // report if low memory
+ if (getSystemAvailMB() < alertMB) {
+ logger.info("System free memory less than " + alertMB + " MB. " + entries.size() + " queries running.");
+
+ for (Map.Entry<Thread, Entry> mapEntry : runningQueries.entrySet()) {
+ Entry e = mapEntry.getValue();
+ int duration = (int) ((now - e.startTime) / 1000);
+ if (duration > killRunningSec) {
+ notify("Kill", duration, e.sqlRequest.getSql());
+ Thread queryThread = mapEntry.getKey();
+ killQueryThread(queryThread);
+ } else {
+ notify("Low mem", duration, e.sqlRequest.getSql());
+ }
+ }
+ }
+ }
+
+ private void killQueryThread(Thread thread) {
+ thread.interrupt();
}
public static final int ONE_MB = 1024 * 1024;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/c611d7ee/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
index c849efd..d7e8ba9 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/BadQueryDetectorTest.java
@@ -35,7 +35,7 @@ public class BadQueryDetectorTest {
String mockSql = "select * from just_a_test";
final ArrayList<String[]> alerts = new ArrayList<>();
- BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec);
+ BadQueryDetector badQueryDetector = new BadQueryDetector(alertRunningSec * 1000, alertMB, alertRunningSec, alertRunningSec * 5);
badQueryDetector.registerNotifier(new BadQueryDetector.Notifier() {
@Override
public void badQueryFound(String adj, int runningSec, String sql) {
@@ -63,7 +63,7 @@ public class BadQueryDetectorTest {
// first check founds Low mem
assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(0));
// second check founds Low mem & Slow
- assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(1));
- assertArrayEquals(new String[] { "Slow", mockSql }, alerts.get(2));
+ assertArrayEquals(new String[] { "Slow", mockSql }, alerts.get(1));
+ assertArrayEquals(new String[] { "Low mem", mockSql }, alerts.get(2));
}
}