You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/22 06:11:52 UTC
[23/50] [abbrv] kylin git commit: KYLIN-1546 Refine bad query detect
KYLIN-1546 Refine bad query detect
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/30aa60d6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/30aa60d6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/30aa60d6
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 30aa60d6586292bd875e5a6150249bf9b0d6638a
Parents: fc14ef7
Author: lidongsjtu <li...@apache.org>
Authored: Fri Apr 1 23:28:50 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Apr 1 23:28:50 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 ++++-
.../kylin/metadata/badquery/BadQueryEntry.java | 28 ++++++++++++++++++--
.../badquery/BadQueryHistoryManager.java | 22 ++++++++++++++-
.../badquery/BadQueryHistoryManagerTest.java | 21 +++++++++++++--
.../kylin/rest/service/BadQueryDetector.java | 11 +++++---
.../apache/kylin/tool/CubeMetaExtractor.java | 11 +++++---
6 files changed, 87 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 2fe13a0..6a0f860 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -451,7 +451,11 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getBadQueryDefaultDetectIntervalSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery.default.alerting.seconds", "60"));
+ return Integer.parseInt(getOptional("kylin.query.badquery.detect.interval.seconds", "60"));
+ }
+
+ public boolean getBadQueryPersistentEnabled() {
+ return Boolean.parseBoolean(getOptional("kylin.query.badquery.persistent.enable", "true"));
}
public int getCachedDictMaxEntrySize() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
index b15846c..d795954 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
@@ -103,11 +103,35 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
@Override
public int compareTo(BadQueryEntry obj) {
- return this.startTime >= obj.startTime ? 1 : -1;
+ if (this.startTime == obj.startTime) {
+ return 0;
+ } else if (this.startTime > obj.startTime) {
+ return 1;
+ } else {
+ return -1;
+ }
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ BadQueryEntry entry = (BadQueryEntry) o;
+
+ if (!sql.equals(entry.sql))
+ return false;
+
+ if (startTime != entry.startTime)
+ return false;
+
+ return true;
}
@Override
public String toString() {
- return "BadQueryEntry [ adj=" + adj + ", server=" + server + ", startTime=" + DateFormat.formatToTimeStr(startTime) + "]";
+ return "BadQueryEntry [ adj=" + adj + ", server=" + server + ", startTime=" + DateFormat.formatToTimeStr(startTime) + " ]";
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
index 1945fed..5d532aa 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
@@ -100,10 +100,30 @@ public class BadQueryHistoryManager {
return badQueryHistory;
}
- public BadQueryHistory addEntryToProject(String sql, String adj, long startTime, int runningSecs, String server, String threadName, String project) throws IOException {
+ public BadQueryHistory updateEntryToProject(BadQueryEntry badQueryEntry, String project) throws IOException {
+ if (StringUtils.isEmpty(project) || badQueryEntry.getAdj() == null || badQueryEntry.getSql() == null)
+ throw new IllegalArgumentException();
+
+ BadQueryHistory badQueryHistory = getBadQueriesForProject(project);
+ NavigableSet<BadQueryEntry> entries = badQueryHistory.getEntries();
+ BadQueryEntry entry = entries.floor(badQueryEntry);
+ entry.setAdj(badQueryEntry.getAdj());
+ entry.setRunningSec(badQueryEntry.getRunningSec());
+ entry.setServer(badQueryEntry.getServer());
+ entry.setThread(badQueryEntry.getThread());
+ getStore().putResource(badQueryHistory.getResourcePath(), badQueryHistory, BAD_QUERY_INSTANCE_SERIALIZER);
+
+ return badQueryHistory;
+ }
+
+ public BadQueryHistory addEntryToProject(String sql, long startTime, String adj, int runningSecs, String server, String threadName, String project) throws IOException {
return addEntryToProject(new BadQueryEntry(sql, adj, startTime, runningSecs, server, threadName), project);
}
+ public BadQueryHistory updateEntryToProject(String sql, long startTime, String adj, int runningSecs, String server, String threadName, String project) throws IOException {
+ return updateEntryToProject(new BadQueryEntry(sql, adj, startTime, runningSecs, server, threadName), project);
+ }
+
public void removeBadQueryHistory(String project) throws IOException {
getStore().deleteResource(getResourcePathForProject(project));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
index c684598..949f11f 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
@@ -63,7 +63,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
public void testAddEntryToProject() throws IOException {
KylinConfig kylinConfig = getTestConfig();
BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
- BadQueryHistory history = manager.addEntryToProject("sql", "adj", 1459362239992L, 100, "server", "t-0", "default");
+ BadQueryHistory history = manager.addEntryToProject("sql", 1459362239992L, "adj", 100, "server", "t-0", "default");
NavigableSet<BadQueryEntry> entries = history.getEntries();
assertEquals(3, entries.size());
@@ -77,9 +77,26 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
assertEquals("t-0", newEntry.getThread());
for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) {
- history = manager.addEntryToProject("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "default");
+ history = manager.addEntryToProject("sql", 1459362239993L + i, "adj", 100 + i, "server", "t-0", "default");
}
assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size());
}
+ @Test
+ public void testUpdateEntryToProject() throws IOException {
+ KylinConfig kylinConfig = getTestConfig();
+ BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
+
+ manager.addEntryToProject("sql", 1459362239000L, "adj", 100, "server", "t-0", "default");
+ BadQueryHistory history = manager.updateEntryToProject("sql", 1459362239000L, "adj2", 120, "server2", "t-1", "default");
+
+ NavigableSet<BadQueryEntry> entries = history.getEntries();
+ BadQueryEntry newEntry = entries.floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1"));
+ System.out.println(newEntry);
+ assertEquals("adj2", newEntry.getAdj());
+ assertEquals("server2", newEntry.getServer());
+ assertEquals("t-1", newEntry.getThread());
+ assertEquals(120, newEntry.getRunningSec());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 90d095b..9ce7f76 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -24,6 +24,7 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashSet;
import java.util.NavigableSet;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentMap;
@@ -36,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
public class BadQueryDetector extends Thread {
@@ -73,7 +75,9 @@ public class BadQueryDetector extends Thread {
private void initNotifiers() {
this.notifiers.add(new LoggerNotifier());
- this.notifiers.add(new PersistenceNotifier());
+ if (kylinConfig.getBadQueryPersistentEnabled()) {
+ this.notifiers.add(new PersistenceNotifier());
+ }
}
public void registerNotifier(Notifier notifier) {
@@ -116,7 +120,6 @@ public class BadQueryDetector extends Thread {
}
}
});
-
public PersistenceNotifier() {
try {
serverHostname = InetAddress.getLocalHost().getHostName();
@@ -132,11 +135,13 @@ public class BadQueryDetector extends Thread {
long cachingSeconds = (kylinConfig.getBadQueryDefaultAlertingSeconds() + 1) * 30;
Pair<Long, String> sqlPair = new Pair<>(startTime, sql);
if (!cacheQueue.contains(sqlPair)) {
- badQueryManager.addEntryToProject(sql, adj, startTime, runningSec, serverHostname, t.getName(), project);
+ badQueryManager.addEntryToProject(sql, startTime, adj, runningSec, serverHostname, t.getName(), project);
cacheQueue.add(sqlPair);
while (!cacheQueue.isEmpty() && (System.currentTimeMillis() - cacheQueue.first().getFirst() > cachingSeconds * 1000 || cacheQueue.size() > kylinConfig.getBadQueryHistoryNum() * 3)) {
cacheQueue.pollFirst();
}
+ } else {
+ badQueryManager.updateEntryToProject(sql, startTime, adj, runningSec, serverHostname, t.getName(), project);
}
} catch (IOException e) {
logger.error("Error in bad query persistence.", e);
http://git-wip-us.apache.org/repos/asf/kylin/blob/30aa60d6/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 020f9ca..649eb71 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -44,6 +44,7 @@ import org.apache.kylin.job.dao.ExecutableDao;
import org.apache.kylin.job.dao.ExecutablePO;
import org.apache.kylin.job.exception.PersistentException;
import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
@@ -98,6 +99,7 @@ public class CubeMetaExtractor extends AbstractApplication {
private CubeDescManager cubeDescManager;
private ExecutableDao executableDao;
private RealizationRegistry realizationRegistry;
+ private BadQueryHistoryManager badQueryHistoryManager;
boolean includeSegments;
boolean includeJobs;
@@ -158,17 +160,20 @@ public class CubeMetaExtractor extends AbstractApplication {
kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
executableDao = ExecutableDao.getInstance(kylinConfig);
realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
+ badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
if (optionsHelper.hasOption(OPTION_PROJECT)) {
- ProjectInstance projectInstance = projectManager.getProject(optionsHelper.getOptionValue(OPTION_PROJECT));
+ String projectName = optionsHelper.getOptionValue(OPTION_PROJECT);
+ ProjectInstance projectInstance = projectManager.getProject(projectName);
if (projectInstance == null) {
- throw new IllegalArgumentException("Project " + optionsHelper.getOptionValue(OPTION_PROJECT) + " does not exist");
+ throw new IllegalArgumentException("Project " + projectName + " does not exist");
}
- addRequired(ProjectInstance.concatResourcePath(projectInstance.getName()));
+ addRequired(projectInstance.getResourcePath());
List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
for (RealizationEntry realizationEntry : realizationEntries) {
retrieveResourcePath(getRealization(realizationEntry));
}
+ addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath());
} else if (optionsHelper.hasOption(OPTION_CUBE)) {
String cubeName = optionsHelper.getOptionValue(OPTION_CUBE);
IRealization realization;