You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/22 06:11:40 UTC
[11/50] [abbrv] kylin git commit: KYLIN-1546 Add running seconds to
diagnosis tool
KYLIN-1546 Add running seconds to diagnosis tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/51871d87
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/51871d87
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/51871d87
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 51871d8706153bb49e2988529275f5c902d376c5
Parents: 9abd421
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 31 21:17:38 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 31 21:17:55 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/common/KylinConfigBase.java | 6 ++-
.../kylin/metadata/badquery/BadQueryEntry.java | 13 +++++-
.../badquery/BadQueryHistoryManager.java | 4 +-
.../badquery/BadQueryHistoryManagerTest.java | 4 +-
.../localmeta/bad_query/default.json | 6 ++-
.../kylin/rest/service/BadQueryDetector.java | 31 +++++++++++++-
.../kylin/rest/service/DiagnosisService.java | 2 +-
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 45 +++++++++-----------
8 files changed, 75 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 41ac188..e97051b 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -443,7 +443,11 @@ abstract public class KylinConfigBase implements Serializable {
}
public int getBadQueryDefaultAlertingSeconds() {
- return Integer.parseInt(getOptional("kylin.query.badquery.default.alerting.seconds", "90"));
+ return Integer.parseInt(getOptional("kylin.query.badquery.alerting.seconds", "90"));
+ }
+
+ public int getBadQueryDefaultDetectIntervalSeconds() {
+ return Integer.parseInt(getOptional("kylin.query.badquery.detect.interval.seconds", "60"));
}
public int getCachedDictMaxEntrySize() {
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
index 8a21e9c..b15846c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryEntry.java
@@ -33,16 +33,19 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
private String sql;
@JsonProperty("start_time")
private long startTime;
+ @JsonProperty("running_seconds")
+ private int runningSec;
@JsonProperty("server")
private String server;
@JsonProperty("thread")
private String thread;
- public BadQueryEntry(String sql, String adj, long startTime, String server, String thread) {
+ public BadQueryEntry(String sql, String adj, long startTime, int runningSec, String server, String thread) {
this.updateRandomUuid();
this.adj = adj;
this.sql = sql;
this.startTime = startTime;
+ this.runningSec = runningSec;
this.server = server;
this.thread = thread;
}
@@ -50,6 +53,14 @@ public class BadQueryEntry extends RootPersistentEntity implements Comparable<Ba
public BadQueryEntry() {
}
+ public int getRunningSec() {
+ return runningSec;
+ }
+
+ public void setRunningSec(int runningSec) {
+ this.runningSec = runningSec;
+ }
+
public String getAdj() {
return adj;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
index 3165c13..1945fed 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManager.java
@@ -100,8 +100,8 @@ public class BadQueryHistoryManager {
return badQueryHistory;
}
- public BadQueryHistory addEntryToProject(String sql, String adj, long startTime, String server, String threadName, String project) throws IOException {
- return addEntryToProject(new BadQueryEntry(sql, adj, startTime, server, threadName), project);
+ public BadQueryHistory addEntryToProject(String sql, String adj, long startTime, int runningSecs, String server, String threadName, String project) throws IOException {
+ return addEntryToProject(new BadQueryEntry(sql, adj, startTime, runningSecs, server, threadName), project);
}
public void removeBadQueryHistory(String project) throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
index f37908b..c684598 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/badquery/BadQueryHistoryManagerTest.java
@@ -63,7 +63,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
public void testAddEntryToProject() throws IOException {
KylinConfig kylinConfig = getTestConfig();
BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);
- BadQueryHistory history = manager.addEntryToProject("sql", "adj", 1459362239992L, "server", "t-0", "default");
+ BadQueryHistory history = manager.addEntryToProject("sql", "adj", 1459362239992L, 100, "server", "t-0", "default");
NavigableSet<BadQueryEntry> entries = history.getEntries();
assertEquals(3, entries.size());
@@ -77,7 +77,7 @@ public class BadQueryHistoryManagerTest extends LocalFileMetadataTestCase {
assertEquals("t-0", newEntry.getThread());
for (int i = 0; i < kylinConfig.getBadQueryHistoryNum(); i++) {
- history = manager.addEntryToProject("sql", "adj", 1459362239993L + i, "server", "t-0", "default");
+ history = manager.addEntryToProject("sql", "adj", 1459362239993L + i, 100 + i, "server", "t-0", "default");
}
assertEquals(kylinConfig.getBadQueryHistoryNum(), history.getEntries().size());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/examples/test_case_data/localmeta/bad_query/default.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/bad_query/default.json b/examples/test_case_data/localmeta/bad_query/default.json
index c7f26e1..077145c 100644
--- a/examples/test_case_data/localmeta/bad_query/default.json
+++ b/examples/test_case_data/localmeta/bad_query/default.json
@@ -9,7 +9,8 @@
"server": "sandbox.hortonworks.com",
"thread": "t-0",
"last_modified": 0,
- "start_time": 1459362236585
+ "start_time": 1459362236585,
+ "running_seconds": 100
},
{
"uuid": "da0c9cad-35c1-4f4b-8c10-669248842c2f",
@@ -18,7 +19,8 @@
"server": "sandbox.hortonworks.com",
"thread": "t-0",
"last_modified": 0,
- "start_time": 1459362239990
+ "start_time": 1459362239990,
+ "running_seconds": 100
}
],
"last_modified": 1459362294902
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
index 4e313ba..35f3928 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/BadQueryDetector.java
@@ -23,9 +23,13 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
+import java.util.NavigableSet;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.Pair;
import org.apache.kylin.metadata.badquery.BadQueryHistoryManager;
import org.apache.kylin.rest.request.SQLRequest;
import org.slf4j.Logger;
@@ -49,7 +53,7 @@ public class BadQueryDetector extends Thread {
super("BadQueryDetector");
this.setDaemon(true);
this.kylinConfig = KylinConfig.getInstanceFromEnv();
- this.detectionInterval = 60 * 1000;
+ this.detectionInterval = kylinConfig.getBadQueryDefaultDetectIntervalSeconds() * 1000;
this.alertMB = 100;
this.alertRunningSec = kylinConfig.getBadQueryDefaultAlertingSeconds();
@@ -100,6 +104,18 @@ public class BadQueryDetector extends Thread {
private class PersistenceNotifier implements Notifier {
BadQueryHistoryManager badQueryManager = BadQueryHistoryManager.getInstance(kylinConfig);
String serverHostname;
+ NavigableSet<Pair<Long, String>> cacheQueue = new TreeSet<>(new Comparator<Pair<Long, String>>() {
+ @Override
+ public int compare(Pair<Long, String> o1, Pair<Long, String> o2) {
+ if (o1.equals(o2)){
+ return 0;
+ } else if (o1.getFirst().equals(o2.getFirst())) {
+ return o2.getSecond().compareTo(o2.getSecond());
+ } else {
+ return (int)(o1.getFirst() - o2.getFirst());
+ }
+ }
+ });
public PersistenceNotifier() {
try {
@@ -113,7 +129,18 @@ public class BadQueryDetector extends Thread {
@Override
public void badQueryFound(String adj, int runningSec, long startTime, String project, String sql, Thread t) {
try {
- badQueryManager.addEntryToProject(sql, adj, startTime, serverHostname, t.getName(), project);
+ long cachingSeconds = (kylinConfig.getBadQueryDefaultAlertingSeconds() + 1) * 10;
+ Pair<Long, String> sqlPair = new Pair<>(startTime, sql);
+ if (!cacheQueue.contains(sqlPair)) {
+ badQueryManager.addEntryToProject(sql, adj, startTime, runningSec, serverHostname, t.getName(), project);
+ cacheQueue.add(sqlPair);
+ logger.info(Long.toString(System.currentTimeMillis() - cacheQueue.first().getFirst()));
+ logger.info(Long.toString(cachingSeconds * 1000));
+ while (!cacheQueue.isEmpty() && (System.currentTimeMillis() - cacheQueue.first().getFirst() > cachingSeconds * 1000 || cacheQueue.size() > kylinConfig.getBadQueryHistoryNum() * 3)) {
+ cacheQueue.pollFirst();
+ logger.info("Poll first");
+ }
+ }
} catch (IOException e) {
logger.error("Error in bad query persistence.", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
index e7d6cd1..83de689 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
@@ -38,7 +38,7 @@ public class DiagnosisService extends BasicService {
public String dumpDiagnosisInfo(String project) throws IOException {
String tempLocation = System.getProperty("java.io.tmpdir");
String[] args = { "-project", project, "-destDir", tempLocation, "-compress", "true" };
- DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI("metadata");
+ DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI();
diagnosisInfoCli.execute(args);
return diagnosisInfoCli.getExportDest();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51871d87/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 62379f2..84e1a3e 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -21,7 +21,6 @@ package org.apache.kylin.tool;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Calendar;
import org.apache.commons.cli.Option;
@@ -49,33 +48,29 @@ public class DiagnosisInfoCLI extends AbstractApplication {
@SuppressWarnings("static-access")
private static final Option OPTION_COMPRESS = OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify whether to compress the output with zip. Default false.").create("compress");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify the dest dir to save the related metadata").create("destDir");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
+
private CubeMetaExtractor cubeMetaExtractor;
- private JobInfoExtractor jobInfoExtractor;
private Options options;
- private String type;
private String exportDest;
- public DiagnosisInfoCLI(String type) {
- this.type = type;
-
- jobInfoExtractor = new JobInfoExtractor();
+ public DiagnosisInfoCLI() {
cubeMetaExtractor = new CubeMetaExtractor();
- if (this.type.equalsIgnoreCase("job")) {
- options = jobInfoExtractor.getOptions();
- } else if (this.type.equalsIgnoreCase("metadata")) {
- options = cubeMetaExtractor.getOptions();
- } else {
- throw new RuntimeException("Only job and metadata are allowed.");
- }
-
+ options = new Options();
options.addOption(OPTION_LOG_PERIOD);
options.addOption(OPTION_COMPRESS);
+ options.addOption(OPTION_DEST);
+ options.addOption(OPTION_PROJECT);
}
public static void main(String args[]) {
- DiagnosisInfoCLI diagnosisInfoCLI = new DiagnosisInfoCLI(args[0]);
- diagnosisInfoCLI.execute(Arrays.copyOfRange(args, 1, args.length));
+ DiagnosisInfoCLI diagnosisInfoCLI = new DiagnosisInfoCLI();
+ diagnosisInfoCLI.execute(args);
}
@Override
@@ -85,14 +80,8 @@ public class DiagnosisInfoCLI extends AbstractApplication {
@Override
protected void execute(OptionsHelper optionsHelper) throws Exception {
-
- if (this.type.equals("job")) {
- jobInfoExtractor.execute(optionsHelper);
- exportDest = optionsHelper.getOptionValue(options.getOption("destDir"));
- } else if (this.type.equals("metadata")) {
- cubeMetaExtractor.execute(optionsHelper);
- exportDest = optionsHelper.getOptionValue(options.getOption("destDir"));
- }
+ final String project = optionsHelper.getOptionValue(options.getOption("project"));
+ exportDest = optionsHelper.getOptionValue(options.getOption("destDir"));
if (StringUtils.isEmpty(exportDest)) {
throw new RuntimeException("destDir is not set, exit directly without extracting");
@@ -101,9 +90,14 @@ public class DiagnosisInfoCLI extends AbstractApplication {
exportDest = exportDest + "/";
}
+ // export cube metadata
+ String[] cubeMetaArgs = { "-destDir", exportDest + File.pathSeparator, "-project", project };
+ cubeMetaExtractor.execute(cubeMetaArgs);
+
int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD;
boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : false;
+ // export logs
if (logPeriod > 0) {
logger.info("Start to extract kylin logs in {} days", logPeriod);
@@ -131,6 +125,7 @@ public class DiagnosisInfoCLI extends AbstractApplication {
}
}
+ // compress to zip package
if (compress) {
File tempZipFile = File.createTempFile("diagnosis_", ".zip");
ZipFileUtils.compressZipFile(exportDest, tempZipFile.getAbsolutePath());