You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/25 09:26:44 UTC
[1/2] kylin git commit: KYLIN-1614 Add gc logs to diagnosis
Repository: kylin
Updated Branches:
refs/heads/master 85d9dff37 -> d899a6796
KYLIN-1614 Add gc logs to diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7e38e67b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7e38e67b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7e38e67b
Branch: refs/heads/master
Commit: 7e38e67b9b8138961421dc9b31c0650cdfee9831
Parents: 85d9dff
Author: lidongsjtu <li...@apache.org>
Authored: Mon Apr 25 12:11:22 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Apr 25 15:26:34 2016 +0800
----------------------------------------------------------------------
tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7e38e67b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index f41a728..89ade69 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -19,6 +19,7 @@
package org.apache.kylin.tool;
import java.io.File;
+import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
@@ -141,10 +142,12 @@ public class DiagnosisInfoCLI extends AbstractApplication {
final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod);
logFileNames.add(defaultLogFilename);
+ logFileNames.add("kylin.gc");
for (int i = 1; i < logPeriod; i++) {
Calendar todayCal = Calendar.getInstance();
todayCal.add(Calendar.DAY_OF_MONTH, 0 - i);
logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime()));
+ logFileNames.add("kylin.gc." + Integer.toString(i - 1));
}
for (String logFilename : logFileNames) {
[2/2] kylin git commit: KYLIN-1614 Correct hbase master
Posted by li...@apache.org.
KYLIN-1614 Correct hbase master
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d899a679
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d899a679
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d899a679
Branch: refs/heads/master
Commit: d899a67963af14f80fc047a9253ac7736c1983f3
Parents: 7e38e67
Author: lidongsjtu <li...@apache.org>
Authored: Mon Apr 25 15:21:01 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Apr 25 15:26:40 2016 +0800
----------------------------------------------------------------------
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 76 ++++++++++++--------
.../apache/kylin/tool/HBaseUsageExtractor.java | 23 +++++-
2 files changed, 68 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d899a679/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 89ade69..a316773 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -68,12 +68,14 @@ public class DiagnosisInfoCLI extends AbstractApplication {
private CubeMetaExtractor cubeMetaExtractor;
private HBaseUsageExtractor hBaseUsageExtractor;
+ private KylinConfig kylinConfig;
private Options options;
private String exportDest;
public DiagnosisInfoCLI() {
cubeMetaExtractor = new CubeMetaExtractor();
hBaseUsageExtractor = new HBaseUsageExtractor();
+ kylinConfig = KylinConfig.getInstanceFromEnv();
options = new Options();
options.addOption(OPTION_LOG_PERIOD);
@@ -128,36 +130,6 @@ public class DiagnosisInfoCLI extends AbstractApplication {
hBaseUsageExtractor.execute(hbaseArgs);
}
- // export logs
- if (logPeriod > 0) {
- logger.info("Start to extract kylin logs in {} days", logPeriod);
-
- final String logFolder = KylinConfig.getKylinHome() + "/logs/";
- final String defaultLogFilename = "kylin.log";
- final File logsDir = new File(exportDir, "logs");
- final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
-
- FileUtils.forceMkdir(logsDir);
-
- final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod);
-
- logFileNames.add(defaultLogFilename);
- logFileNames.add("kylin.gc");
- for (int i = 1; i < logPeriod; i++) {
- Calendar todayCal = Calendar.getInstance();
- todayCal.add(Calendar.DAY_OF_MONTH, 0 - i);
- logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime()));
- logFileNames.add("kylin.gc." + Integer.toString(i - 1));
- }
-
- for (String logFilename : logFileNames) {
- File logFile = new File(logFolder + logFilename);
- if (logFile.exists()) {
- FileUtils.copyFileToDirectory(logFile, logsDir);
- }
- }
- }
-
// export conf
if (includeConf) {
logger.info("Start to extract kylin conf files.");
@@ -198,6 +170,50 @@ public class DiagnosisInfoCLI extends AbstractApplication {
logger.warn("Error in export commit id.", e);
}
+ // export process info
+ try {
+ File basicDir = new File(exportDir, "basic");
+ FileUtils.forceMkdir(basicDir);
+ String output = kylinConfig.getCliCommandExecutor().execute("ps -ef|grep kylin").getSecond();
+ FileUtils.writeStringToFile(new File(basicDir, "process"), output);
+ output = kylinConfig.getCliCommandExecutor().execute("lsb_release -a").getSecond();
+ FileUtils.writeStringToFile(new File(basicDir, "lsb_release"), output);
+ } catch (Exception e) {
+ logger.warn("Error in export process info.", e);
+ }
+
+ // export logs
+ if (logPeriod > 0) {
+ logger.info("Start to extract kylin logs in {} days", logPeriod);
+
+ final String logFolder = KylinConfig.getKylinHome() + "/logs/";
+ final String defaultLogFilename = "kylin.log";
+ final String defaultGCLogFilename = "kylin.gc";
+ final File logsDir = new File(exportDir, "logs");
+ final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+
+ FileUtils.forceMkdir(logsDir);
+
+ final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod);
+
+ logFileNames.add(defaultLogFilename);
+ logFileNames.add(defaultGCLogFilename);
+ for (int i = 1; i < logPeriod; i++) {
+ Calendar todayCal = Calendar.getInstance();
+ todayCal.add(Calendar.DAY_OF_MONTH, 0 - i);
+ logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime()));
+ logFileNames.add(defaultGCLogFilename + "." + Integer.toString(i - 1));
+ }
+
+ for (String logFilename : logFileNames) {
+ File logFile = new File(logFolder + logFilename);
+ logger.info("Log file:" + logFile.getAbsolutePath());
+ if (logFile.exists()) {
+ FileUtils.copyFileToDirectory(logFile, logsDir);
+ }
+ }
+ }
+
// compress to zip package
if (compress) {
File tempZipFile = File.createTempFile("diagnosis_", ".zip");
http://git-wip-us.apache.org/repos/asf/kylin/blob/d899a679/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
index 811ba79..0c1bf47 100644
--- a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
@@ -30,6 +30,10 @@ import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.HConnection;
+import org.apache.hadoop.hbase.client.HConnectionManager;
+import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.CliCommandExecutor;
@@ -42,6 +46,8 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -89,8 +95,12 @@ public class HBaseUsageExtractor extends AbstractApplication {
return options;
}
- private String getHBaseMasterUrl() {
+ private String getHBaseMasterUrl() throws IOException, KeeperException {
String host = conf.get("hbase.master.info.bindAddress");
+ if (host.equals("0.0.0.0")) {
+ host = MasterAddressTracker.getMasterAddress(new ZooKeeperWatcher(conf, null, null)).getHostname();
+ }
+
String port = conf.get("hbase.master.info.port");
return "http://" + host + ":" + port + "/";
}
@@ -186,6 +196,17 @@ public class HBaseUsageExtractor extends AbstractApplication {
logger.warn("HBase conf fetch failed: ", e);
}
+ // hbase jmx
+ try {
+ File jmxDir = new File(dest, "jmx");
+ FileUtils.forceMkdir(jmxDir);
+ URL srcJmxUrl = new URL(getHBaseMasterUrl() + "jmx");
+ File jmxDestFile = new File(jmxDir, "jmx.html");
+ FileUtils.copyURLToFile(srcJmxUrl, jmxDestFile);
+ } catch (Exception e) {
+ logger.warn("HBase JMX fetch failed: ", e);
+ }
+
// hbase hdfs status
try {
File hdfsDir = new File(dest, "hdfs");