You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/21 10:22:32 UTC
[2/2] kylin git commit: Add HBase usage to Diagnosis tool
Add HBase usage to Diagnosis tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3598768a
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3598768a
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3598768a
Branch: refs/heads/yang-m1
Commit: 3598768af4b6b4e242ee7a90d5c3138a10ac1129
Parents: 638f7b3
Author: lidongsjtu <li...@apache.org>
Authored: Thu Apr 21 15:58:10 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Apr 21 16:22:20 2016 +0800
----------------------------------------------------------------------
tool/pom.xml | 12 ++
.../apache/kylin/tool/CubeMetaExtractor.java | 5 +-
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 17 ++
.../apache/kylin/tool/HBaseUsageExtractor.java | 207 +++++++++++++++++++
4 files changed, 239 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3598768a/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 2cc9a7e..e6ad667 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -61,6 +61,18 @@
<artifactId>kylin-invertedindex</artifactId>
<version>${project.parent.version}</version>
</dependency>
+
+ <!--Env-->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/kylin/blob/3598768a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 649eb71..af7504b 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -276,6 +276,7 @@ public class CubeMetaExtractor extends AbstractApplication {
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
+ addRequired(segment.getStatisticsResourcePath());
if (includeSegmentDetails) {
for (String dictPat : segment.getDictionaryPaths()) {
addRequired(dictPat);
@@ -322,9 +323,9 @@ public class CubeMetaExtractor extends AbstractApplication {
retrieveResourcePath(iRealization);
}
} else if (realization instanceof IIInstance) {
- throw new IllegalStateException("Does not support extract II instance or hybrid that contains II");
+ logger.warn("Does not support extract II instance or hybrid that contains II");
} else {
- throw new IllegalStateException("Unknown realization type: " + realization.getType());
+ logger.warn("Unknown realization type: " + realization.getType());
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3598768a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index adfd6a4..847f075 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -57,12 +57,17 @@ public class DiagnosisInfoCLI extends AbstractApplication {
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_CONF = OptionBuilder.withArgName("includeConf").hasArg().isRequired(false).withDescription("Specify whether to include conf files to extract. Default true.").create("includeConf");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_HBASE = OptionBuilder.withArgName("includeHBase").hasArg().isRequired(false).withDescription("Specify whether to include hbase files to extract. Default true.").create("includeHBase");
+
private CubeMetaExtractor cubeMetaExtractor;
+ private HBaseUsageExtractor hBaseUsageExtractor;
private Options options;
private String exportDest;
public DiagnosisInfoCLI() {
cubeMetaExtractor = new CubeMetaExtractor();
+ hBaseUsageExtractor = new HBaseUsageExtractor();
options = new Options();
options.addOption(OPTION_LOG_PERIOD);
@@ -94,6 +99,11 @@ public class DiagnosisInfoCLI extends AbstractApplication {
exportDest = exportDest + "/";
}
+ // create new folder to contain the output
+ if (new File(exportDest).exists()) {
+ exportDest = exportDest + "diagnosis_" + new SimpleDateFormat("YYYY_MM_dd_HH_mm_ss").format(new Date()) + "/";
+ }
+
// export cube metadata
String[] cubeMetaArgs = { "-destDir", exportDest + "metadata", "-project", project };
cubeMetaExtractor.execute(cubeMetaArgs);
@@ -101,6 +111,13 @@ public class DiagnosisInfoCLI extends AbstractApplication {
int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD;
boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : false;
boolean includeConf = optionsHelper.hasOption(OPTION_INCLUDE_CONF) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CONF)) : true;
+ boolean includeHBase = optionsHelper.hasOption(OPTION_INCLUDE_HBASE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_HBASE)) : true;
+
+ // export HBase
+ if (includeHBase) {
+ String[] hbaseArgs = { "-destDir", exportDest + "hbase", "-project", project };
+ hBaseUsageExtractor.execute(hbaseArgs);
+ }
// export logs
if (logPeriod > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/3598768a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
new file mode 100644
index 0000000..813a885
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/HBaseUsageExtractor.java
@@ -0,0 +1,207 @@
+package org.apache.kylin.tool;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class HBaseUsageExtractor extends AbstractApplication {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class);
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(false).withDescription("specify the dest dir to save the related metadata").create("destDir");
+
+ private List<String> htables = Lists.newArrayList();
+ private Configuration conf;
+ private CubeManager cubeManager;
+ private RealizationRegistry realizationRegistry;
+ private KylinConfig kylinConfig;
+ private ProjectManager projectManager;
+ private Options options = null;
+
+ public HBaseUsageExtractor() {
+ options = new Options();
+
+ OptionGroup realizationOrProject = new OptionGroup();
+ realizationOrProject.addOption(OPTION_CUBE);
+ realizationOrProject.addOption(OPTION_PROJECT);
+ realizationOrProject.setRequired(true);
+
+ options.addOptionGroup(realizationOrProject);
+ options.addOption(OPTION_DEST);
+
+ conf = HBaseConfiguration.create();
+ }
+
+ public static void main(String[] args) {
+ HBaseUsageExtractor extractor = new HBaseUsageExtractor();
+ extractor.execute(args);
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ private String getHBaseMasterUrl() {
+ String host = conf.get("hbase.master.info.bindAddress");
+ String port = conf.get("hbase.master.info.port");
+ return "http://" + host + ":" + port + "/";
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ String dest = null;
+ if (optionsHelper.hasOption(OPTION_DEST)) {
+ dest = optionsHelper.getOptionValue(OPTION_DEST);
+ }
+
+ if (org.apache.commons.lang3.StringUtils.isEmpty(dest)) {
+ throw new RuntimeException("destDir is not set, exit directly without extracting");
+ }
+
+ if (!dest.endsWith("/")) {
+ dest = dest + "/";
+ }
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
+ projectManager = ProjectManager.getInstance(kylinConfig);
+
+ if (optionsHelper.hasOption(OPTION_PROJECT)) {
+ String projectName = optionsHelper.getOptionValue(OPTION_PROJECT);
+ ProjectInstance projectInstance = projectManager.getProject(projectName);
+ if (projectInstance == null) {
+ throw new IllegalArgumentException("Project " + projectName + " does not exist");
+ }
+ List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ retrieveResourcePath(getRealization(realizationEntry));
+ }
+ } else if (optionsHelper.hasOption(OPTION_CUBE)) {
+ String cubeName = optionsHelper.getOptionValue(OPTION_CUBE);
+ IRealization realization;
+ if ((realization = cubeManager.getRealization(cubeName)) != null) {
+ retrieveResourcePath(realization);
+ } else {
+ throw new IllegalArgumentException("No cube found with name of " + cubeName);
+ }
+ }
+
+ extractCommonInfo(dest);
+ extractHTables(dest);
+
+ logger.info("Extracted metadata files located at: " + new File(dest).getAbsolutePath());
+ }
+
+ private void extractHTables(String dest) throws IOException {
+ logger.info("These htables are going to be extracted:");
+ for (String htable : htables) {
+ logger.info(htable + "(required)");
+ }
+
+ File tableDir = new File(dest, "table");
+ FileUtils.forceMkdir(tableDir);
+
+ for (String htable : htables) {
+ try {
+ URL srcUrl = new URL(getHBaseMasterUrl() + "table.jsp?name=" + htable);
+ File destFile = new File(tableDir, htable + ".html");
+ FileUtils.copyURLToFile(srcUrl, destFile);
+ } catch (Exception e) {
+ logger.warn("HTable " + htable + "info fetch failed: ", e);
+ }
+ }
+ }
+
+ private void extractCommonInfo(String dest) throws IOException {
+ logger.info("The hbase master info/conf are going to be extracted...");
+
+ // hbase master page
+ try {
+ File masterDir = new File(dest, "master");
+ FileUtils.forceMkdir(masterDir);
+ URL srcMasterUrl = new URL(getHBaseMasterUrl() + "master-status");
+ File masterDestFile = new File(masterDir, "master-status.html");
+ FileUtils.copyURLToFile(srcMasterUrl, masterDestFile);
+ } catch (Exception e) {
+ logger.warn("HBase Master status fetch failed: ", e);
+ }
+
+ // hbase conf
+ try {
+ File confDir = new File(dest, "conf");
+ FileUtils.forceMkdir(confDir);
+ URL srcConfUrl = new URL(getHBaseMasterUrl() + "conf");
+ File destConfFile = new File(confDir, "hbase-conf.xml");
+ FileUtils.copyURLToFile(srcConfUrl, destConfFile);
+ } catch (Exception e) {
+ logger.warn("HBase conf fetch failed: ", e);
+ }
+
+ // hbase hdfs status
+ try {
+ File hdfsDir = new File(dest, "hdfs");
+ FileUtils.forceMkdir(hdfsDir);
+ CliCommandExecutor cliCommandExecutor = kylinConfig.getCliCommandExecutor();
+ String output = cliCommandExecutor.execute("hadoop fs -ls -R " + conf.get("hbase.rootdir") + "/data/default/KYLIN_*").getSecond();
+ FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output);
+ output = cliCommandExecutor.execute("hadoop fs -ls -R " + conf.get("hbase.rootdir") + "/data/default/kylin_*").getSecond();
+ FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output, true);
+ } catch (Exception e) {
+ logger.warn("HBase hdfs status fetch failed: ", e);
+ }
+ }
+
+ private IRealization getRealization(RealizationEntry realizationEntry) {
+ return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
+ }
+
+ private void retrieveResourcePath(IRealization realization) {
+
+ logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
+
+ if (realization instanceof CubeInstance) {
+ CubeInstance cube = (CubeInstance) realization;
+ for (CubeSegment segment : cube.getSegments()) {
+ addHTable(segment.getStorageLocationIdentifier());
+ }
+ } else {
+ logger.warn("Unknown realization type: " + realization.getType());
+ }
+ }
+
+ private void addHTable(String record) {
+ logger.info("adding required resource {}", record);
+ htables.add(record);
+ }
+}