You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/04/22 06:11:33 UTC
[04/50] [abbrv] kylin git commit: KYLIN-1546 Tool for dump some
information for diagnosis
KYLIN-1546 Tool for dump some information for diagnosis
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/539decfe
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/539decfe
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/539decfe
Branch: refs/heads/1.5.x-HBase1.1.3
Commit: 539decfe2ccaf6edb39673ce56e2ea4527c2ee56
Parents: b121b27
Author: lidongsjtu <li...@apache.org>
Authored: Thu Mar 31 13:56:16 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Thu Mar 31 14:01:06 2016 +0800
----------------------------------------------------------------------
.../apache/kylin/admin/CubeMetaExtractor.java | 340 -------------------
.../apache/kylin/admin/DataExtractorCLI.java | 142 --------
.../apache/kylin/admin/JobInfoExtractor.java | 178 ----------
.../apache/kylin/tool/CubeMetaExtractor.java | 340 +++++++++++++++++++
.../org/apache/kylin/tool/DiagnosisInfoCLI.java | 147 ++++++++
.../org/apache/kylin/tool/JobInfoExtractor.java | 178 ++++++++++
.../apache/kylin/common/util/ZipFileUtils.java | 13 +-
7 files changed, 675 insertions(+), 663 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/assembly/src/main/java/org/apache/kylin/admin/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/assembly/src/main/java/org/apache/kylin/admin/CubeMetaExtractor.java b/assembly/src/main/java/org/apache/kylin/admin/CubeMetaExtractor.java
deleted file mode 100644
index b31418b..0000000
--- a/assembly/src/main/java/org/apache/kylin/admin/CubeMetaExtractor.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.admin;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceTool;
-import org.apache.kylin.common.util.AbstractApplication;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.engine.streaming.StreamingConfig;
-import org.apache.kylin.engine.streaming.StreamingManager;
-import org.apache.kylin.invertedindex.IIInstance;
-import org.apache.kylin.job.dao.ExecutableDao;
-import org.apache.kylin.job.dao.ExecutablePO;
-import org.apache.kylin.job.exception.PersistentException;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.source.kafka.KafkaConfigManager;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-import org.apache.kylin.storage.hybrid.HybridInstance;
-import org.apache.kylin.storage.hybrid.HybridManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * extract cube related info for debugging/distributing purpose
- * TODO: deal with II case
- */
-public class CubeMetaExtractor extends AbstractApplication {
-
- private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class);
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube");
- @SuppressWarnings("static-access")
- private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid");
- @SuppressWarnings("static-access")
- private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments");
- @SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_JOB = OptionBuilder.withArgName("includeJobs").hasArg().isRequired(false).withDescription("set this to true if want to extract job info/outputs too. Default false").create("includeJobs");
- @SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_SEGMENT_DETAILS = OptionBuilder.withArgName("includeSegmentDetails").hasArg().isRequired(false).withDescription("set this to true if want to extract segment details too, such as dict, tablesnapshot. Default false").create("includeSegmentDetails");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(false).withDescription("specify the dest dir to save the related metadata").create("destDir");
-
- private Options options = null;
- private KylinConfig kylinConfig;
- private MetadataManager metadataManager;
- private ProjectManager projectManager;
- private HybridManager hybridManager;
- private CubeManager cubeManager;
- private StreamingManager streamingManager;
- private KafkaConfigManager kafkaConfigManager;
- private CubeDescManager cubeDescManager;
- private ExecutableDao executableDao;
- private RealizationRegistry realizationRegistry;
-
- boolean includeSegments;
- boolean includeJobs;
- boolean includeSegmentDetails;
-
- List<String> requiredResources = Lists.newArrayList();
- List<String> optionalResources = Lists.newArrayList();
- List<CubeInstance> cubesToTrimAndSave = Lists.newArrayList();//these cubes needs to be saved skipping segments
-
- public CubeMetaExtractor() {
- options = new Options();
-
- OptionGroup realizationOrProject = new OptionGroup();
- realizationOrProject.addOption(OPTION_CUBE);
- realizationOrProject.addOption(OPTION_PROJECT);
- realizationOrProject.addOption(OPTION_HYBRID);
- realizationOrProject.setRequired(true);
-
- options.addOptionGroup(realizationOrProject);
- options.addOption(OPTION_INCLUDE_SEGMENTS);
- options.addOption(OPTION_INCLUDE_JOB);
- options.addOption(OPTION_INCLUDE_SEGMENT_DETAILS);
- options.addOption(OPTION_DEST);
-
- }
-
- @Override
- protected Options getOptions() {
- return options;
- }
-
- @Override
- protected void execute(OptionsHelper optionsHelper) throws Exception {
- includeSegments = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENTS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENTS)) : true;
- includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false;
- includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false;
-
- String dest = null;
- if (optionsHelper.hasOption(OPTION_DEST)) {
- dest = optionsHelper.getOptionValue(OPTION_DEST);
- }
-
- if (StringUtils.isEmpty(dest)) {
- throw new RuntimeException("destDir is not set, exit directly without extracting");
- }
-
- if (!dest.endsWith("/")) {
- dest = dest + "/";
- }
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- metadataManager = MetadataManager.getInstance(kylinConfig);
- projectManager = ProjectManager.getInstance(kylinConfig);
- hybridManager = HybridManager.getInstance(kylinConfig);
- cubeManager = CubeManager.getInstance(kylinConfig);
- cubeDescManager = CubeDescManager.getInstance(kylinConfig);
- streamingManager = StreamingManager.getInstance(kylinConfig);
- kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
- executableDao = ExecutableDao.getInstance(kylinConfig);
- realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
-
- if (optionsHelper.hasOption(OPTION_PROJECT)) {
- ProjectInstance projectInstance = projectManager.getProject(optionsHelper.getOptionValue(OPTION_PROJECT));
- if (projectInstance == null) {
- throw new IllegalArgumentException("Project " + optionsHelper.getOptionValue(OPTION_PROJECT) + " does not exist");
- }
- addRequired(ProjectInstance.concatResourcePath(projectInstance.getName()));
- List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
- for (RealizationEntry realizationEntry : realizationEntries) {
- retrieveResourcePath(getRealization(realizationEntry));
- }
- } else if (optionsHelper.hasOption(OPTION_CUBE)) {
- String cubeName = optionsHelper.getOptionValue(OPTION_CUBE);
- IRealization realization;
-
- if ((realization = cubeManager.getRealization(cubeName)) != null) {
- retrieveResourcePath(realization);
- } else {
- throw new IllegalArgumentException("No cube found with name of " + cubeName);
- }
- } else if (optionsHelper.hasOption(OPTION_HYBRID)) {
- String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID);
- IRealization realization;
-
- if ((realization = hybridManager.getRealization(hybridName)) != null) {
- retrieveResourcePath(realization);
- } else {
- throw new IllegalArgumentException("No hybrid found with name of" + hybridName);
- }
- }
-
- executeExtraction(dest);
-
- logger.info("Extracted metadata files located at: " + new File(dest).getAbsolutePath());
- }
-
- private void executeExtraction(String dest) {
- logger.info("The resource paths going to be extracted:");
- for (String s : requiredResources) {
- logger.info(s + "(required)");
- }
- for (String s : optionalResources) {
- logger.info(s + "(optional)");
- }
- for (CubeInstance cube : cubesToTrimAndSave) {
- logger.info("Cube {} will be trimmed and extracted", cube);
- }
-
- try {
- ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest));
-
- for (String path : requiredResources) {
- ResourceTool.copyR(src, dst, path);
- }
-
- for (String path : optionalResources) {
- try {
- ResourceTool.copyR(src, dst, path);
- } catch (Exception e) {
- logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it.");
- }
- }
-
- for (CubeInstance cube : cubesToTrimAndSave) {
- CubeInstance trimmedCube = CubeInstance.getCopyOf(cube);
- trimmedCube.getSegments().clear();
- trimmedCube.setUuid(cube.getUuid());
- dst.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER);
- }
-
- } catch (IOException e) {
- throw new RuntimeException("IOException", e);
- }
- }
-
- private IRealization getRealization(RealizationEntry realizationEntry) {
- return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
- }
-
- private void dealWithStreaming(CubeInstance cube) {
- for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) {
- if (streamingConfig.getName() != null && streamingConfig.getName().equalsIgnoreCase(cube.getFactTable())) {
- addRequired(StreamingConfig.concatResourcePath(streamingConfig.getName()));
- addRequired(KafkaConfig.concatResourcePath(streamingConfig.getName()));
- }
- }
- }
-
- private void retrieveResourcePath(IRealization realization) {
-
- logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
-
- if (realization instanceof CubeInstance) {
- CubeInstance cube = (CubeInstance) realization;
- String descName = cube.getDescName();
- CubeDesc cubeDesc = cubeDescManager.getCubeDesc(descName);
- String modelName = cubeDesc.getModelName();
- DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName);
-
- dealWithStreaming(cube);
-
- for (String tableName : modelDesc.getAllTables()) {
- addRequired(TableDesc.concatResourcePath(tableName));
- addOptional(TableDesc.concatExdResourcePath(tableName));
- }
-
- addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
- addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
-
- if (includeSegments) {
- addRequired(CubeInstance.concatResourcePath(cube.getName()));
- for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
- if (includeSegmentDetails) {
- for (String dictPat : segment.getDictionaryPaths()) {
- addRequired(dictPat);
- }
- for (String snapshotPath : segment.getSnapshotPaths()) {
- addRequired(snapshotPath);
- }
- addRequired(segment.getStatisticsResourcePath());
- }
-
- if (includeJobs) {
- String lastJobId = segment.getLastBuildJobID();
- if (StringUtils.isEmpty(lastJobId)) {
- throw new RuntimeException("No job exist for segment :" + segment);
- } else {
- try {
- ExecutablePO executablePO = executableDao.getJob(lastJobId);
- addRequired(ExecutableDao.pathOfJob(lastJobId));
- addRequired(ExecutableDao.pathOfJobOutput(lastJobId));
- for (ExecutablePO task : executablePO.getTasks()) {
- addRequired(ExecutableDao.pathOfJob(task.getUuid()));
- addRequired(ExecutableDao.pathOfJobOutput(task.getUuid()));
- }
- } catch (PersistentException e) {
- throw new RuntimeException("PersistentException", e);
- }
- }
- }
- }
- } else {
- if (includeJobs) {
- logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
- }
-
- cubesToTrimAndSave.add(cube);
- }
- } else if (realization instanceof HybridInstance) {
- HybridInstance hybridInstance = (HybridInstance) realization;
- addRequired(HybridInstance.concatResourcePath(hybridInstance.getName()));
- for (IRealization iRealization : hybridInstance.getRealizations()) {
- if (iRealization.getType() != RealizationType.CUBE) {
- throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child " + iRealization.getName() + " with type " + iRealization.getType());
- }
- retrieveResourcePath(iRealization);
- }
- } else if (realization instanceof IIInstance) {
- throw new IllegalStateException("Does not support extract II instance or hybrid that contains II");
- } else {
- throw new IllegalStateException("Unknown realization type: " + realization.getType());
- }
- }
-
- private void addRequired(String record) {
- logger.info("adding required resource {}", record);
- requiredResources.add(record);
- }
-
- private void addOptional(String record) {
- logger.info("adding optional resource {}", record);
- optionalResources.add(record);
- }
-
- public static void main(String[] args) {
- CubeMetaExtractor extractor = new CubeMetaExtractor();
- extractor.execute(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/assembly/src/main/java/org/apache/kylin/admin/DataExtractorCLI.java
----------------------------------------------------------------------
diff --git a/assembly/src/main/java/org/apache/kylin/admin/DataExtractorCLI.java b/assembly/src/main/java/org/apache/kylin/admin/DataExtractorCLI.java
deleted file mode 100644
index 80328cd..0000000
--- a/assembly/src/main/java/org/apache/kylin/admin/DataExtractorCLI.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.admin;
-
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.AbstractApplication;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.common.util.ZipFileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-public class DataExtractorCLI extends AbstractApplication {
- private static final Logger logger = LoggerFactory.getLogger(DataExtractorCLI.class);
-
- private static final int DEFAULT_LOG_PERIOD = 3;
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_LOG_PERIOD = OptionBuilder.withArgName("logPeriod").hasArg().isRequired(false).withDescription("specify how many days of kylin logs to extract. Default 3.").create("logPeriod");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_COMPRESS = OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify whether to compress the output with zip. Default false.").create("compress");
-
- private CubeMetaExtractor cubeMetaExtractor;
- private JobInfoExtractor jobInfoExtractor;
- private Options options;
- private String type;
-
- public DataExtractorCLI(String type) {
- this.type = type;
-
- jobInfoExtractor = new JobInfoExtractor();
- cubeMetaExtractor = new CubeMetaExtractor();
-
- if (this.type.equalsIgnoreCase("job")) {
- options = jobInfoExtractor.getOptions();
- } else if (this.type.equalsIgnoreCase("metadata")) {
- options = cubeMetaExtractor.getOptions();
- } else {
- throw new RuntimeException("Only job and metadata are allowed.");
- }
-
- options.addOption(OPTION_LOG_PERIOD);
- options.addOption(OPTION_COMPRESS);
- }
-
- @Override
- protected Options getOptions() {
- return options;
- }
-
- @Override
- protected void execute(OptionsHelper optionsHelper) throws Exception {
- String dest = null;
- if (this.type.equals("job")) {
- jobInfoExtractor.execute(optionsHelper);
- dest = optionsHelper.getOptionValue(options.getOption("destDir"));
- } else if (this.type.equals("metadata")) {
- cubeMetaExtractor.execute(optionsHelper);
- dest = optionsHelper.getOptionValue(options.getOption("destDir"));
- }
-
- if (StringUtils.isEmpty(dest)) {
- throw new RuntimeException("destDir is not set, exit directly without extracting");
- }
- if (!dest.endsWith("/")) {
- dest = dest + "/";
- }
-
- int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD;
- boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : false;
-
- if (logPeriod > 0) {
- logger.info("Start to extract kylin logs in {} days", logPeriod);
-
- final String logFolder = KylinConfig.getKylinHome() + "/logs/";
- final String defaultLogFilename = "kylin.log";
- final File logsDir = new File(dest + "/logs/");
- final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
-
- FileUtils.forceMkdir(logsDir);
-
- final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod);
-
- logFileNames.add(defaultLogFilename);
- for (int i = 1; i < logPeriod; i++) {
- Calendar todayCal = Calendar.getInstance();
- todayCal.add(Calendar.DAY_OF_MONTH, 0 - i);
- logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime()));
- }
-
- for (String logFilename : logFileNames) {
- File logFile = new File(logFolder + logFilename);
- if (logFile.exists()) {
- FileUtils.copyFileToDirectory(logFile, logsDir);
- }
- }
- }
-
- if (compress) {
- File tempZipFile = File.createTempFile("extraction_", ".zip");
- ZipFileUtils.compressZipFile(dest, tempZipFile.getAbsolutePath());
- FileUtils.forceDelete(new File(dest));
- FileUtils.moveFileToDirectory(tempZipFile, new File(dest), true);
- }
-
- logger.info("Extraction finished at: " + new File(dest).getAbsolutePath());
- }
-
- public static void main(String args[]) {
- DataExtractorCLI dataExtractorCLI = new DataExtractorCLI(args[0]);
- dataExtractorCLI.execute(Arrays.copyOfRange(args, 1, args.length));
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/assembly/src/main/java/org/apache/kylin/admin/JobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/assembly/src/main/java/org/apache/kylin/admin/JobInfoExtractor.java b/assembly/src/main/java/org/apache/kylin/admin/JobInfoExtractor.java
deleted file mode 100644
index 8e83379..0000000
--- a/assembly/src/main/java/org/apache/kylin/admin/JobInfoExtractor.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.admin;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.common.persistence.ResourceTool;
-import org.apache.kylin.common.util.AbstractApplication;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.job.common.ShellExecutable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.dao.ExecutableDao;
-import org.apache.kylin.job.dao.ExecutablePO;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by dongli on 3/29/16.
- */
-public class JobInfoExtractor extends AbstractApplication {
- private static final Logger logger = LoggerFactory.getLogger(JobInfoExtractor.class);
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_JOB_ID = OptionBuilder.withArgName("jobId").hasArg().isRequired(true).withDescription("specify the Job ID to extract information. ").create("jobId");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify the dest dir to save the related information").create("destDir");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_CUBE = OptionBuilder.withArgName("includeCube").hasArg().isRequired(false).withDescription("set this to true if want to extract related cube info too. Default true").create("includeCube");
-
- @SuppressWarnings("static-access")
- private static final Option OPTION_INCLUDE_YARN_LOGS = OptionBuilder.withArgName("includeYarnLogs").hasArg().isRequired(false).withDescription("set this to true if want to extract related yarn logs too. Default true").create("includeYarnLogs");
-
- private Options options;
-
- private KylinConfig kylinConfig;
- private CubeMetaExtractor cubeMetaExtractor;
-
- private ExecutableDao executableDao;
-
- List<String> requiredResources = Lists.newArrayList();
- List<String> yarnLogsResources = Lists.newArrayList();
-
- public JobInfoExtractor() {
- cubeMetaExtractor = new CubeMetaExtractor();
-
- options = new Options();
- options.addOption(OPTION_JOB_ID);
- options.addOption(OPTION_DEST);
- options.addOption(OPTION_INCLUDE_CUBE);
- options.addOption(OPTION_INCLUDE_YARN_LOGS);
-
- kylinConfig = KylinConfig.getInstanceFromEnv();
- executableDao = ExecutableDao.getInstance(kylinConfig);
- }
-
- @Override
- protected Options getOptions() {
- return options;
- }
-
- @Override
- protected void execute(OptionsHelper optionsHelper) throws Exception {
- String jobId = optionsHelper.getOptionValue(OPTION_JOB_ID);
- String dest = optionsHelper.getOptionValue(OPTION_DEST);
- boolean includeCube = optionsHelper.hasOption(OPTION_INCLUDE_CUBE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CUBE)) : true;
- boolean includeYarnLogs = optionsHelper.hasOption(OPTION_INCLUDE_YARN_LOGS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_YARN_LOGS)) : true;
-
- if (StringUtils.isEmpty(dest)) {
- throw new RuntimeException("destDir is not set, exit directly without extracting");
- }
-
- if (!dest.endsWith("/")) {
- dest = dest + "/";
- }
-
- ExecutablePO executablePO = executableDao.getJob(jobId);
- addRequired(ExecutableDao.pathOfJob(jobId));
- addRequired(ExecutableDao.pathOfJobOutput(jobId));
- for (ExecutablePO task : executablePO.getTasks()) {
- addRequired(ExecutableDao.pathOfJob(task.getUuid()));
- addRequired(ExecutableDao.pathOfJobOutput(task.getUuid()));
- if (includeYarnLogs) {
- yarnLogsResources.add(task.getUuid());
- }
- }
- executeExtraction(dest);
-
- if (includeCube) {
- String cubeName = CubingExecutableUtil.getCubeName(executablePO.getParams());
- String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", dest + "cube_" + cubeName + "/", "-includeJobs", "false" };
- logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs));
- cubeMetaExtractor.execute(cubeMetaArgs);
- }
-
- if (includeYarnLogs) {
- logger.info("Start to related yarn job logs: " + jobId);
- for (String taskId : yarnLogsResources) {
- extractYarnLog(taskId, dest + "yarn_" + jobId + "/");
- }
- }
-
- logger.info("Extracted kylin jobs located at: " + new File(dest).getAbsolutePath());
- }
-
- private void executeExtraction(String dest) {
- logger.info("The resource paths going to be extracted:");
- for (String s : requiredResources) {
- logger.info(s + "(required)");
- }
-
- try {
- ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
- ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest));
-
- for (String path : requiredResources) {
- ResourceTool.copyR(src, dst, path);
- }
-
- } catch (IOException e) {
- throw new RuntimeException("IOException", e);
- }
- }
-
- private void extractYarnLog(String taskId, String dest) throws Exception {
- final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo();
- if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) {
- String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job", "application");
- File destFile = new File(dest + applicationId + ".log");
-
- ShellExecutable yarnExec = new ShellExecutable();
- yarnExec.setCmd("yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath());
- yarnExec.setName(yarnExec.getCmd());
-
- logger.info(yarnExec.getCmd());
- kylinConfig.getCliCommandExecutor().execute(yarnExec.getCmd(), null);
- }
- }
-
- private void addRequired(String record) {
- logger.info("adding required resource {}", record);
- requiredResources.add(record);
- }
-
- public static void main(String args[]) {
- JobInfoExtractor extractor = new JobInfoExtractor();
- extractor.execute(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
new file mode 100644
index 0000000..020f9ca
--- /dev/null
+++ b/assembly/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ResourceTool;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.engine.streaming.StreamingConfig;
+import org.apache.kylin.engine.streaming.StreamingManager;
+import org.apache.kylin.invertedindex.IIInstance;
+import org.apache.kylin.job.dao.ExecutableDao;
+import org.apache.kylin.job.dao.ExecutablePO;
+import org.apache.kylin.job.exception.PersistentException;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.SegmentStatusEnum;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.source.kafka.KafkaConfigManager;
+import org.apache.kylin.source.kafka.config.KafkaConfig;
+import org.apache.kylin.storage.hybrid.HybridInstance;
+import org.apache.kylin.storage.hybrid.HybridManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * extract cube related info for debugging/distributing purpose
+ * TODO: deal with II case
+ */
+public class CubeMetaExtractor extends AbstractApplication {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeMetaExtractor.class);
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_JOB = OptionBuilder.withArgName("includeJobs").hasArg().isRequired(false).withDescription("set this to true if want to extract job info/outputs too. Default false").create("includeJobs");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_SEGMENT_DETAILS = OptionBuilder.withArgName("includeSegmentDetails").hasArg().isRequired(false).withDescription("set this to true if want to extract segment details too, such as dict, tablesnapshot. Default false").create("includeSegmentDetails");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(false).withDescription("specify the dest dir to save the related metadata").create("destDir");
+
+ private Options options = null;
+ private KylinConfig kylinConfig;
+ private MetadataManager metadataManager;
+ private ProjectManager projectManager;
+ private HybridManager hybridManager;
+ private CubeManager cubeManager;
+ private StreamingManager streamingManager;
+ private KafkaConfigManager kafkaConfigManager;
+ private CubeDescManager cubeDescManager;
+ private ExecutableDao executableDao;
+ private RealizationRegistry realizationRegistry;
+
+ boolean includeSegments;
+ boolean includeJobs;
+ boolean includeSegmentDetails;
+
+ List<String> requiredResources = Lists.newArrayList();
+ List<String> optionalResources = Lists.newArrayList();
+ List<CubeInstance> cubesToTrimAndSave = Lists.newArrayList();//these cubes needs to be saved skipping segments
+
+ public CubeMetaExtractor() {
+ options = new Options();
+
+ OptionGroup realizationOrProject = new OptionGroup();
+ realizationOrProject.addOption(OPTION_CUBE);
+ realizationOrProject.addOption(OPTION_PROJECT);
+ realizationOrProject.addOption(OPTION_HYBRID);
+ realizationOrProject.setRequired(true);
+
+ options.addOptionGroup(realizationOrProject);
+ options.addOption(OPTION_INCLUDE_SEGMENTS);
+ options.addOption(OPTION_INCLUDE_JOB);
+ options.addOption(OPTION_INCLUDE_SEGMENT_DETAILS);
+ options.addOption(OPTION_DEST);
+
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ includeSegments = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENTS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENTS)) : true;
+ includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false;
+ includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false;
+
+ String dest = null;
+ if (optionsHelper.hasOption(OPTION_DEST)) {
+ dest = optionsHelper.getOptionValue(OPTION_DEST);
+ }
+
+ if (StringUtils.isEmpty(dest)) {
+ throw new RuntimeException("destDir is not set, exit directly without extracting");
+ }
+
+ if (!dest.endsWith("/")) {
+ dest = dest + "/";
+ }
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ metadataManager = MetadataManager.getInstance(kylinConfig);
+ projectManager = ProjectManager.getInstance(kylinConfig);
+ hybridManager = HybridManager.getInstance(kylinConfig);
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ cubeDescManager = CubeDescManager.getInstance(kylinConfig);
+ streamingManager = StreamingManager.getInstance(kylinConfig);
+ kafkaConfigManager = KafkaConfigManager.getInstance(kylinConfig);
+ executableDao = ExecutableDao.getInstance(kylinConfig);
+ realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
+
+ if (optionsHelper.hasOption(OPTION_PROJECT)) {
+ ProjectInstance projectInstance = projectManager.getProject(optionsHelper.getOptionValue(OPTION_PROJECT));
+ if (projectInstance == null) {
+ throw new IllegalArgumentException("Project " + optionsHelper.getOptionValue(OPTION_PROJECT) + " does not exist");
+ }
+ addRequired(ProjectInstance.concatResourcePath(projectInstance.getName()));
+ List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ retrieveResourcePath(getRealization(realizationEntry));
+ }
+ } else if (optionsHelper.hasOption(OPTION_CUBE)) {
+ String cubeName = optionsHelper.getOptionValue(OPTION_CUBE);
+ IRealization realization;
+
+ if ((realization = cubeManager.getRealization(cubeName)) != null) {
+ retrieveResourcePath(realization);
+ } else {
+ throw new IllegalArgumentException("No cube found with name of " + cubeName);
+ }
+ } else if (optionsHelper.hasOption(OPTION_HYBRID)) {
+ String hybridName = optionsHelper.getOptionValue(OPTION_HYBRID);
+ IRealization realization;
+
+ if ((realization = hybridManager.getRealization(hybridName)) != null) {
+ retrieveResourcePath(realization);
+ } else {
+ throw new IllegalArgumentException("No hybrid found with name of" + hybridName);
+ }
+ }
+
+ executeExtraction(dest);
+
+ logger.info("Extracted metadata files located at: " + new File(dest).getAbsolutePath());
+ }
+
+ private void executeExtraction(String dest) {
+ logger.info("The resource paths going to be extracted:");
+ for (String s : requiredResources) {
+ logger.info(s + "(required)");
+ }
+ for (String s : optionalResources) {
+ logger.info(s + "(optional)");
+ }
+ for (CubeInstance cube : cubesToTrimAndSave) {
+ logger.info("Cube {} will be trimmed and extracted", cube);
+ }
+
+ try {
+ ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest));
+
+ for (String path : requiredResources) {
+ ResourceTool.copyR(src, dst, path);
+ }
+
+ for (String path : optionalResources) {
+ try {
+ ResourceTool.copyR(src, dst, path);
+ } catch (Exception e) {
+ logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it.");
+ }
+ }
+
+ for (CubeInstance cube : cubesToTrimAndSave) {
+ CubeInstance trimmedCube = CubeInstance.getCopyOf(cube);
+ trimmedCube.getSegments().clear();
+ trimmedCube.setUuid(cube.getUuid());
+ dst.putResource(trimmedCube.getResourcePath(), trimmedCube, CubeManager.CUBE_SERIALIZER);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException("IOException", e);
+ }
+ }
+
+ private IRealization getRealization(RealizationEntry realizationEntry) {
+ return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
+ }
+
+ private void dealWithStreaming(CubeInstance cube) {
+ for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) {
+ if (streamingConfig.getName() != null && streamingConfig.getName().equalsIgnoreCase(cube.getFactTable())) {
+ addRequired(StreamingConfig.concatResourcePath(streamingConfig.getName()));
+ addRequired(KafkaConfig.concatResourcePath(streamingConfig.getName()));
+ }
+ }
+ }
+
+ private void retrieveResourcePath(IRealization realization) {
+
+ logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
+
+ if (realization instanceof CubeInstance) {
+ CubeInstance cube = (CubeInstance) realization;
+ String descName = cube.getDescName();
+ CubeDesc cubeDesc = cubeDescManager.getCubeDesc(descName);
+ String modelName = cubeDesc.getModelName();
+ DataModelDesc modelDesc = metadataManager.getDataModelDesc(modelName);
+
+ dealWithStreaming(cube);
+
+ for (String tableName : modelDesc.getAllTables()) {
+ addRequired(TableDesc.concatResourcePath(tableName));
+ addOptional(TableDesc.concatExdResourcePath(tableName));
+ }
+
+ addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+ addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
+
+ if (includeSegments) {
+ addRequired(CubeInstance.concatResourcePath(cube.getName()));
+ for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
+ if (includeSegmentDetails) {
+ for (String dictPat : segment.getDictionaryPaths()) {
+ addRequired(dictPat);
+ }
+ for (String snapshotPath : segment.getSnapshotPaths()) {
+ addRequired(snapshotPath);
+ }
+ addRequired(segment.getStatisticsResourcePath());
+ }
+
+ if (includeJobs) {
+ String lastJobId = segment.getLastBuildJobID();
+ if (StringUtils.isEmpty(lastJobId)) {
+ throw new RuntimeException("No job exist for segment :" + segment);
+ } else {
+ try {
+ ExecutablePO executablePO = executableDao.getJob(lastJobId);
+ addRequired(ExecutableDao.pathOfJob(lastJobId));
+ addRequired(ExecutableDao.pathOfJobOutput(lastJobId));
+ for (ExecutablePO task : executablePO.getTasks()) {
+ addRequired(ExecutableDao.pathOfJob(task.getUuid()));
+ addRequired(ExecutableDao.pathOfJobOutput(task.getUuid()));
+ }
+ } catch (PersistentException e) {
+ throw new RuntimeException("PersistentException", e);
+ }
+ }
+ }
+ }
+ } else {
+ if (includeJobs) {
+ logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
+ }
+
+ cubesToTrimAndSave.add(cube);
+ }
+ } else if (realization instanceof HybridInstance) {
+ HybridInstance hybridInstance = (HybridInstance) realization;
+ addRequired(HybridInstance.concatResourcePath(hybridInstance.getName()));
+ for (IRealization iRealization : hybridInstance.getRealizations()) {
+ if (iRealization.getType() != RealizationType.CUBE) {
+ throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child " + iRealization.getName() + " with type " + iRealization.getType());
+ }
+ retrieveResourcePath(iRealization);
+ }
+ } else if (realization instanceof IIInstance) {
+ throw new IllegalStateException("Does not support extract II instance or hybrid that contains II");
+ } else {
+ throw new IllegalStateException("Unknown realization type: " + realization.getType());
+ }
+ }
+
+ private void addRequired(String record) {
+ logger.info("adding required resource {}", record);
+ requiredResources.add(record);
+ }
+
+ private void addOptional(String record) {
+ logger.info("adding optional resource {}", record);
+ optionalResources.add(record);
+ }
+
+ public static void main(String[] args) {
+ CubeMetaExtractor extractor = new CubeMetaExtractor();
+ extractor.execute(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
----------------------------------------------------------------------
diff --git a/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
new file mode 100644
index 0000000..62379f2
--- /dev/null
+++ b/assembly/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.ZipFileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+public class DiagnosisInfoCLI extends AbstractApplication {
+ private static final Logger logger = LoggerFactory.getLogger(DiagnosisInfoCLI.class);
+
+ private static final int DEFAULT_LOG_PERIOD = 3;
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_LOG_PERIOD = OptionBuilder.withArgName("logPeriod").hasArg().isRequired(false).withDescription("specify how many days of kylin logs to extract. Default 3.").create("logPeriod");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_COMPRESS = OptionBuilder.withArgName("compress").hasArg().isRequired(false).withDescription("specify whether to compress the output with zip. Default false.").create("compress");
+
+ private CubeMetaExtractor cubeMetaExtractor;
+ private JobInfoExtractor jobInfoExtractor;
+ private Options options;
+ private String type;
+ private String exportDest;
+
+ public DiagnosisInfoCLI(String type) {
+ this.type = type;
+
+ jobInfoExtractor = new JobInfoExtractor();
+ cubeMetaExtractor = new CubeMetaExtractor();
+
+ if (this.type.equalsIgnoreCase("job")) {
+ options = jobInfoExtractor.getOptions();
+ } else if (this.type.equalsIgnoreCase("metadata")) {
+ options = cubeMetaExtractor.getOptions();
+ } else {
+ throw new RuntimeException("Only job and metadata are allowed.");
+ }
+
+ options.addOption(OPTION_LOG_PERIOD);
+ options.addOption(OPTION_COMPRESS);
+ }
+
+ public static void main(String args[]) {
+ DiagnosisInfoCLI diagnosisInfoCLI = new DiagnosisInfoCLI(args[0]);
+ diagnosisInfoCLI.execute(Arrays.copyOfRange(args, 1, args.length));
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+
+ if (this.type.equals("job")) {
+ jobInfoExtractor.execute(optionsHelper);
+ exportDest = optionsHelper.getOptionValue(options.getOption("destDir"));
+ } else if (this.type.equals("metadata")) {
+ cubeMetaExtractor.execute(optionsHelper);
+ exportDest = optionsHelper.getOptionValue(options.getOption("destDir"));
+ }
+
+ if (StringUtils.isEmpty(exportDest)) {
+ throw new RuntimeException("destDir is not set, exit directly without extracting");
+ }
+ if (!exportDest.endsWith("/")) {
+ exportDest = exportDest + "/";
+ }
+
+ int logPeriod = optionsHelper.hasOption(OPTION_LOG_PERIOD) ? Integer.valueOf(optionsHelper.getOptionValue(OPTION_LOG_PERIOD)) : DEFAULT_LOG_PERIOD;
+ boolean compress = optionsHelper.hasOption(OPTION_COMPRESS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_COMPRESS)) : false;
+
+ if (logPeriod > 0) {
+ logger.info("Start to extract kylin logs in {} days", logPeriod);
+
+ final String logFolder = KylinConfig.getKylinHome() + "/logs/";
+ final String defaultLogFilename = "kylin.log";
+ final File logsDir = new File(exportDest + "/logs/");
+ final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+
+ FileUtils.forceMkdir(logsDir);
+
+ final ArrayList<String> logFileNames = Lists.newArrayListWithCapacity(logPeriod);
+
+ logFileNames.add(defaultLogFilename);
+ for (int i = 1; i < logPeriod; i++) {
+ Calendar todayCal = Calendar.getInstance();
+ todayCal.add(Calendar.DAY_OF_MONTH, 0 - i);
+ logFileNames.add(defaultLogFilename + "." + format.format(todayCal.getTime()));
+ }
+
+ for (String logFilename : logFileNames) {
+ File logFile = new File(logFolder + logFilename);
+ if (logFile.exists()) {
+ FileUtils.copyFileToDirectory(logFile, logsDir);
+ }
+ }
+ }
+
+ if (compress) {
+ File tempZipFile = File.createTempFile("diagnosis_", ".zip");
+ ZipFileUtils.compressZipFile(exportDest, tempZipFile.getAbsolutePath());
+ FileUtils.forceDelete(new File(exportDest));
+ FileUtils.moveFileToDirectory(tempZipFile, new File(exportDest), true);
+ exportDest = exportDest + tempZipFile.getName();
+ }
+ logger.info("Diagnosis info locates at: " + new File(exportDest).getAbsolutePath());
+ }
+
+ public String getExportDest() {
+ return exportDest;
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
----------------------------------------------------------------------
diff --git a/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java b/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
new file mode 100644
index 0000000..43758e0
--- /dev/null
+++ b/assembly/src/main/java/org/apache/kylin/tool/JobInfoExtractor.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.common.persistence.ResourceTool;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
+import org.apache.kylin.job.common.ShellExecutable;
+import org.apache.kylin.job.constant.ExecutableConstants;
+import org.apache.kylin.job.dao.ExecutableDao;
+import org.apache.kylin.job.dao.ExecutablePO;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Created by dongli on 3/29/16.
+ */
+public class JobInfoExtractor extends AbstractApplication {
+ private static final Logger logger = LoggerFactory.getLogger(JobInfoExtractor.class);
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_JOB_ID = OptionBuilder.withArgName("jobId").hasArg().isRequired(true).withDescription("specify the Job ID to extract information. ").create("jobId");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_DEST = OptionBuilder.withArgName("destDir").hasArg().isRequired(true).withDescription("specify the dest dir to save the related information").create("destDir");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_CUBE = OptionBuilder.withArgName("includeCube").hasArg().isRequired(false).withDescription("set this to true if want to extract related cube info too. Default true").create("includeCube");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_INCLUDE_YARN_LOGS = OptionBuilder.withArgName("includeYarnLogs").hasArg().isRequired(false).withDescription("set this to true if want to extract related yarn logs too. Default true").create("includeYarnLogs");
+
+ private Options options;
+
+ private KylinConfig kylinConfig;
+ private CubeMetaExtractor cubeMetaExtractor;
+
+ private ExecutableDao executableDao;
+
+ List<String> requiredResources = Lists.newArrayList();
+ List<String> yarnLogsResources = Lists.newArrayList();
+
+ public JobInfoExtractor() {
+ cubeMetaExtractor = new CubeMetaExtractor();
+
+ options = new Options();
+ options.addOption(OPTION_JOB_ID);
+ options.addOption(OPTION_DEST);
+ options.addOption(OPTION_INCLUDE_CUBE);
+ options.addOption(OPTION_INCLUDE_YARN_LOGS);
+
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ executableDao = ExecutableDao.getInstance(kylinConfig);
+ }
+
+ @Override
+ protected Options getOptions() {
+ return options;
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ String jobId = optionsHelper.getOptionValue(OPTION_JOB_ID);
+ String dest = optionsHelper.getOptionValue(OPTION_DEST);
+ boolean includeCube = optionsHelper.hasOption(OPTION_INCLUDE_CUBE) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_CUBE)) : true;
+ boolean includeYarnLogs = optionsHelper.hasOption(OPTION_INCLUDE_YARN_LOGS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_YARN_LOGS)) : true;
+
+ if (StringUtils.isEmpty(dest)) {
+ throw new RuntimeException("destDir is not set, exit directly without extracting");
+ }
+
+ if (!dest.endsWith("/")) {
+ dest = dest + "/";
+ }
+
+ ExecutablePO executablePO = executableDao.getJob(jobId);
+ addRequired(ExecutableDao.pathOfJob(jobId));
+ addRequired(ExecutableDao.pathOfJobOutput(jobId));
+ for (ExecutablePO task : executablePO.getTasks()) {
+ addRequired(ExecutableDao.pathOfJob(task.getUuid()));
+ addRequired(ExecutableDao.pathOfJobOutput(task.getUuid()));
+ if (includeYarnLogs) {
+ yarnLogsResources.add(task.getUuid());
+ }
+ }
+ executeExtraction(dest);
+
+ if (includeCube) {
+ String cubeName = CubingExecutableUtil.getCubeName(executablePO.getParams());
+ String[] cubeMetaArgs = { "-cube", cubeName, "-destDir", dest + "cube_" + cubeName + "/", "-includeJobs", "false" };
+ logger.info("Start to extract related cube: " + StringUtils.join(cubeMetaArgs));
+ cubeMetaExtractor.execute(cubeMetaArgs);
+ }
+
+ if (includeYarnLogs) {
+ logger.info("Start to related yarn job logs: " + jobId);
+ for (String taskId : yarnLogsResources) {
+ extractYarnLog(taskId, dest + "yarn_" + jobId + "/");
+ }
+ }
+
+ logger.info("Extracted kylin jobs located at: " + new File(dest).getAbsolutePath());
+ }
+
+ private void executeExtraction(String dest) {
+ logger.info("The resource paths going to be extracted:");
+ for (String s : requiredResources) {
+ logger.info(s + "(required)");
+ }
+
+ try {
+ ResourceStore src = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ ResourceStore dst = ResourceStore.getStore(KylinConfig.createInstanceFromUri(dest));
+
+ for (String path : requiredResources) {
+ ResourceTool.copyR(src, dst, path);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException("IOException", e);
+ }
+ }
+
+ private void extractYarnLog(String taskId, String dest) throws Exception {
+ final Map<String, String> jobInfo = executableDao.getJobOutput(taskId).getInfo();
+ if (jobInfo.containsKey(ExecutableConstants.MR_JOB_ID)) {
+ String applicationId = jobInfo.get(ExecutableConstants.MR_JOB_ID).replace("job", "application");
+ File destFile = new File(dest + applicationId + ".log");
+
+ ShellExecutable yarnExec = new ShellExecutable();
+ yarnExec.setCmd("yarn logs -applicationId " + applicationId + " > " + destFile.getAbsolutePath());
+ yarnExec.setName(yarnExec.getCmd());
+
+ logger.info(yarnExec.getCmd());
+ kylinConfig.getCliCommandExecutor().execute(yarnExec.getCmd(), null);
+ }
+ }
+
+ private void addRequired(String record) {
+ logger.info("adding required resource {}", record);
+ requiredResources.add(record);
+ }
+
+ public static void main(String args[]) {
+ JobInfoExtractor extractor = new JobInfoExtractor();
+ extractor.execute(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/539decfe/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
index ef04bb1..b5ce829 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
@@ -34,16 +34,16 @@ public class ZipFileUtils {
throw new RuntimeException("Zipfile must end with .zip");
}
ZipOutputStream zipFile = new ZipOutputStream(new FileOutputStream(zipFilename));
- compressDirectoryToZipfile(sourceDir, sourceDir, zipFile);
+ compressDirectoryToZipfile(normDir(new File(sourceDir).getParent()), normDir(sourceDir), zipFile);
IOUtils.closeQuietly(zipFile);
}
private static void compressDirectoryToZipfile(String rootDir, String sourceDir, ZipOutputStream out) throws IOException {
for (File sourceFile : new File(sourceDir).listFiles()) {
if (sourceFile.isDirectory()) {
- compressDirectoryToZipfile(rootDir, sourceDir + sourceFile.getName() + File.separator, out);
+ compressDirectoryToZipfile(rootDir, sourceDir + normDir(sourceFile.getName()), out);
} else {
- ZipEntry entry = new ZipEntry(sourceDir.replace(rootDir, "") + sourceFile.getName());
+ ZipEntry entry = new ZipEntry(normDir(StringUtils.isEmpty(rootDir) ? sourceDir : sourceDir.replace(rootDir, "")) + sourceFile.getName());
out.putNextEntry(entry);
FileInputStream in = new FileInputStream(sourceDir + sourceFile.getName());
@@ -60,4 +60,11 @@ public class ZipFileUtils {
return false;
}
+
+ private static String normDir(String dirName) {
+ if (!StringUtils.isEmpty(dirName) && !dirName.endsWith(File.separator)) {
+ dirName = dirName + File.separator;
+ }
+ return dirName;
+ }
}