You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/14 11:30:14 UTC
kylin git commit: minor, add extract all projects command
Repository: kylin
Updated Branches:
refs/heads/yang23 17a4bcfae -> 2dfbffa46
minor, add extract all projects command
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2dfbffa4
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2dfbffa4
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2dfbffa4
Branch: refs/heads/yang23
Commit: 2dfbffa46eeb27b4c6aec982116912a545ca55f8
Parents: 17a4bcf
Author: Roger Shi <ro...@hotmail.com>
Authored: Tue Mar 14 19:22:58 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Mar 14 19:27:00 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 45 +++++++++++++-------
1 file changed, 29 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2dfbffa4/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 35831f7..e370e48 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -23,8 +23,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Set;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
@@ -52,6 +50,7 @@ import org.apache.kylin.metadata.project.ProjectManager;
import org.apache.kylin.metadata.project.RealizationEntry;
import org.apache.kylin.metadata.realization.IRealization;
import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import org.apache.kylin.metadata.streaming.StreamingConfig;
import org.apache.kylin.metadata.streaming.StreamingManager;
@@ -61,8 +60,10 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -80,6 +81,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_HYBRID = OptionBuilder.withArgName("hybrid").hasArg().isRequired(false).withDescription("Specify which hybrid to extract").create("hybrid");
@SuppressWarnings("static-access")
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_All_PROJECT = OptionBuilder.withArgName("allProjects").hasArg(false).isRequired(false).withDescription("Specify realizations in all projects to extract").create("allProjects");
@SuppressWarnings("static-access")
private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageType");
@@ -130,6 +133,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
realizationOrProject.addOption(OPTION_CUBE);
realizationOrProject.addOption(OPTION_PROJECT);
realizationOrProject.addOption(OPTION_HYBRID);
+ realizationOrProject.addOption(OPTION_All_PROJECT);
realizationOrProject.setRequired(true);
options.addOptionGroup(realizationOrProject);
@@ -160,23 +164,15 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
- if (optionsHelper.hasOption(OPTION_PROJECT)) {
+ if (optionsHelper.hasOption(OPTION_All_PROJECT)) {
+ for (ProjectInstance projectInstance : projectManager.listAllProjects()) {
+ requireProject(projectInstance);
+ }
+ } else if (optionsHelper.hasOption(OPTION_PROJECT)) {
String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
for (String projectName : projectNames.split(",")) {
ProjectInstance projectInstance = projectManager.getProject(projectName);
- if (projectInstance == null) {
- throw new IllegalArgumentException("Project " + projectName + " does not exist");
- }
- addRequired(projectInstance.getResourcePath());
- List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
- for (RealizationEntry realizationEntry : realizationEntries) {
- retrieveResourcePath(getRealization(realizationEntry));
- }
- List<DataModelDesc> modelDescs = metadataManager.getModels(projectName);
- for (DataModelDesc modelDesc : modelDescs) {
- addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
- }
- addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath());
+ requireProject(projectInstance);
}
} else if (optionsHelper.hasOption(OPTION_CUBE)) {
String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
@@ -205,6 +201,22 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
engineOverwrite(new File(exportDir.getAbsolutePath()));
}
+ private void requireProject(ProjectInstance projectInstance) throws IOException {
+ if (projectInstance == null) {
+ throw new IllegalArgumentException("Project " + projectInstance.getName() + " does not exist");
+ }
+ addRequired(projectInstance.getResourcePath());
+ List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
+ for (RealizationEntry realizationEntry : realizationEntries) {
+ retrieveResourcePath(getRealization(realizationEntry));
+ }
+ List<DataModelDesc> modelDescs = metadataManager.getModels(projectInstance.getName());
+ for (DataModelDesc modelDesc : modelDescs) {
+ addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+ }
+ addOptional(badQueryHistoryManager.getBadQueriesForProject(projectInstance.getName()).getResourcePath());
+ }
+
private void executeExtraction(String dest) {
logger.info("The resource paths going to be extracted:");
for (String s : requiredResources) {
@@ -359,6 +371,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
+ cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
} else if (realization instanceof HybridInstance) {