You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/09/21 09:15:02 UTC
[kylin] branch kylin-on-parquet-v2 updated: KYLIN-4712 Optimize
Optimize CubeMetaIngester.java CLI
This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push:
new cd03d6d KYLIN-4712 Optimize Optimize CubeMetaIngester.java CLI
cd03d6d is described below
commit cd03d6d41b6cec6756de1247fc9ae74dfe122685
Author: yaqian.zhang <59...@qq.com>
AuthorDate: Tue Sep 15 17:35:12 2020 +0800
KYLIN-4712 Optimize Optimize CubeMetaIngester.java CLI
---
.../java/org/apache/kylin/tool/CubeMetaIngester.java | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
index 40971c1..dfd6a84 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
@@ -33,6 +33,7 @@ import org.apache.kylin.common.persistence.ResourceTool;
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.ZipFileUtils;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
@@ -75,12 +76,15 @@ public class CubeMetaIngester extends AbstractApplication {
@SuppressWarnings("static-access")
private static final Option OPTION_OVERWRITE_TABLES = OptionBuilder.withArgName("overwriteTables").hasArg().isRequired(false).withDescription("If table meta conflicts, overwrite the one in metadata store with the one in srcPath. Use in caution because it might break existing cubes! Suggest to backup metadata store first").create("overwriteTables");
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_CREATE_PROJECT = OptionBuilder.withArgName("createProjectIfNotExists").hasArg().isRequired(false).withDescription("If project is not exists, kylin will create it.").create("createProjectIfNotExists");
private KylinConfig kylinConfig;
Set<String> requiredResources = Sets.newLinkedHashSet();
private String targetProjectName;
private boolean overwriteTables = false;
private boolean forceIngest = false;
+ private boolean createProjectIfNotExists = false;
@Override
protected Options getOptions() {
@@ -89,6 +93,7 @@ public class CubeMetaIngester extends AbstractApplication {
options.addOption(OPTION_PROJECT);
options.addOption(OPTION_FORCE_INGEST);
options.addOption(OPTION_OVERWRITE_TABLES);
+ options.addOption(OPTION_CREATE_PROJECT);
return options;
}
@@ -104,6 +109,10 @@ public class CubeMetaIngester extends AbstractApplication {
overwriteTables = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_OVERWRITE_TABLES));
}
+ if (optionsHelper.hasOption(OPTION_CREATE_PROJECT)) {
+ createProjectIfNotExists = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_CREATE_PROJECT));
+ }
+
targetProjectName = optionsHelper.getOptionValue(OPTION_PROJECT);
String srcPath = optionsHelper.getOptionValue(OPTION_SRC);
@@ -154,7 +163,7 @@ public class CubeMetaIngester extends AbstractApplication {
}
- private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
+ private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) throws IOException {
if (srcHybridManager.listHybridInstances().size() > 0) {
throw new IllegalStateException("Does not support ingest hybrid yet");
}
@@ -162,7 +171,11 @@ public class CubeMetaIngester extends AbstractApplication {
ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
ProjectInstance targetProject = projectManager.getProject(targetProjectName);
if (targetProject == null) {
- throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
+ if (createProjectIfNotExists) {
+ projectManager.createProject(targetProjectName, null, "This is a project automatically added when ingest cube", null);
+ } else {
+ throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
+ }
}
TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
@@ -179,6 +192,9 @@ public class CubeMetaIngester extends AbstractApplication {
logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
}
}
+ tableDesc.setUuid(RandomUtil.randomUUID().toString());
+ tableDesc.setLastModified(0);
+ metadataManager.saveSourceTable(tableDesc, targetProjectName);
requiredResources.add(tableDesc.getResourcePath());
}