You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2016/08/19 10:21:02 UTC
kylin git commit: KYLIN-1964 add CubeMetaIngester
Repository: kylin
Updated Branches:
refs/heads/master a4ad980c1 -> 6b26a3100
KYLIN-1964 add CubeMetaIngester
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b26a310
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b26a310
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b26a310
Branch: refs/heads/master
Commit: 6b26a3100336c1affc172714d6003432438967e7
Parents: a4ad980
Author: Hongbin Ma <ma...@apache.org>
Authored: Fri Aug 19 18:19:06 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Fri Aug 19 18:19:13 2016 +0800
----------------------------------------------------------------------
build/bin/kylin.sh | 28 ++-
build/bin/metastore.sh | 5 +
.../apache/kylin/common/util/ZipFileUtils.java | 28 +++
.../apache/kylin/metadata/MetadataManager.java | 12 +-
.../apache/kylin/metadata/model/ColumnDesc.java | 15 +-
.../apache/kylin/metadata/model/TableDesc.java | 37 +++-
.../kylin/storage/hybrid/HybridManager.java | 5 +
.../table/DEFAULT.STREAMING_TABLE.json | 1 -
tool/pom.xml | 9 +
.../apache/kylin/tool/CubeMetaExtractor.java | 11 +-
.../org/apache/kylin/tool/CubeMetaIngester.java | 204 +++++++++++++++++++
.../apache/kylin/tool/CubeMetaIngesterTest.java | 126 ++++++++++++
tool/src/test/resources/benchmark_meta.zip | Bin 0 -> 9769 bytes
.../test/resources/cloned_cube_and_model.zip | Bin 0 -> 9709 bytes
tool/src/test/resources/cloned_cube_meta.zip | Bin 0 -> 9768 bytes
15 files changed, 443 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index e7a0061..88f9296 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -26,6 +26,12 @@ dir="$KYLIN_HOME/bin"
source ${dir}/check-env.sh
mkdir -p ${KYLIN_HOME}/logs
+
+mkdir -p ${KYLIN_HOME}/ext
+export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+
+
# start command
if [ "$1" == "start" ]
then
@@ -69,9 +75,8 @@ then
then source ${dir}/setenv.sh
fi
- export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:$HBASE_CLASSPATH_PREFIX
- mkdir -p ${KYLIN_HOME}/ext
- export HBASE_CLASSPATH=$hive_dependency:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH}:${KYLIN_HOME}/conf
+ #additionally add tomcat libs to HBASE_CLASSPATH_PREFIX
+ export HBASE_CLASSPATH_PREFIX=${tomcat_root}/bin/bootstrap.jar:${tomcat_root}/bin/tomcat-juli.jar:${tomcat_root}/lib/*:${HBASE_CLASSPATH_PREFIX}
if [ -z "$KYLIN_REST_ADDRESS" ]
then
@@ -83,7 +88,8 @@ then
fi
#debug if encounter NoClassDefError
- #hbase classpath
+ echo "hbase classpath is:"
+ hbase classpath
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
@@ -146,10 +152,7 @@ then
if [ -f "${dir}/setenv.sh" ]
then source ${dir}/setenv.sh
fi
-
- mkdir -p ${KYLIN_HOME}/ext
- export HBASE_CLASSPATH=$hive_dependency:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH}
-
+
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
-Dlog4j.configuration=kylin-log4j.properties\
@@ -195,9 +198,6 @@ then
then source ${dir}/setenv.sh
fi
- mkdir -p ${KYLIN_HOME}/ext
- export HBASE_CLASSPATH=$hive_dependency:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH}
-
# KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
hbase ${KYLIN_EXTRA_START_OPTS} \
-Dlog4j.configuration=kylin-log4j.properties\
@@ -209,14 +209,12 @@ then
elif [ "$1" = "version" ]
then
- export HBASE_CLASSPATH=${KYLIN_HOME}/lib/*
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.common.KylinVersion
exit 0
elif [ "$1" = "diag" ]
then
- shift
- exec $KYLIN_HOME/bin/diag.sh $@
+ echo "kylin.sh diag no longer supported, use diag.sh instead"
exit 0
# tool command
@@ -230,8 +228,6 @@ then
then source ${dir}/setenv-tool.sh
fi
- export HBASE_CLASSPATH=${KYLIN_HOME}/conf/:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:$hive_dependency:${HBASE_CLASSPATH}
-
exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=kylin-log4j.properties "$@"
else
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/build/bin/metastore.sh
----------------------------------------------------------------------
diff --git a/build/bin/metastore.sh b/build/bin/metastore.sh
index 1592043..5f34bff 100755
--- a/build/bin/metastore.sh
+++ b/build/bin/metastore.sh
@@ -25,6 +25,11 @@
dir=$(dirname ${0})
+
+# We should set KYLIN_HOME here for multiple tomcat instsances that are on the same node.
+# In addition, we should set a KYLIN_HOME for the global use as normal.
+export KYLIN_HOME=${dir}/../
+
source ${dir}/check-env.sh
if [ "$1" == "backup" ]
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
index d82a880..525a34c 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ZipFileUtils.java
@@ -23,12 +23,17 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
+import org.slf4j.LoggerFactory;
public class ZipFileUtils {
+
+ private static final org.slf4j.Logger logger = LoggerFactory.getLogger(ZipFileUtils.class);
+
public static void compressZipFile(String sourceDir, String zipFilename) throws IOException {
if (!validateZipFilename(zipFilename)) {
throw new RuntimeException("Zipfile must end with .zip");
@@ -38,6 +43,29 @@ public class ZipFileUtils {
IOUtils.closeQuietly(zipFile);
}
+ public static void decompressZipfileToDirectory(String zipFileName, File outputFolder) throws IOException {
+
+ ZipInputStream zipInputStream = new ZipInputStream(new FileInputStream(zipFileName));
+ ZipEntry zipEntry = null;
+ while ((zipEntry = zipInputStream.getNextEntry()) != null) {
+ logger.info("decompressing " + zipEntry.getName() + " is directory:" + zipEntry.isDirectory() + " available: " + zipInputStream.available());
+
+ File temp = new File(outputFolder, zipEntry.getName());
+ if (zipEntry.isDirectory()) {
+ temp.mkdirs();
+ } else {
+ temp.getParentFile().mkdirs();
+ temp.createNewFile();
+ temp.setLastModified(zipEntry.getTime());
+ FileOutputStream outputStream = new FileOutputStream(temp);
+ IOUtils.copy(zipInputStream, outputStream);
+ IOUtils.closeQuietly(outputStream);
+ }
+ }
+ IOUtils.closeQuietly(zipInputStream);
+
+ }
+
private static void compressDirectoryToZipfile(String rootDir, String sourceDir, ZipOutputStream out) throws IOException {
for (File sourceFile : new File(sourceDir).listFiles()) {
if (sourceFile.isDirectory()) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 7d45710..a74dd58 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -137,6 +137,10 @@ public class MetadataManager {
return ResourceStore.getStore(this.config);
}
+ public List<DataModelDesc> listDataModels() {
+ return Lists.newArrayList(this.dataModelDescMap.values());
+ }
+
public List<TableDesc> listAllTables() {
return Lists.newArrayList(srcTableMap.values());
}
@@ -160,17 +164,17 @@ public class MetadataManager {
int cut = tableDotColumnName.lastIndexOf('.');
if (cut < 0)
throw new IllegalArgumentException();
-
+
String tableName = tableDotColumnName.substring(0, cut);
String columnName = tableDotColumnName.substring(cut + 1);
-
+
TableDesc table = getTableDesc(tableName);
if (table == null)
return null;
-
+
return table.findColumnByName(columnName);
}
-
+
/**
* Get TableDesc by name
*/
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
index 73cad1f..257cb3b 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/ColumnDesc.java
@@ -173,21 +173,28 @@ public class ColumnDesc implements Serializable {
if (getClass() != obj.getClass())
return false;
ColumnDesc other = (ColumnDesc) obj;
+
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
- if (table == null) {
- if (other.table != null)
+
+ if (datatype == null) {
+ if (other.datatype != null)
return false;
- } else if (!table.equals(other.table))
+ } else if (!datatype.equals(other.datatype))
return false;
+
return true;
}
@Override
public String toString() {
- return "ColumnDesc [name=" + name + ",table=" + table.getIdentity() + "]";
+ return "ColumnDesc{" +
+ "id='" + id + '\'' +
+ ", name='" + name + '\'' +
+ ", datatype='" + datatype + '\'' +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
index f3d5318..e163d1d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/TableDesc.java
@@ -26,8 +26,8 @@ import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.StringSplitter;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
/**
* Table Metadata from Source. All name should be uppercase.
@@ -184,16 +184,37 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
return getIdentity().hashCode();
}
+ // @Override
+ // public boolean equals(Object obj) {
+ // if (this == obj)
+ // return true;
+ // if (!super.equals(obj))
+ // return false;
+ // if (getClass() != obj.getClass())
+ // return false;
+ // TableDesc other = (TableDesc) obj;
+ // return getIdentity().equals(other.getIdentity());
+ // }
+
@Override
- public boolean equals(Object obj) {
- if (this == obj)
+ public boolean equals(Object o) {
+ if (this == o)
return true;
- if (!super.equals(obj))
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ TableDesc tableDesc = (TableDesc) o;
+
+ if (sourceType != tableDesc.sourceType)
return false;
- if (getClass() != obj.getClass())
+ if (name != null ? !name.equals(tableDesc.name) : tableDesc.name != null)
return false;
- TableDesc other = (TableDesc) obj;
- return getIdentity().equals(other.getIdentity());
+ if (!Arrays.equals(columns, tableDesc.columns))
+ return false;
+ // if (tableType != null ? !tableType.equals(tableDesc.tableType) : tableDesc.tableType != null)
+ // return false;
+ return getIdentity().equals(tableDesc.getIdentity());
+
}
public String getMaterializedName() {
@@ -202,7 +223,7 @@ public class TableDesc extends RootPersistentEntity implements ISourceAware {
@Override
public String toString() {
- return "TableDesc [database=" + getDatabase() + " name=" + name + "]";
+ return "TableDesc{" + "name='" + name + '\'' + ", columns=" + Arrays.toString(columns) + ", sourceType=" + sourceType + ", tableType='" + tableType + '\'' + ", database=" + database + ", identity='" + getIdentity() + '\'' + '}';
}
/** create a mockup table for unit test */
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
index e43331e..d67ebc9 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridManager.java
@@ -18,6 +18,7 @@
package org.apache.kylin.storage.hybrid;
import java.io.IOException;
+import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@@ -148,6 +149,10 @@ public class HybridManager implements IRealizationProvider {
return getHybridInstance(name);
}
+ public Collection<HybridInstance> listHybridInstances() {
+ return hybridMap.values();
+ }
+
public HybridInstance getHybridInstance(String name) {
return hybridMap.get(name);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
index 64f359b..f28683f 100644
--- a/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
+++ b/examples/test_case_data/localmeta/table/DEFAULT.STREAMING_TABLE.json
@@ -39,7 +39,6 @@
}
],
"database": "DEFAULT",
- "source_type": 1,
"last_modified": 0,
"source_type" : 1
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 701fd13..a290d68 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -49,6 +49,15 @@
<artifactId>hbase-client</artifactId>
<scope>provided</scope>
</dependency>
+
+ <!-- Env & Test -->
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-core-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index e744549..5505c0a 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -62,7 +62,6 @@ import com.google.common.collect.Sets;
/**
* extract cube related info for debugging/distributing purpose
- * TODO: deal with II case
*/
public class CubeMetaExtractor extends AbstractInfoExtractor {
@@ -204,10 +203,12 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(requiredResources));
- try {
- ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(optionalResources));
- } catch (Exception e) {
- logger.warn("Exception when copying optional resource {}. May be caused by resource missing. Ignore it.");
+ for (String r : optionalResources) {
+ try {
+ ResourceTool.copy(srcConfig, dstConfig, Lists.newArrayList(r));
+ } catch (Exception e) {
+ logger.warn("Exception when copying optional resource {}. May be caused by resource missing. skip it.", r);
+ }
}
ResourceStore dstStore = ResourceStore.getStore(dstConfig);
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
new file mode 100644
index 0000000..7313d80
--- /dev/null
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceTool;
+import org.apache.kylin.common.util.AbstractApplication;
+import org.apache.kylin.common.util.OptionsHelper;
+import org.apache.kylin.common.util.ZipFileUtils;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.realization.RealizationRegistry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.apache.kylin.storage.hybrid.HybridManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+/**
+ * companion tool for CubeMetaExtractor, ingest the extracted cube meta into another metadata store
+ *
+ * TODO: only support ingest cube now
+ * TODO: ingest job history
+ */
+public class CubeMetaIngester extends AbstractApplication {
+
+ private static final Logger logger = LoggerFactory.getLogger(CubeMetaIngester.class);
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_SRC = OptionBuilder.withArgName("srcPath").hasArg().isRequired(true).withDescription("specify the path to the extracted cube metadata zip file").create("srcPath");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(true).withDescription("specify the target project for the new cubes").create("project");
+
+ @SuppressWarnings("static-access")
+ private static final Option OPTION_OVERWRITE_TABLES = OptionBuilder.withArgName("overwriteTables").hasArg().isRequired(false).withDescription("If table meta conflicts, overwrite the one in metadata store with the one in srcPath. Use in caution because it might break existing cubes! Suggest to backup metadata store first").create("overwriteTables");
+
+ private KylinConfig kylinConfig;
+ private MetadataManager metadataManager;
+ private ProjectManager projectManager;
+ private CubeManager cubeManager;
+ private CubeDescManager cubeDescManager;
+ private RealizationRegistry realizationRegistry;
+
+ Set<String> requiredResources = Sets.newLinkedHashSet();
+ private String targetProjectName;
+ private boolean overwriteTables = false;
+
+ @Override
+ protected Options getOptions() {
+ Options options = new Options();
+ options.addOption(OPTION_SRC);
+ options.addOption(OPTION_PROJECT);
+ options.addOption(OPTION_OVERWRITE_TABLES);
+ return options;
+ }
+
+ @Override
+ protected void execute(OptionsHelper optionsHelper) throws Exception {
+ kylinConfig = KylinConfig.getInstanceFromEnv();
+ metadataManager = MetadataManager.getInstance(kylinConfig);
+ projectManager = ProjectManager.getInstance(kylinConfig);
+ cubeManager = CubeManager.getInstance(kylinConfig);
+ cubeDescManager = CubeDescManager.getInstance(kylinConfig);
+ realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
+
+ if (optionsHelper.hasOption(OPTION_OVERWRITE_TABLES)) {
+ overwriteTables = Boolean.valueOf(optionsHelper.getOptionValue(OPTION_OVERWRITE_TABLES));
+ }
+ targetProjectName = optionsHelper.getOptionValue(OPTION_PROJECT);
+
+ String srcPath = optionsHelper.getOptionValue(OPTION_SRC);
+ if (!srcPath.endsWith(".zip")) {
+ throw new IllegalArgumentException(OPTION_SRC.getArgName() + " has to be a zip file");
+ }
+ File zipFile = new File(srcPath);
+ if (zipFile.isDirectory() || !zipFile.exists()) {
+ throw new IllegalArgumentException(OPTION_SRC.getArgName() + " file does does exist");
+ }
+
+ File tempFolder = File.createTempFile("_unzip", "folder");
+ tempFolder.deleteOnExit();
+ tempFolder.delete();
+ tempFolder.mkdir();
+ ZipFileUtils.decompressZipfileToDirectory(srcPath, tempFolder);
+ if (tempFolder.list().length != 1) {
+ throw new IllegalStateException(tempFolder.list().toString());
+ }
+
+ injest(tempFolder.listFiles()[0].getAbsoluteFile());
+ }
+
+ private void injest(File metaRoot) throws IOException {
+ KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
+ MetadataManager srcMetadataManager = MetadataManager.getInstance(srcConfig);
+ HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
+ CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
+ CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);
+
+ checkAndMark(srcMetadataManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
+ ResourceTool.copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));
+
+ for (TableDesc tableDesc : srcMetadataManager.listAllTables()) {
+ projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
+ }
+
+ for (CubeInstance cube : srcCubeManager.listAllCubes()) {
+ projectManager.updateModelToProject(cube.getDataModelDesc().getName(), targetProjectName);
+ projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
+ }
+
+ }
+
+ private void checkAndMark(MetadataManager srcMetadataManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
+ if (srcHybridManager.listHybridInstances().size() > 0) {
+ throw new IllegalStateException("Does not support ingest hybrid yet");
+ }
+
+ ProjectInstance targetProject = projectManager.getProject(targetProjectName);
+ if (targetProject == null) {
+ throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
+ }
+
+ for (TableDesc tableDesc : srcMetadataManager.listAllTables()) {
+ TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity());
+ if (existing != null && !existing.equals(tableDesc)) {
+ logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
+ logger.info("Existing version: " + existing);
+ logger.info("New version: " + tableDesc);
+
+ if (!overwriteTables) {
+ throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
+ } else {
+ logger.warn("Overwriting the old table desc: " + tableDesc.getIdentity());
+ }
+ }
+ requiredResources.add(TableDesc.concatResourcePath(tableDesc.getIdentity()));
+ }
+
+ for (DataModelDesc dataModelDesc : srcMetadataManager.listDataModels()) {
+ DataModelDesc existing = metadataManager.getDataModelDesc(dataModelDesc.getName());
+ if (existing != null) {
+ throw new IllegalStateException("Already exist a model called " + dataModelDesc.getName());
+ }
+ requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
+ }
+
+ for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
+ CubeDesc existing = cubeDescManager.getCubeDesc(cubeDesc.getName());
+ if (existing != null) {
+ throw new IllegalStateException("Already exist a cube desc called " + cubeDesc.getName());
+ }
+ requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
+ }
+
+ for (CubeInstance cube : srcCubeManager.listAllCubes()) {
+ CubeInstance existing = cubeManager.getCube(cube.getName());
+ if (existing != null) {
+ throw new IllegalStateException("Already exist a cube desc called " + cube.getName());
+ }
+ requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
+ }
+
+
+ }
+
+ public static void main(String[] args) {
+ CubeMetaIngester extractor = new CubeMetaIngester();
+ extractor.execute(args);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
----------------------------------------------------------------------
diff --git a/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
new file mode 100644
index 0000000..581221c
--- /dev/null
+++ b/tool/src/test/java/org/apache/kylin/tool/CubeMetaIngesterTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.tool;
+
+import java.util.Collections;
+
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectInstance;
+import org.apache.kylin.metadata.project.ProjectManager;
+import org.apache.kylin.metadata.project.RealizationEntry;
+import org.apache.kylin.metadata.realization.RealizationType;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.Description;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class CubeMetaIngesterTest extends LocalFileMetadataTestCase {
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testHappyIngest() {
+ String srcPath = Thread.currentThread().getContextClassLoader().getResource("cloned_cube_and_model.zip").getPath();
+ CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath });
+
+ ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default");
+ Assert.assertEquals(1, Collections.frequency(project.getTables(), "DEFAULT.TEST_KYLIN_FACT"));
+ Assert.assertTrue(project.getModels().contains("cloned_model"));
+ Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube")));
+
+ MetadataManager.clearCache();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("cloned_cube");
+ Assert.assertTrue(instance != null);
+ }
+
+ @Test
+ public void testHappyIngest2() {
+ String srcPath = Thread.currentThread().getContextClassLoader().getResource("benchmark_meta.zip").getPath();
+ CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath, "-overwriteTables", "true" });
+
+ ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default");
+ Assert.assertEquals(1, Collections.frequency(project.getTables(), "SSB.CUSTOMER"));
+ Assert.assertTrue(project.getModels().contains("benchmark_model"));
+ Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "benchmark_cube")));
+
+ MetadataManager.clearCache();
+ CubeDescManager.clearCache();
+ CubeManager.clearCache();
+ CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("benchmark_cube");
+ Assert.assertTrue(instance != null);
+ }
+
+ @Test
+ public void testBadIngest() {
+ thrown.expect(RuntimeException.class);
+
+ //should not break at table duplicate check, should fail at model duplicate check
+ thrown.expectCause(new BaseMatcher<Throwable>() {
+ @Override
+ public boolean matches(Object item) {
+ if (item instanceof IllegalStateException) {
+ if (((IllegalStateException) item).getMessage().equals("Already exist a model called test_kylin_inner_join_model_desc")) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void describeTo(Description description) {
+ }
+ });
+
+ String srcPath = this.getClass().getResource("/cloned_cube_meta.zip").getPath();
+ CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath });
+ }
+
+ @Test
+ public void testProjectNotExist() {
+
+ thrown.expect(RuntimeException.class);
+ thrown.expectCause(CoreMatchers.<IllegalStateException> instanceOf(IllegalStateException.class));
+
+ String srcPath = this.getClass().getResource("/cloned_cube_meta.zip").getPath();
+ CubeMetaIngester.main(new String[] { "-project", "Xdefault", "-srcPath", srcPath });
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/src/test/resources/benchmark_meta.zip
----------------------------------------------------------------------
diff --git a/tool/src/test/resources/benchmark_meta.zip b/tool/src/test/resources/benchmark_meta.zip
new file mode 100644
index 0000000..2e0ae51
Binary files /dev/null and b/tool/src/test/resources/benchmark_meta.zip differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/src/test/resources/cloned_cube_and_model.zip
----------------------------------------------------------------------
diff --git a/tool/src/test/resources/cloned_cube_and_model.zip b/tool/src/test/resources/cloned_cube_and_model.zip
new file mode 100644
index 0000000..b8cc35c
Binary files /dev/null and b/tool/src/test/resources/cloned_cube_and_model.zip differ
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b26a310/tool/src/test/resources/cloned_cube_meta.zip
----------------------------------------------------------------------
diff --git a/tool/src/test/resources/cloned_cube_meta.zip b/tool/src/test/resources/cloned_cube_meta.zip
new file mode 100644
index 0000000..8a749b9
Binary files /dev/null and b/tool/src/test/resources/cloned_cube_meta.zip differ