You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/07/04 01:46:43 UTC

kylin git commit: minor, remove dependency on kylin-tool

Repository: kylin
Updated Branches:
  refs/heads/master 561cd0b98 -> c7a8b389f


minor, remove dependency on kylin-tool


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c7a8b389
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c7a8b389
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c7a8b389

Branch: refs/heads/master
Commit: c7a8b389fbd94ba7f11d5a011ff25691c197c57b
Parents: 561cd0b
Author: lidongsjtu <li...@apache.org>
Authored: Mon Jul 4 09:46:10 2016 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Mon Jul 4 09:46:10 2016 +0800

----------------------------------------------------------------------
 assembly/pom.xml                                |  5 --
 build/bin/diag.sh                               | 74 ++++++++++++++++++++
 build/bin/kylin.sh                              | 23 +-----
 build/deploy/diag.sh                            | 74 --------------------
 build/script/compress.sh                        |  2 +-
 build/script/package_diag.sh                    |  6 +-
 build/script/prepare_libs.sh                    |  9 ++-
 server-base/pom.xml                             |  5 --
 .../kylin/rest/service/DiagnosisService.java    | 58 ++++++++++-----
 tool/pom.xml                                    |  6 +-
 .../kylin/tool/AbstractInfoExtractor.java       |  2 +-
 .../apache/kylin/tool/CubeMetaExtractor.java    | 13 ++--
 .../kylin/tool/util/ResourceStoreUtil.java      |  4 +-
 13 files changed, 144 insertions(+), 137 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index d7764a0..3904225 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -55,11 +55,6 @@
             <artifactId>kylin-engine-streaming</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-tool</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
 
         <dependency>
             <groupId>org.apache.kylin</groupId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/build/bin/diag.sh
----------------------------------------------------------------------
diff --git a/build/bin/diag.sh b/build/bin/diag.sh
new file mode 100644
index 0000000..2ddfa80
--- /dev/null
+++ b/build/bin/diag.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+
+# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node.
+# In addition, we should set a KYLIN_HOME for the global use as normal.
+export KYLIN_HOME=${dir}/../
+source ${dir}/check-env.sh
+
+if [ $# -eq 1 ] || [ $# -eq 2 ]
+then
+    patient="$1"
+    if [ -z "$patient" ]
+    then
+        echo "You need to specify a Project or Job Id for diagnosis."
+        exit 1
+    fi
+    destDir="$2"
+    if [ -z "$destDir" ]
+    then
+        destDir="$KYLIN_HOME/diagnosis_dump/"
+        mkdir -p $destDir
+    fi
+
+    #retrive $hive_dependency and $hbase_dependency
+    source ${dir}/find-hive-dependency.sh
+    source ${dir}/find-hbase-dependency.sh
+
+    #retrive $KYLIN_EXTRA_START_OPTS
+    if [ -f "${dir}/setenv-tool.sh" ]
+        then source ${dir}/setenv-tool.sh
+    fi
+    export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
+
+    diagJar=`ls ${KYLIN_HOME}/tool/kylin-tool-*.jar`
+    if [ -f "${diagJar}" ]; then
+        if [ -f "${KYLIN_HOME}/commit_SHA1" ]; then
+            export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${diagJar}:${KYLIN_HOME}/lib/*
+        else
+            export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${KYLIN_HOME}/lib/*:${diagJar}
+        fi
+    else
+        echo "missing diagnosis jar file."
+        exit 1
+    fi
+
+    if [ ${#patient} -eq 36 ]; then
+        exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir
+    else
+        exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project -all -destDir $destDir
+    fi
+
+    exit 0
+else
+    echo "usage: diag.sh Project|JobId [target_path]"
+    exit 1
+fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index f3710c3..bcdeb6b 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -215,27 +215,8 @@ then
 
 elif [ "$1" = "diag" ]
 then
-    patient="$2"
-    if [ -z "$patient" ]
-    then
-        echo "You need to specify a Project or Job Id for diagnosis."
-        exit 1
-    fi
-
-    destDir="$3"
-    if [ -z "$destDir" ]
-    then
-        destDir="$KYLIN_HOME/diagnosis_dump/"
-        mkdir -p $destDir
-    fi
-
-    export HBASE_CLASSPATH=${KYLIN_HOME}/lib/*
-
-    if [ ${#patient} -eq 36 ]; then
-        exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir
-    else
-        exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir
-    fi
+    shift
+    exec $KYLIN_HOME/bin/diag.sh $@
     exit 0
 
 # tool command

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/build/deploy/diag.sh
----------------------------------------------------------------------
diff --git a/build/deploy/diag.sh b/build/deploy/diag.sh
deleted file mode 100644
index fdd6e6b..0000000
--- a/build/deploy/diag.sh
+++ /dev/null
@@ -1,74 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-
-# We should set KYLIN_HOME here for multiple tomcat instances that are on the same node.
-# In addition, we should set a KYLIN_HOME for the global use as normal.
-export KYLIN_HOME=${dir}/../
-source ${dir}/check-env.sh
-
-if [ $# -eq 1 ] || [ $# -eq 2 ]
-then
-    patient="$1"
-    if [ -z "$patient" ]
-    then
-        echo "You need to specify a Project or Job Id for diagnosis."
-        exit 1
-    fi
-    destDir="$2"
-    if [ -z "$destDir" ]
-    then
-        destDir="$KYLIN_HOME/diagnosis_dump/"
-        mkdir -p $destDir
-    fi
-
-    #retrive $hive_dependency and $hbase_dependency
-    source ${dir}/find-hive-dependency.sh
-    source ${dir}/find-hbase-dependency.sh
-
-    #retrive $KYLIN_EXTRA_START_OPTS
-    if [ -f "${dir}/setenv-tool.sh" ]
-        then source ${dir}/setenv-tool.sh
-    fi
-    export HBASE_CLASSPATH=$hive_dependency:${HBASE_CLASSPATH}
-
-    diagJar=`ls ${KYLIN_HOME}/tool/kylin-diagnosis-*.jar`
-    if [ -f "${diagJar}" ]; then
-        if [ -f "${KYLIN_HOME}/commit_SHA1" ]; then
-            export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${diagJar}:${KYLIN_HOME}/lib/*
-        else
-            export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${KYLIN_HOME}/lib/*:${diagJar}
-        fi
-    else
-        echo "missing diagnosis jar file."
-        exit 1
-    fi
-
-    if [ ${#patient} -eq 36 ]; then
-        exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.JobDiagnosisInfoCLI -jobId $patient -destDir $destDir
-    else
-        exec hbase ${KYLIN_EXTRA_START_OPTS} -Dlog4j.configuration=kylin-log4j.properties org.apache.kylin.tool.DiagnosisInfoCLI -project $patient -destDir $destDir
-    fi
-
-    exit 0
-else
-    echo "usage: diag.sh Project|JobId [target_path]"
-    exit 1
-fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/build/script/compress.sh
----------------------------------------------------------------------
diff --git a/build/script/compress.sh b/build/script/compress.sh
index 503fdf1..4e3592e 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -34,7 +34,7 @@ package_name=apache-kylin-${version}-bin
 cd build/
 rm -rf ${package_name}
 mkdir ${package_name}
-cp -r lib bin conf tomcat ../examples/sample_cube commit_SHA1 ${package_name}
+cp -r lib tool bin conf tomcat ../examples/sample_cube commit_SHA1 ${package_name}
 rm -rf lib tomcat commit_SHA1
 find ${package_name} -type d -exec chmod 755 {} \;
 find ${package_name} -type f -exec chmod 644 {} \;

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/build/script/package_diag.sh
----------------------------------------------------------------------
diff --git a/build/script/package_diag.sh b/build/script/package_diag.sh
index 28d5ee0..d0542ab 100755
--- a/build/script/package_diag.sh
+++ b/build/script/package_diag.sh
@@ -54,9 +54,9 @@ mvn clean install -DskipTests	 || { exit 1; }
 echo "copy libraries"
 rm -rf build/tool
 mkdir build/tool
-cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-diagnosis-${version}.jar
+cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
 # Copied file becomes 000 for some env (e.g. my Cygwin)
-chmod 644 build/tool/kylin-diagnosis-${version}.jar
+chmod 644 build/tool/kylin-tool-${version}.jar
 
 echo 'package tar.gz'
 package_name=apache-kylin-${version}-diag
@@ -65,7 +65,7 @@ rm -rf ${package_name}
 mkdir ${package_name}
 mkdir ${package_name}/bin
 cp -r tool ${package_name}
-cp deploy/diag.sh ${package_name}/bin/
+cp bin/diag.sh ${package_name}/bin/
 rm -rf tool
 find ${package_name} -type d -exec chmod 755 {} \;
 find ${package_name} -type f -exec chmod 644 {} \;

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/build/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh
index ddcfb86..aec9923 100755
--- a/build/script/prepare_libs.sh
+++ b/build/script/prepare_libs.sh
@@ -28,12 +28,15 @@ fi
 echo "version ${version}"
 
 echo "copy lib file"
-rm -rf build/lib
-mkdir build/lib
+rm -rf build/lib build/tool
+mkdir build/lib build/tool
 cp assembly/target/kylin-assembly-${version}-job.jar build/lib/kylin-job-${version}.jar
 cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar build/lib/kylin-coprocessor-${version}.jar
 cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar
+cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
+
 # Copied file becomes 000 for some env (e.g. my Cygwin)
 chmod 644 build/lib/kylin-job-${version}.jar
 chmod 644 build/lib/kylin-coprocessor-${version}.jar
-chmod 644 build/lib/kylin-jdbc-${version}.jar
\ No newline at end of file
+chmod 644 build/lib/kylin-jdbc-${version}.jar
+chmod 644 build/tool/kylin-tool-${version}.jar
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/server-base/pom.xml
----------------------------------------------------------------------
diff --git a/server-base/pom.xml b/server-base/pom.xml
index ad54ecf..e6a4cc5 100644
--- a/server-base/pom.xml
+++ b/server-base/pom.xml
@@ -65,11 +65,6 @@
             <artifactId>kylin-source-kafka</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-tool</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
 
         <dependency>
             <groupId>net.sf.ehcache</groupId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
index ba51ea1..9f2f261 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/DiagnosisService.java
@@ -22,10 +22,12 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.CliCommandExecutor;
+import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.metadata.badquery.BadQueryHistory;
 import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.tool.DiagnosisInfoCLI;
-import org.apache.kylin.tool.JobDiagnosisInfoCLI;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.security.access.prepost.PreAuthorize;
@@ -42,6 +44,19 @@ public class DiagnosisService extends BasicService {
         return Files.createTempDir();
     }
 
+    private String getDiagnosisPackageName(File destDir) {
+        for (File subDir : destDir.listFiles()) {
+            if (subDir.isDirectory()) {
+                for (File file : subDir.listFiles()) {
+                    if (file.getName().endsWith(".zip")) {
+                        return file.getAbsolutePath();
+                    }
+                }
+            }
+        }
+        throw new RuntimeException("Diagnosis package not found in directory: " + destDir.getAbsolutePath());
+    }
+
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
     public BadQueryHistory getProjectBadQueryHistory(String project) throws IOException {
         return getBadQueryHistoryManager().getBadQueriesForProject(project);
@@ -49,26 +64,37 @@ public class DiagnosisService extends BasicService {
 
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
     public String dumpProjectDiagnosisInfo(String project) throws IOException {
-        String[] args = { "-project", "-all", "-destDir", getDumpDir().getAbsolutePath() };
-        logger.info("DiagnosisInfoCLI args: " + Arrays.toString(args));
-        DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI();
-        diagnosisInfoCli.execute(args);
-        return diagnosisInfoCli.getExportDest();
+        File exportPath = getDumpDir();
+        String[] args = { project, exportPath.getAbsolutePath() };
+        runDiagnosisCLI(args);
+        return getDiagnosisPackageName(exportPath);
     }
 
     @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
     public String dumpJobDiagnosisInfo(String jobId) throws IOException {
-        String[] args = { "-jobId", jobId, "-destDir", getDumpDir().getAbsolutePath() };
-        logger.info("JobDiagnosisInfoCLI args: " + Arrays.toString(args));
-        JobDiagnosisInfoCLI jobInfoExtractor = new JobDiagnosisInfoCLI();
-        jobInfoExtractor.execute(args);
-        return jobInfoExtractor.getExportDest();
+        File exportPath = getDumpDir();
+        String[] args = { jobId, exportPath.getAbsolutePath() };
+        runDiagnosisCLI(args);
+        return getDiagnosisPackageName(exportPath);
     }
 
-    public static void main(String[] args1) {
-        String[] args = { "-project", "-all", "-destDir", Files.createTempDir().getAbsolutePath() };
+    private void runDiagnosisCLI(String[] args) throws IOException {
+        File cwd = new File("");
+        logger.info("Current path: " + cwd.getAbsolutePath());
+
         logger.info("DiagnosisInfoCLI args: " + Arrays.toString(args));
-        DiagnosisInfoCLI diagnosisInfoCli = new DiagnosisInfoCLI();
-        diagnosisInfoCli.execute(args);
+        File script = new File(KylinConfig.getKylinHome() + File.separator + "bin", "diag.sh");
+        if (!script.exists()) {
+            throw new RuntimeException("diag.sh not found at " + script.getAbsolutePath());
+        }
+
+        String diagCmd = script.getAbsolutePath() + " " + StringUtils.join(args, " ");
+        CliCommandExecutor executor = KylinConfig.getInstanceFromEnv().getCliCommandExecutor();
+        Pair<Integer, String> cmdOutput = executor.execute(diagCmd);
+        logger.info(cmdOutput.getValue());
+
+        if (cmdOutput.getKey() != 0) {
+            throw new RuntimeException("Failed to generate diagnosis package.");
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 2657b12..171644a 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -17,7 +17,8 @@
  limitations under the License.
 -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>kylin</artifactId>
         <groupId>org.apache.kylin</groupId>
@@ -48,6 +49,7 @@
         </dependency>
     </dependencies>
 
+
     <build>
         <plugins>
             <plugin>
@@ -65,7 +67,7 @@
                             <shadedClassifierName>assembly</shadedClassifierName>
                             <artifactSet>
                                 <includes>
-                                    <include>org.apache.kylin:*</include>
+                                    <include>org.apache.kylin:kylin-tool</include>
                                 </includes>
                             </artifactSet>
                             <filters>

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
index 014d004..cb2ebd0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/AbstractInfoExtractor.java
@@ -87,7 +87,7 @@ public abstract class AbstractInfoExtractor extends AbstractApplication {
 
         // create new folder to contain the output
         String packageName = packageType.toLowerCase() + "_" + new SimpleDateFormat("YYYY_MM_dd_HH_mm_ss").format(new Date());
-        if (new File(exportDest).exists()) {
+        if (!isSubmodule && new File(exportDest).exists()) {
             exportDest = exportDest + packageName + "/";
         }
         exportDir = new File(exportDest);

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 895c4e0..baeb78f 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -20,6 +20,7 @@ package org.apache.kylin.tool;
 
 import java.io.File;
 import java.util.List;
+import java.util.Set;
 
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
@@ -53,7 +54,7 @@ import org.apache.kylin.tool.util.ResourceStoreUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 /**
  * extract cube related info for debugging/distributing purpose
@@ -98,9 +99,9 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
     boolean includeSegmentDetails;
     boolean onlyJobOutput;
 
-    List<String> requiredResources = Lists.newArrayList();
-    List<String> optionalResources = Lists.newArrayList();
-    List<CubeInstance> cubesToTrimAndSave = Lists.newArrayList();//these cubes needs to be saved skipping segments
+    Set<String> requiredResources = Sets.newLinkedHashSet();
+    Set<String> optionalResources = Sets.newLinkedHashSet();
+    Set<CubeInstance> cubesToTrimAndSave = Sets.newLinkedHashSet();//these cubes needs to be saved skipping segments
 
     public CubeMetaExtractor() {
         super();
@@ -149,6 +150,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
                 for (RealizationEntry realizationEntry : realizationEntries) {
                     retrieveResourcePath(getRealization(realizationEntry));
                 }
+                List<DataModelDesc> modelDescs = metadataManager.getModels(projectName);
+                for (DataModelDesc modelDesc : modelDescs) {
+                    addRequired(DataModelDesc.concatResourcePath(modelDesc.getName()));
+                }
                 addOptional(badQueryHistoryManager.getBadQueriesForProject(projectName).getResourcePath());
             }
         } else if (optionsHelper.hasOption(OPTION_CUBE)) {

http://git-wip-us.apache.org/repos/asf/kylin/blob/c7a8b389/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java b/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
index 104c534..f7e2617 100644
--- a/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
+++ b/tool/src/main/java/org/apache/kylin/tool/util/ResourceStoreUtil.java
@@ -20,7 +20,7 @@
 package org.apache.kylin.tool.util;
 
 import java.lang.reflect.Method;
-import java.util.List;
+import java.util.Collection;
 
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.persistence.RawResource;
@@ -31,7 +31,7 @@ import org.apache.kylin.metadata.MetadataConstants;
  * Created by dongli on 5/5/16.
  */
 public class ResourceStoreUtil {
-    public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, List<String> paths) throws Exception {
+    public static void copy(KylinConfig srcConfig, KylinConfig dstConfig, Collection<String> paths) throws Exception {
         ResourceStore src = ResourceStore.getStore(srcConfig);
         ResourceStore dst = ResourceStore.getStore(dstConfig);
         for (String path : paths) {