You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by xx...@apache.org on 2020/07/28 01:34:50 UTC

[kylin] 04/04: KYLIN-4660 Fix UT and compile error

This is an automated email from the ASF dual-hosted git repository.

xxyu pushed a commit to branch kylin-on-parquet-v2
in repository https://gitbox.apache.org/repos/asf/kylin.git

commit 82045c24a4b1cc3c4e541ec66acbf8009878d41f
Author: XiaoxiangYu <hi...@126.com>
AuthorDate: Mon Jul 27 23:41:44 2020 +0800

    KYLIN-4660 Fix UT and compile error
---
 build/bin/download-flink.sh                        |   81 --
 build/bin/find-flink-dependency.sh                 |   52 -
 build/script/download-flink.sh                     |   71 --
 .../org/apache/kylin/common/KylinConfigBase.java   |  246 +++--
 .../java/org/apache/kylin/common/QueryContext.java |    5 +
 .../kylin/common/annotation/Clarification.java     |   37 +-
 .../apache/kylin/common/annotation/ConfigTag.java  |   74 ++
 .../apache/kylin/common/livy/LivyRestBuilder.java  |    5 +-
 .../org/apache/kylin/job/JobBuilderSupport.java    |  184 ++++
 .../test_kylin_cube_with_slr_left_join_desc.json   |    4 +-
 .../kylin/source/hive/ITHiveTableReaderTest.java   |   28 +-
 .../org/apache/spark/conf/rule/SparkConfRule.scala |    8 +-
 .../kylin/rest/controller/CubeController.java      |  140 ++-
 .../kylin/rest/controller/JobController.java       |   40 +-
 .../kylin/rest/controller/StreamingController.java |  564 +++++------
 .../controller/StreamingCoordinatorController.java |  302 +++---
 .../rest/controller/StreamingV2Controller.java     |  964 +++++++++---------
 .../kylin/rest/controller/TableController.java     |   62 +-
 .../kylin/rest/job/StorageCleanJobHbaseUtil.java   |   35 -
 .../apache/kylin/rest/service/BasicService.java    |    9 +-
 .../org/apache/kylin/rest/service/CubeService.java |   59 +-
 .../apache/kylin/rest/service/HBaseInfoUtil.java   |   59 +-
 .../org/apache/kylin/rest/service/JobService.java  |   95 +-
 .../kylin/rest/service/KafkaConfigService.java     |  129 ++-
 .../apache/kylin/rest/service/QueryService.java    |   27 +-
 .../rest/service/StreamingCoordinatorService.java  |  173 ++--
 .../kylin/rest/service/StreamingV2Service.java     | 1057 +++++++++-----------
 .../apache/kylin/rest/service/TableService.java    |  202 ++--
 .../kylin/rest/job/StorageCleanupJobTest.java      |    1 -
 server/ServiceConfig.md                            |   37 -
 .../kylin/rest/service/AdminServiceTest.java       |    2 +-
 .../apache/kylin/source/hive/HiveFlinkInput.java   |   92 --
 .../apache/kylin/source/hive/HiveInputBase.java    |   57 --
 .../org/apache/kylin/source/hive/HiveSource.java   |   13 +-
 .../org/apache/kylin/tool/CubeMigrationCLI.java    |    9 +-
 .../org/apache/kylin/tool/DiagnosisInfoCLI.java    |   31 +-
 .../kylin/tool/extractor/ClientEnvExtractor.java   |    1 -
 .../kylin/tool/extractor/CubeMetaExtractor.java    |   64 +-
 .../kylin/tool/extractor/HBaseUsageExtractor.java  |  249 -----
 .../kylin/tool/extractor/NamedThreadFactory.java   |   34 +-
 .../kylin/tool/metrics/systemcube/SCCreator.java   |   27 +-
 .../streamingv2/StreamingMetadataCreator.java      |  100 --
 42 files changed, 2315 insertions(+), 3114 deletions(-)

diff --git a/build/bin/download-flink.sh b/build/bin/download-flink.sh
deleted file mode 100755
index 1f361bd..0000000
--- a/build/bin/download-flink.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-
-if [ -d "${KYLIN_HOME}/flink" ]; then
-  echo "Flink binary exists"
-  exit 0;
-else
-  echo "Downloading flink package..."
-fi
-
-flink_package_dir=/tmp/flink_package
-
-mkdir -p -- "${flink_package_dir}" && cd -P -- "${flink_package_dir}"
-
-alias md5cmd="md5sum"
-if [[ `uname -a` =~ "Darwin" ]]; then
-    alias md5cmd="md5 -q"
-fi
-
-flink_version="1.9.0"
-scala_version="2.11"
-flink_shaded_version="10.0"
-hadoop_version="2.7.5"
-flink_pkg_md5="e0b5ce7f6352009c74b6c369f5872a5a"
-flink_shaded_hadoop_md5="4287a314bfb09a3dc957cbda3f91d7ca"
-
-if [ ! -f "flink-${flink_version}-bin-scala_${scala_version}.tgz" ]; then
-    echo "No binary file found, start to download package to ${flink_package_dir}"
-    wget http://archive.apache.org/dist/flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz || echo "Download flink failed"
-else
-    if [ `md5cmd flink-${flink_version}-bin-scala_${scala_version}.tgz | awk '{print $1}'` != "${flink_pkg_md5}" ]; then
-        echo "md5 check failed"
-        rm flink-${flink_version}-bin-scala_${scala_version}.tgz
-        wget http://archive.apache.org/dist/flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz || echo "Download flink failed"
-    fi
-fi
-
-flink_shaded_hadoop_jar="flink-shaded-hadoop-2-uber-${hadoop_version}-${flink_shaded_version}.jar"
-flink_shaded_hadoop_path="https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/${hadoop_version}-${flink_shaded_version}/${flink_shaded_hadoop_jar}"
-
-if [ ! -f $flink_shaded_hadoop_jar ]; then
-  echo "Start to download $flink_shaded_hadoop_jar"
-  wget $flink_shaded_hadoop_path || echo "Download flink shaded hadoop jar failed"
-else
-  if [ `md5cmd $flink_shaded_hadoop_jar | awk '{print $1}'` != $flink_shaded_hadoop_md5 ]; then
-    echo "md5 check failed"
-    rm $flink_shaded_hadoop_jar
-    wget $flink_shaded_hadoop_path || echo "Download flink shaded hadoop jar failed"
-  fi
-fi
-unalias md5cmd
-
-echo "Start to decompress package"
-tar -zxvf flink-${flink_version}-bin-scala_${scala_version}.tgz || { exit 1; }
-mv flink-${flink_version} flink
-mv $flink_shaded_hadoop_jar flink/lib
-
-# mv flink binary to KYLIN_HOME
-mv flink ${KYLIN_HOME}
-
-rm -rf ${flink_package_dir}
-
-echo "Download flink binary done"
\ No newline at end of file
diff --git a/build/bin/find-flink-dependency.sh b/build/bin/find-flink-dependency.sh
deleted file mode 100755
index 4485730..0000000
--- a/build/bin/find-flink-dependency.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-source $(cd -P -- "$(dirname -- "$0")" && pwd -P)/header.sh
-
-echo Retrieving Flink dependency...
-
-flink_home=
-
-if [ -n "$FLINK_HOME" ]
-then
-    verbose "FLINK_HOME is set to: $FLINK_HOME, use it to locate Flink dependencies."
-    flink_home=$FLINK_HOME
-fi
-
-if [ -z "$FLINK_HOME" ]
-then
-    verbose "FLINK_HOME wasn't set, use $KYLIN_HOME/flink"
-    flink_home=$KYLIN_HOME/flink
-fi
-
-if [ ! -d "$flink_home/lib" ]
-  then
-    echo "Optional dependency flink not found, if you need this; set FLINK_HOME, or run bin/download-flink.sh"
-    echo "echo 'skip flink_dependency'" > ${dir}/cached-flink-dependency.sh
-  else
-    flink_dependency=`find -L $flink_home/lib -name '*.jar' ! -name '*shaded-hadoop*' ! -name 'kafka*' ! -name '*log4j*' ! -name '*slf4j*' ! -name '*calcite*' ! -name '*doc*' ! -name '*test*' ! -name '*sources*' ''-printf '%p:' | sed 's/:$//'`
-    if [ -z "$flink_dependency" ]
-    then
-        quit "flink jars not found"
-    else
-        verbose "flink dependency: $flink_dependency"
-        export flink_dependency
-    fi
-    echo "export flink_dependency=$flink_dependency" > ${dir}/cached-flink-dependency.sh
-fi
\ No newline at end of file
diff --git a/build/script/download-flink.sh b/build/script/download-flink.sh
deleted file mode 100644
index 1185904..0000000
--- a/build/script/download-flink.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-dir=$(dirname ${0})
-cd ${dir}/../..
-
-rm -rf build/flink
-
-alias md5cmd="md5sum"
-if [[ `uname -a` =~ "Darwin" ]]; then
-    alias md5cmd="md5 -q"
-fi
-
-flink_version="1.9.0"
-scala_version="2.11"
-flink_pkg_md5="e0b5ce7f6352009c74b6c369f5872a5a"
-guava_dependency_version="14.0.1"
-jersey_version="1.9"
-hadoop_version="2.7.5"
-hbase_version="1.1.1"
-yammer_version="2.2.0"
-htrace_version="3.1.0-incubating"
-
-if [ ! -f "build/flink-${flink_version}-bin-scala_${scala_version}.tgz" ]; then
-    echo "no binary file found"
-    wget --directory-prefix=build/ http://archive.apache.org/dist/flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz || echo "Download flink failed"
-else
-    if [ `md5cmd build/ | awk '{print $1}'` != "${flink_pkg_md5}" ]; then
-        echo "md5 check failed"
-        rm build/flink-${flink_version}-bin-scala_${scala_version}.tgz
-        wget --directory-prefix=build/ http://archive.apache.org/dist/flink/flink-${flink_version}/flink-${flink_version}-bin-scala_${scala_version}.tgz || echo "Download flink failed"
-    fi
-fi
-unalias md5cmd
-
-tar -zxvf build/flink-${flink_version}-bin-scala_${scala_version}.tgz -C build/   || { exit 1; }
-mv build/flink-${flink_version} build/flink
-
-# Remove unused components in Flink
-rm -rf build/flink/examples
-rm -rf build/flink/opt
-
-# Download some dependencies
-wget --directory-prefix=build/flink/lib/ https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/${hadoop_version}-7.0/flink-shaded-hadoop-2-uber-${hadoop_version}-7.0.jar || echo "Download flink shaded hadoop dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/com/google/guava/guava/${guava_dependency_version}/guava-${guava_dependency_version}.jar || echo "Download guava dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/flink/flink-hadoop-compatibility_${scala_version}/${flink_version}/flink-hadoop-compatibility_${scala_version}-${flink_version}.jar || echo "Download flink-hadoop-compatibility dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/com/sun/jersey/jersey-core/${jersey_version}/jersey-core-${jersey_version}.jar || echo "Download jersey-core dependency failed."
-## For FlinkCubeHFile
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/hbase/hbase-common/${hbase_version}/hbase-common-${hbase_version}.jar || echo "Download hbase-common dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/hbase/hbase-server/${hbase_version}/hbase-server-${hbase_version}.jar || echo "Download hbase-server dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/hbase/hbase-client/${hbase_version}/hbase-client-${hbase_version}.jar || echo "Download hbase-client dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/hbase/hbase-protocol/${hbase_version}/hbase-protocol-${hbase_version}.jar || echo "Download hbase-protocol dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/hbase/hbase-hadoop-compat/${hbase_version}/hbase-hadoop-compat-${hbase_version}.jar || echo "Download hbase-hadoop-compat dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/com/yammer/metrics/metrics-core/${yammer_version}/metrics-core-${yammer_version}.jar || echo "Download yammer metrics-core dependency failed."
-wget --directory-prefix=build/flink/lib/ http://central.maven.org/maven2/org/apache/htrace/htrace-core/${htrace_version}/htrace-core-${htrace_version}.jar || echo "Download htrace-core dependency failed."
\ No newline at end of file
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index e713752..be7eeb4 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -14,7 +14,7 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
-*/
+ */
 
 package org.apache.kylin.common;
 
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.text.StrSubstitutor;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.annotation.ConfigTag;
 import org.apache.kylin.common.lock.DistributedLockFactory;
 import org.apache.kylin.common.persistence.HDFSResourceStore;
 import org.apache.kylin.common.util.ClassUtil;
@@ -189,7 +190,6 @@ public abstract class KylinConfigBase implements Serializable {
     }
 
     /**
-     *
      * @param propertyKeys the collection of the properties; if null will return all properties
      * @return properties which contained in propertyKeys
      */
@@ -427,6 +427,7 @@ public abstract class KylinConfigBase implements Serializable {
     public boolean isZKLocal() {
         return Boolean.parseBoolean(getOptional("kylin.env.zookeeper-is-local", FALSE));
     }
+
     public String getZKAuths() {
         return getOptional("kylin.env.zookeeper.zk-auth", "digest:ADMIN:KYLIN");
     }
@@ -505,7 +506,7 @@ public abstract class KylinConfigBase implements Serializable {
 
     public String[] getRealizationProviders() {
         return getOptionalStringArray("kylin.metadata.realization-providers", //
-                new String[] { "org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager" });
+                new String[]{"org.apache.kylin.cube.CubeManager", "org.apache.kylin.storage.hybrid.HybridManager"});
     }
 
     public String[] getCubeDimensionCustomEncodingFactories() {
@@ -586,34 +587,76 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.dictionary.append-version-ttl", "259200000"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getCachedSnapshotMaxEntrySize() {
         return Integer.parseInt(getOptional("kylin.snapshot.max-cache-entry", "500"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getTableSnapshotMaxMB() {
         return Integer.parseInt(getOptional("kylin.snapshot.max-mb", "300"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getExtTableSnapshotShardingMB() {
         return Integer.parseInt(getOptional("kylin.snapshot.ext.shard-mb", "500"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getExtTableSnapshotLocalCachePath() {
         return getOptional("kylin.snapshot.ext.local.cache.path", "lookup_cache");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public double getExtTableSnapshotLocalCacheMaxSizeGB() {
         return Double.parseDouble(getOptional("kylin.snapshot.ext.local.cache.max-size-gb", "200"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public long getExtTableSnapshotLocalCacheCheckVolatileRange() {
         return Long.parseLong(getOptional("kylin.snapshot.ext.local.cache.check.volatile", "3600000"));
     }
 
+    @ConfigTag({ConfigTag.Tag.DEPRECATED, ConfigTag.Tag.CUBE_LEVEL})
     public boolean isShrunkenDictFromGlobalEnabled() {
         return Boolean.parseBoolean(this.getOptional("kylin.dictionary.shrunken-from-global-enabled", TRUE));
     }
 
+
+    // ============================================================================
+    // Hive Global Dictionary
+    //
+    // ============================================================================
+
+    /**
+     * @return if mr-hive dict not enabled, return empty array
+     * else return array contains "{TABLE_NAME}_{COLUMN_NAME}"
+     */
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
+    public String[] getMrHiveDictColumns() {
+        String columnStr = getOptional("kylin.dictionary.mr-hive.columns", "");
+        if (!columnStr.equals("")) {
+            return columnStr.split(",");
+        }
+        return new String[0];
+    }
+
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
+    public String getMrHiveDictDB() {
+        return getOptional("kylin.dictionary.mr-hive.database", getHiveDatabaseForIntermediateTable());
+    }
+
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
+    public String getMrHiveDictTableSuffix() {
+        return getOptional("kylin.dictionary.mr-hive.table.suffix", "_global_dict");
+    }
+
+    // ============================================================================
+    // Distributed/Spark Global dictionary
+    // Add wiki link here
+    // ============================================================================
+
+
     public int getGlobalDictV2MinHashPartitions() {
         return Integer.parseInt(getOptional("kylin.dictionary.globalV2-min-hash-partitions", "10"));
     }
@@ -626,11 +669,6 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.dictionary.slice.eviction.threshold", "5"));
     }
 
-
-    // ============================================================================
-    // mr-hive dict
-    // ============================================================================
-
     public double getGlobalDictV2InitLoadFactor() {
         return Double.parseDouble(getOptional("kylin.dictionary.globalV2-init-load-factor", "0.5"));
     }
@@ -643,34 +681,10 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.dictionary.globalV2-max-versions", "3"));
     }
 
-    // ============================================================================
-    // CUBE
-    // ============================================================================
-
     public long getGlobalDictV2VersionTTL() {
         return Long.parseLong(getOptional("kylin.dictionary.globalV2-version-ttl", "259200000"));
     }
 
-    /**
-     * @return  if mr-hive dict not enabled, return empty array
-     *          else return array contains "{TABLE_NAME}_{COLUMN_NAME}"
-     */
-    public String[] getMrHiveDictColumns() {
-        String columnStr = getOptional("kylin.dictionary.mr-hive.columns", "");
-        if (!columnStr.equals("")) {
-            return columnStr.split(",");
-        }
-        return new String[0];
-    }
-
-    public String getMrHiveDictDB() {
-        return getOptional("kylin.dictionary.mr-hive.database", getHiveDatabaseForIntermediateTable());
-    }
-
-    public String getMrHiveDictTableSuffix() {
-        return getOptional("kylin.dictionary.mr-hive.table.suffix", "_global_dict");
-    }
-
     // ============================================================================
     // CUBE
     // ============================================================================
@@ -1021,12 +1035,12 @@ public abstract class KylinConfigBase implements Serializable {
         Map<Integer, String> r = Maps.newLinkedHashMap();
         // ref constants in ISourceAware
         r.put(0, "org.apache.kylin.source.hive.HiveSource");
-        r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
-        r.put(8, "org.apache.kylin.source.jdbc.JdbcSource");
+//        r.put(1, "org.apache.kylin.source.kafka.KafkaSource");
+//        r.put(8, "org.apache.kylin.source.jdbc.JdbcSource");
         r.put(9, "org.apache.kylin.engine.spark.source.CsvSource");
-        r.put(16, "org.apache.kylin.source.jdbc.extensible.JdbcSource");
-        r.put(20, "org.apache.kylin.stream.source.kafka.KafkaBatchSourceAdaptor");
-        r.put(21, "org.apache.kylin.stream.source.kafka.KafkaBatchSourceAdaptor");
+//        r.put(16, "org.apache.kylin.source.jdbc.extensible.JdbcSource");
+//        r.put(20, "org.apache.kylin.stream.source.kafka.KafkaBatchSourceAdaptor");
+//        r.put(21, "org.apache.kylin.stream.source.kafka.KafkaBatchSourceAdaptor");
         r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.source.provider.")));
         return r;
     }
@@ -1035,10 +1049,12 @@ public abstract class KylinConfigBase implements Serializable {
      * whether to enable quote identifier in create flat table in Kylin, how to quote is chosen by
      * @see KylinConfigBase#getFactTableDialect
      */
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public boolean enableHiveDdlQuote() {
         return Boolean.parseBoolean(getOptional("kylin.source.hive.quote-enabled", TRUE));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getFactTableDialect() {
         return getOptional("kylin.fact.table.dialect", "hive");
     }
@@ -1172,10 +1188,12 @@ public abstract class KylinConfigBase implements Serializable {
         }
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHiveIntermediateTablePrefix() {
         return getOptional("kylin.source.hive.intermediate-table-prefix", "kylin_intermediate_");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHiveMetaDataType() {
         return getOptional("kylin.source.hive.metadata-type", "hcatalog");
     }
@@ -1184,6 +1202,7 @@ public abstract class KylinConfigBase implements Serializable {
     // SOURCE.KAFKA
     // ============================================================================
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public Map<String, String> getKafkaConfigOverride() {
         return getPropertiesByPrefix("kylin.source.kafka.config-override.");
     }
@@ -1192,46 +1211,57 @@ public abstract class KylinConfigBase implements Serializable {
     // SOURCE.JDBC
     // ============================================================================
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getJdbcSourceConnectionUrl() {
         return getOptional("kylin.source.jdbc.connection-url");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getJdbcSourceDriver() {
         return getOptional("kylin.source.jdbc.driver");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getJdbcSourceDialect() {
         return getOptional("kylin.source.jdbc.dialect", DEFAULT);
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getJdbcSourceUser() {
         return getOptional("kylin.source.jdbc.user");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getJdbcSourcePass() {
         return getOptional("kylin.source.jdbc.pass");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getSqoopHome() {
         return getOptional("kylin.source.jdbc.sqoop-home");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public int getSqoopMapperNum() {
         return Integer.parseInt(getOptional("kylin.source.jdbc.sqoop-mapper-num", "4"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getSqoopNullString() {
         return getOptional("kylin.source.jdbc.sqoop-null-string", "\\\\N");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getSqoopNullNonString() {
         return getOptional("kylin.source.jdbc.sqoop-null-non-string", "\\\\N");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public Map<String, String> getSqoopConfigOverride() {
         return getPropertiesByPrefix("kylin.source.jdbc.sqoop-config-override.");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_IMPLEMENTED)
     public String getJdbcSourceFieldDelimiter() {
         return getOptional("kylin.source.jdbc.field-delimiter", "|");
     }
@@ -1243,10 +1273,10 @@ public abstract class KylinConfigBase implements Serializable {
     public Map<Integer, String> getStorageEngines() {
         Map<Integer, String> r = Maps.newLinkedHashMap();
         // ref constants in IStorageAware
-        r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage");
+//        r.put(0, "org.apache.kylin.storage.hbase.HBaseStorage");
         r.put(1, "org.apache.kylin.storage.hybrid.HybridStorage");
-        r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage");
-        r.put(3, "org.apache.kylin.storage.stream.StreamStorage");
+//        r.put(2, "org.apache.kylin.storage.hbase.HBaseStorage");
+//        r.put(3, "org.apache.kylin.storage.stream.StreamStorage");
         r.put(4, "org.apache.kylin.engine.spark.storage.ParquetDataStorage");
         r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.storage.provider.")));
         return r;
@@ -1274,22 +1304,27 @@ public abstract class KylinConfigBase implements Serializable {
         return StorageURL.valueOf(url);
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHBaseTableNamePrefix() {
         return getOptional("kylin.storage.hbase.table-name-prefix", "KYLIN_");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHBaseStorageNameSpace() {
         return getOptional("kylin.storage.hbase.namespace", DEFAULT);
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHBaseClusterFs() {
         return getOptional("kylin.storage.hbase.cluster-fs", "");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHBaseClusterHDFSConfigFile() {
         return getOptional("kylin.storage.hbase.cluster-hdfs-config-file", "");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getCoprocessorLocalJar() {
         final String coprocessorJar = getOptional(KYLIN_STORAGE_HBASE_COPROCESSOR_LOCAL_JAR);
         if (StringUtils.isNotEmpty(coprocessorJar)) {
@@ -1302,114 +1337,141 @@ public abstract class KylinConfigBase implements Serializable {
         return getFileName(kylinHome + File.separator + "lib", COPROCESSOR_JAR_NAME_PATTERN);
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public void overrideCoprocessorLocalJar(String path) {
         logger.info("override {} to {}", KYLIN_STORAGE_HBASE_COPROCESSOR_LOCAL_JAR, path);
         System.setProperty(KYLIN_STORAGE_HBASE_COPROCESSOR_LOCAL_JAR, path);
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseRegionCountMin() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.min-region-count", "1"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseRegionCountMax() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.max-region-count", "500"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public float getHBaseHFileSizeGB() {
         return Float.parseFloat(getOptional("kylin.storage.hbase.hfile-size-gb", "2.0"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public boolean getQueryRunLocalCoprocessor() {
         return Boolean.parseBoolean(getOptional("kylin.storage.hbase.run-local-coprocessor", FALSE));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public double getQueryCoprocessorMemGB() {
         return Double.parseDouble(this.getOptional("kylin.storage.hbase.coprocessor-mem-gb", "3.0"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public boolean getQueryCoprocessorSpillEnabled() {
         return Boolean.parseBoolean(this.getOptional("kylin.storage.partition.aggr-spill-enabled", TRUE));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public long getPartitionMaxScanBytes() {
         long value = Long.parseLong(
                 this.getOptional("kylin.storage.partition.max-scan-bytes", String.valueOf(3L * 1024 * 1024 * 1024)));
         return value > 0 ? value : Long.MAX_VALUE;
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getQueryCoprocessorTimeoutSeconds() {
         return Integer.parseInt(this.getOptional("kylin.storage.hbase.coprocessor-timeout-seconds", "0"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getQueryScanFuzzyKeyMax() {
         return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-fuzzykey-scan", "200"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getQueryScanFuzzyKeySplitMax() {
         return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-fuzzykey-scan-split", "1"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getQueryStorageVisitScanRangeMax() {
         return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-visit-scanrange", "1000000"));
     }
 
+    @ConfigTag(ConfigTag.Tag.UNCATEGORIZED)
     public String getDefaultIGTStorage() {
         return getOptional("kylin.storage.hbase.gtstorage",
                 "org.apache.kylin.storage.hbase.cube.v2.CubeHBaseEndpointRPC");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseScanCacheRows() {
         return Integer.parseInt(this.getOptional("kylin.storage.hbase.scan-cache-rows", "1024"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public float getKylinHBaseRegionCut() {
         return Float.parseFloat(getOptional("kylin.storage.hbase.region-cut-gb", "5.0"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseScanMaxResultSize() {
         return Integer.parseInt(this.getOptional("kylin.storage.hbase.max-scan-result-bytes", "" + (5 * 1024 * 1024))); // 5 MB
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHbaseDefaultCompressionCodec() {
         return getOptional("kylin.storage.hbase.compression-codec", "none");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getHbaseDefaultEncoding() {
         return getOptional("kylin.storage.hbase.rowkey-encoding", "FAST_DIFF");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHbaseDefaultBlockSize() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.block-size-bytes", "1048576"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHbaseSmallFamilyBlockSize() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.small-family-block-size-bytes", "65536"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public String getKylinOwner() {
         return this.getOptional("kylin.storage.hbase.owner-tag", "");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public boolean getCompressionResult() {
         return Boolean.parseBoolean(getOptional("kylin.storage.hbase.endpoint-compress-result", TRUE));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseMaxConnectionThreads() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.max-hconnection-threads", "2048"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseCoreConnectionThreads() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.core-hconnection-threads", "2048"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public long getHBaseConnectionThreadPoolAliveSeconds() {
         return Long.parseLong(getOptional("kylin.storage.hbase.hconnection-threads-alive-seconds", "60"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public int getHBaseReplicationScope() {
         return Integer.parseInt(getOptional("kylin.storage.hbase.replication-scope", "0"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEPRECATED)
     public boolean cleanStorageAfterDelOperation() {
         return Boolean.parseBoolean(getOptional("kylin.storage.clean-after-delete-operation", FALSE));
     }
@@ -1421,11 +1483,11 @@ public abstract class KylinConfigBase implements Serializable {
     public Map<Integer, String> getJobEngines() {
         Map<Integer, String> r = Maps.newLinkedHashMap();
         // ref constants in IEngineAware
-        r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); //IEngineAware.ID_MR_V1
-        r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); //IEngineAware.ID_MR_V2
-        r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2"); //IEngineAware.ID_SPARK
+//        r.put(0, "org.apache.kylin.engine.mr.MRBatchCubingEngine"); //IEngineAware.ID_MR_V1
+//        r.put(2, "org.apache.kylin.engine.mr.MRBatchCubingEngine2"); //IEngineAware.ID_MR_V2
+//        r.put(4, "org.apache.kylin.engine.spark.SparkBatchCubingEngine2"); //IEngineAware.ID_SPARK
         r.put(6, "org.apache.kylin.engine.spark.SparkBatchCubingEngineParquet"); //IEngineAware.ID_SPARK_II
-        r.put(5, "org.apache.kylin.engine.flink.FlinkBatchCubingEngine2"); //IEngineAware.ID_FLINK
+//        r.put(5, "org.apache.kylin.engine.flink.FlinkBatchCubingEngine2"); //IEngineAware.ID_FLINK
         r.putAll(convertKeyToInteger(getPropertiesByPrefix("kylin.engine.provider.")));
         return r;
     }
@@ -1434,22 +1496,7 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.engine.default", "6"));
     }
 
-    public String getKylinParquetJobJarPath() {
-        final String jobJar = getOptional(KYLIN_ENGINE_PARQUET_JOB_JAR);
-        if (StringUtils.isNotEmpty(jobJar)) {
-            return jobJar;
-        }
-        String kylinHome = getKylinHome();
-        if (StringUtils.isEmpty(kylinHome)) {
-            return "";
-        }
-        return getFileName(kylinHome + File.separator + "lib", PARQUET_JOB_JAR_NAME_PATTERN);
-    }
 
-    public void overrideKylinParquetJobJarPath(String path) {
-        logger.info("override {} to {}", KYLIN_ENGINE_PARQUET_JOB_JAR, path);
-        System.setProperty(KYLIN_ENGINE_PARQUET_JOB_JAR, path);
-    }
 
     public String getKylinJobJarPath() {
         final String jobJar = getOptional(KYLIN_ENGINE_MR_JOB_JAR);
@@ -1925,7 +1972,7 @@ public abstract class KylinConfigBase implements Serializable {
 
     public boolean isPushDownEnabled() {
         return Boolean.parseBoolean(this.getOptional("kylin.query.pushdown.enabled", FALSE))
-               || StringUtils.isNotEmpty(getPushDownRunnerClassName());
+                || StringUtils.isNotEmpty(getPushDownRunnerClassName());
     }
 
     public boolean isPushDownUpdateEnabled() {
@@ -1944,7 +1991,7 @@ public abstract class KylinConfigBase implements Serializable {
         List<String> ids = Lists.newArrayList();
         String idsStr = getOptional("kylin.query.pushdown.runner.ids", "");
         if (StringUtils.isNotEmpty(idsStr)) {
-            for (String id: idsStr.split(",")) {
+            for (String id : idsStr.split(",")) {
                 ids.add(id);
             }
         }
@@ -1953,7 +2000,7 @@ public abstract class KylinConfigBase implements Serializable {
 
     public String[] getPushDownConverterClassNames() {
         return getOptionalStringArray("kylin.query.pushdown.converter-class-names",
-                new String[] { "org.apache.kylin.source.adhocquery.HivePushDownConverter" });
+                new String[]{"org.apache.kylin.source.adhocquery.HivePushDownConverter"});
     }
 
     public boolean isPushdownQueryCacheEnabled() {
@@ -2098,7 +2145,7 @@ public abstract class KylinConfigBase implements Serializable {
     }
 
     public int[] getQueryMetricsPercentilesIntervals() {
-        String[] dft = { "60", "300", "3600" };
+        String[] dft = {"60", "300", "3600"};
         return getOptionalIntArray("kylin.server.query-metrics-percentiles-intervals", dft);
     }
 
@@ -2273,7 +2320,7 @@ public abstract class KylinConfigBase implements Serializable {
         return getPropertiesByPrefix("kylin.metrics.");
     }
 
-    public int printSampleEventRatio(){
+    public int printSampleEventRatio() {
         String val = getOptional("kylin.metrics.kafka-sample-ratio", "10000");
         return Integer.parseInt(val);
     }
@@ -2505,7 +2552,7 @@ public abstract class KylinConfigBase implements Serializable {
         return (getOptional("kylin.stream.event.timezone", ""));
     }
 
-    public boolean isAutoResubmitDiscardJob(){
+    public boolean isAutoResubmitDiscardJob() {
         return Boolean.parseBoolean(getOptional("kylin.stream.auto-resubmit-after-discard-enabled", "true"));
     }
 
@@ -2538,8 +2585,27 @@ public abstract class KylinConfigBase implements Serializable {
     }
 
     // ============================================================================
-    // Kylin on parquetv2 related
+    // Kylin 4.x related
     // ============================================================================
+
+    public String getKylinParquetJobJarPath() {
+        final String jobJar = getOptional(KYLIN_ENGINE_PARQUET_JOB_JAR);
+        if (StringUtils.isNotEmpty(jobJar)) {
+            return jobJar;
+        }
+        String kylinHome = getKylinHome();
+        if (StringUtils.isEmpty(kylinHome)) {
+            return "";
+        }
+        return getFileName(kylinHome + File.separator + "lib", PARQUET_JOB_JAR_NAME_PATTERN);
+    }
+
+    public void overrideKylinParquetJobJarPath(String path) {
+        logger.info("override {} to {}", KYLIN_ENGINE_PARQUET_JOB_JAR, path);
+        System.setProperty(KYLIN_ENGINE_PARQUET_JOB_JAR, path);
+    }
+
+    @ConfigTag(ConfigTag.Tag.DEBUG_HACK)
     public String getSparkBuildClassName() {
         return getOptional("kylin.engine.spark.build-class-name", "org.apache.kylin.engine.spark.job.CubeBuildJob");
     }
@@ -2558,7 +2624,7 @@ public abstract class KylinConfigBase implements Serializable {
 
     //Auto adjust the memory of driver
     public int[] getSparkEngineDriverMemoryStrategy() {
-        String[] dft = { "2", "20", "100" };
+        String[] dft = {"2", "20", "100"};
         return getOptionalIntArray("kylin.engine.driver-memory-strategy", dft);
     }
 
@@ -2577,7 +2643,9 @@ public abstract class KylinConfigBase implements Serializable {
         return new Path(path);
     }
 
-    // a_b => a/b/
+    /**
+     * a_b => a/b/
+     */
     private String getNestedPath(String id) {
         String[] ids = id.split("_");
         StringBuilder builder = new StringBuilder();
@@ -2591,6 +2659,7 @@ public abstract class KylinConfigBase implements Serializable {
         return getHdfsWorkingDirectory() + project + "/job_tmp/";
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public int getPersistFlatTableThreshold() {
         return Integer.parseInt(getOptional("kylin.engine.persist-flattable-threshold", "1"));
     }
@@ -2639,24 +2708,29 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.valueOf(getOptional("kylin.storage.columnar.repartition-threshold-size-mb", "128"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public int getParquetStorageShardMin() {
         return Integer.valueOf(getOptional("kylin.storage.columnar.shard-min", "1"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public int getParquetStorageShardMax() {
         return Integer.valueOf(getOptional("kylin.storage.columnar.shard-max", "1000"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public int getParquetStorageBlockSize() {
         int defaultBlockSize = 5 * getParquetStorageShardSizeMB() * 1024 * 1024; //default (5 * shard_size)
         return Integer.valueOf(getOptional("kylin.storage.columnar.hdfs-blocksize-bytes",
                 String.valueOf(defaultBlockSize < 0 ? Integer.MAX_VALUE : defaultBlockSize)));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public int getParquetSpliceShardExpandFactor() {
         return Integer.valueOf(getOptional("kylin.storage.columnar.shard-expand-factor", "10"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public int getParquetDfsReplication() {
         return Integer.valueOf(getOptional("kylin.storage.columnar.dfs-replication", "3"));
     }
@@ -2673,11 +2747,13 @@ public abstract class KylinConfigBase implements Serializable {
         return Boolean.parseBoolean(getOptional("kylin.job.log-print-enabled", "true"));
     }
 
+    @ConfigTag(ConfigTag.Tag.DEBUG_HACK)
     public String getClusterInfoFetcherClassName() {
         return getOptional("kylin.engine.spark.cluster-info-fetcher-class-name",
                 "org.apache.kylin.cluster.YarnInfoFetcher");
     }
 
+    @ConfigTag(ConfigTag.Tag.DEBUG_HACK)
     public String getSparkMergeClassName() {
         return getOptional("kylin.engine.spark.merge-class-name", "org.apache.kylin.engine.spark.job.CubeMergeJob");
     }
@@ -2698,11 +2774,11 @@ public abstract class KylinConfigBase implements Serializable {
         return Double.parseDouble(getOptional("kylin.engine.max-allocation-proportion", "0.9"));
     }
 
-    public int getSparkEngineBaseExuctorInstances() {
+    public int getSparkEngineBaseExecutorInstances() {
         return Integer.parseInt(getOptional("kylin.engine.base-executor-instance", "5"));
     }
 
-    public String getSparkEngineExuctorInstanceStrategy() {
+    public String getSparkEngineExecutorInstanceStrategy() {
         return getOptional("kylin.engine.executor-instance-strategy", "100,2,500,3,1000,4");
     }
 
@@ -2710,12 +2786,14 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(getOptional("kylin.snapshot.shard-size-mb", "128"));
     }
 
+    /**
+     * driver memory that can be used by join(mostly BHJ)
+     */
     public double getJoinMemoryFraction() {
-        // driver memory that can be used by join(mostly BHJ)
         return Double.parseDouble(getOptional("kylin.query.spark-engine.join-memory-fraction", "0.3"));
     }
 
-    //  spark parquet
+    @ConfigTag(ConfigTag.Tag.DEBUG_HACK)
     public boolean isSparkEngineEnabled() {
         return Boolean.parseBoolean(getOptional("kylin.query.spark-engine.enabled", "true"));
     }
@@ -2748,6 +2826,11 @@ public abstract class KylinConfigBase implements Serializable {
         return getPropertiesByPrefix("kylin.query.spark-conf.");
     }
 
+    public String getIntersectFilterOrSeparator() {
+        return getOptional("kylin.query.intersect.separator", "|");
+    }
+
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String sparderFiles() {
         try {
             File storageFile = FileUtils.findFile(KylinConfigBase.getKylinHome() + "/conf",
@@ -2763,6 +2846,7 @@ public abstract class KylinConfigBase implements Serializable {
         }
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String sparderJars() {
         try {
             File storageFile = FileUtils.findFile(KylinConfigBase.getKylinHome() + "/lib", "newten-job.jar");
@@ -2786,9 +2870,9 @@ public abstract class KylinConfigBase implements Serializable {
         return Integer.parseInt(this.getOptional("kylin.query.pushdown.base-shuffle-partition-size", "48"));
     }
 
-    /**
-     * Kerberos
-     */
+    // ============================================================================
+    // Spark with Kerberos
+    // ============================================================================
 
     public Boolean isKerberosEnabled() {
         return Boolean.valueOf(getOptional("kylin.kerberos.enabled", FALSE));
@@ -2802,38 +2886,47 @@ public abstract class KylinConfigBase implements Serializable {
         return KylinConfig.getKylinConfDir() + File.separator + getKerberosKeytab();
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String getKerberosZKPrincipal() {
         return getOptional("kylin.kerberos.zookeeper.server.principal", "zookeeper/hadoop");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public Long getKerberosTicketRefreshInterval() {
         return Long.valueOf(getOptional("kylin.kerberos.ticket.refresh.interval.minutes", "720"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public Long getKerberosMonitorInterval() {
         return Long.valueOf(getOptional("kylin.kerberos.monitor.interval.minutes", "10"));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String getKerberosPlatform() {
         return getOptional("kylin.kerberos.platform", "");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public Boolean getPlatformZKEnable() {
         return Boolean.valueOf(getOptional("kylin.platform.zk.kerberos.enable", FALSE));
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String getKerberosKrb5Conf() {
         return getOptional("kylin.kerberos.krb5.conf", "krb5.conf");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String getKerberosKrb5ConfPath() {
         return KylinConfig.getKylinConfDir() + File.separator + getKerberosKrb5Conf();
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String getKerberosJaasConf() {
         return getOptional("kylin.kerberos.jaas.conf", "jaas.conf");
     }
 
+    @ConfigTag(ConfigTag.Tag.NOT_CLEAR)
     public String getKerberosJaasConfPath() {
         return KylinConfig.getKylinConfDir() + File.separator + getKerberosJaasConf();
     }
@@ -2841,7 +2934,4 @@ public abstract class KylinConfigBase implements Serializable {
     public String getKerberosPrincipal() {
         return getOptional("kylin.kerberos.principal");
     }
-    public String getIntersectFilterOrSeparator() {
-        return getOptional("kylin.query.intersect.separator", "|");
-    }
 }
diff --git a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
index 845154a..6e71e6a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
+++ b/core-common/src/main/java/org/apache/kylin/common/QueryContext.java
@@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.apache.kylin.common.annotation.Clarification;
 import org.apache.kylin.common.exceptions.KylinTimeoutException;
 import org.apache.kylin.common.util.RandomUtil;
 import org.slf4j.Logger;
@@ -167,10 +168,12 @@ public class QueryContext {
         return sourceScanRows.addAndGet(rows);
     }
 
+    @Clarification(priority = Clarification.Priority.MAJOR, msg = "remove this")
     public void addQueryStopListener(QueryStopListener listener) {
         this.stopListeners.add(listener);
     }
 
+    @Clarification(priority = Clarification.Priority.MAJOR, msg = "Maybe we can change spark.scheduler.pool to implement resource isolation.")
     public void setHighPriorityQuery(boolean highPriorityQuery) {
         isHighPriorityQuery = highPriorityQuery;
     }
@@ -203,10 +206,12 @@ public class QueryContext {
         this.dataset = dataset;
     }
 
+    @Clarification(priority = Clarification.Priority.MAJOR, msg = "remove this")
     public boolean isTableIndex() {
         return isTableIndex;
     }
 
+    @Clarification(priority = Clarification.Priority.MAJOR, msg = "remove this")
     public void setTableIndex(boolean tableIndex) {
         isTableIndex = tableIndex;
     }
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java b/core-common/src/main/java/org/apache/kylin/common/annotation/Clarification.java
similarity index 50%
copy from core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java
copy to core-common/src/main/java/org/apache/kylin/common/annotation/Clarification.java
index f473059..b039e50 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java
+++ b/core-common/src/main/java/org/apache/kylin/common/annotation/Clarification.java
@@ -14,26 +14,29 @@
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
-*/
+ */
+package org.apache.kylin.common.annotation;
 
-package org.apache.kylin.dict.lookup;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
 
-import org.apache.kylin.metadata.model.TableDesc;
+/**
+ * Please add more documentation to me and remove me after that.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER,
+        ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, ElementType.PACKAGE})
+public @interface Clarification {
 
-public interface IExtLookupTableCache {
-    enum CacheState {NONE, IN_BUILDING, AVAILABLE}
+    Priority priority();
 
-    /**
-     * @param tableDesc
-     * @param extTableSnapshotInfo
-     * @param buildIfNotExist if true, when the cached lookup table not exist, build it.
-     * @return null if no cached lookup table exist
-     */
-    ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, boolean buildIfNotExist);
+    String msg() default "N/A";
 
-    void buildSnapshotCache(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, ILookupTable sourceTable);
-
-    void removeSnapshotCache(ExtTableSnapshotInfo extTableSnapshotInfo);
-
-    CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo);
+    enum Priority {
+        MINOR,
+        MAJOR,
+        CRITICAL
+    }
 }
diff --git a/core-common/src/main/java/org/apache/kylin/common/annotation/ConfigTag.java b/core-common/src/main/java/org/apache/kylin/common/annotation/ConfigTag.java
new file mode 100644
index 0000000..53a6967
--- /dev/null
+++ b/core-common/src/main/java/org/apache/kylin/common/annotation/ConfigTag.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Standardize and clarify config property of KylinConfig,
+ * which helps to make them easy to understand.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER,
+        ElementType.CONSTRUCTOR, ElementType.LOCAL_VARIABLE, ElementType.PACKAGE})
+public @interface ConfigTag {
+
+    Tag[] value();
+
+    enum Tag {
+        /**
+         * Indicate this property will be removed soon.
+         */
+        DEPRECATED,
+
+        NOT_CLEAR,
+
+        /**
+         * The implementation is not complement
+         */
+        NOT_IMPLEMENTED,
+
+        /**
+         * To be categorized
+         */
+        UNCATEGORIZED,
+
+        /**
+         * For hacker or developer
+         */
+        DEBUG_HACK,
+
+        /**
+         * Support cube level configuration
+         */
+        CUBE_LEVEL,
+
+        /**
+         * Support project level configuration
+         */
+        PROJECT_LEVEL,
+
+        /**
+         * Only support global configuration
+         */
+        GLOBAL_LEVEL
+    }
+}
\ No newline at end of file
diff --git a/core-common/src/main/java/org/apache/kylin/common/livy/LivyRestBuilder.java b/core-common/src/main/java/org/apache/kylin/common/livy/LivyRestBuilder.java
index 218b4e4..18dbe5a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/livy/LivyRestBuilder.java
+++ b/core-common/src/main/java/org/apache/kylin/common/livy/LivyRestBuilder.java
@@ -20,6 +20,7 @@ package org.apache.kylin.common.livy;
 
 import com.google.common.collect.Lists;
 import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.annotation.Clarification;
 import org.apache.kylin.common.util.SourceConfigurationUtil;
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -28,9 +29,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.Map;
 
-/**
- *
- */
+@Clarification(priority = Clarification.Priority.MINOR, msg = "I guess it should be removed, we may not continue support this  feature.")
 public class LivyRestBuilder {
     protected static final org.slf4j.Logger logger = LoggerFactory.getLogger(LivyRestBuilder.class);
 
diff --git a/core-job/src/main/java/org/apache/kylin/job/JobBuilderSupport.java b/core-job/src/main/java/org/apache/kylin/job/JobBuilderSupport.java
new file mode 100644
index 0000000..4216145
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/JobBuilderSupport.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.StorageURL;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.job.engine.JobEngineConfig;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ * Hold reusable steps for builders.
+ */
+public class JobBuilderSupport {
+
+    final protected JobEngineConfig config;
+    final protected CubeSegment seg;
+    final protected String submitter;
+    final protected Integer priorityOffset;
+
+    final public static String LayeredCuboidFolderPrefix = "level_";
+
+    final public static String PathNameCuboidBase = "base_cuboid";
+    final public static String PathNameCuboidOld = "old";
+    final public static String PathNameCuboidInMem = "in_memory";
+    final public static Pattern JOB_NAME_PATTERN = Pattern.compile("kylin-([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})");
+
+    public JobBuilderSupport(CubeSegment seg, String submitter) {
+        this(seg, submitter, 0);
+    }
+
+    public JobBuilderSupport(CubeSegment seg, String submitter, Integer priorityOffset) {
+        Preconditions.checkNotNull(seg, "segment cannot be null");
+        this.config = new JobEngineConfig(seg.getConfig());
+        this.seg = seg;
+        this.submitter = submitter;
+        this.priorityOffset = priorityOffset;
+    }
+
+    // ============================================================================
+
+    public String getJobWorkingDir(String jobId) {
+        return getJobWorkingDir(config, jobId);
+    }
+
+    public String getRealizationRootPath(String jobId) {
+        return getJobWorkingDir(jobId) + "/" + seg.getRealization().getName();
+    }
+
+    public String getCuboidRootPath(String jobId) {
+        return getRealizationRootPath(jobId) + "/cuboid/";
+    }
+
+    public String getCuboidRootPath(CubeSegment seg) {
+        return getCuboidRootPath(seg.getLastBuildJobID());
+    }
+
+    public void appendMapReduceParameters(StringBuilder buf) {
+        appendMapReduceParameters(buf, JobEngineConfig.DEFAULT_JOB_CONF_SUFFIX);
+    }
+
+    public void appendMapReduceParameters(StringBuilder buf, String jobType) {
+        try {
+            String jobConf = config.getHadoopJobConfFilePath(jobType);
+            if (jobConf != null && jobConf.length() > 0) {
+                buf.append(" -conf ").append(jobConf);
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public String getOptimizationRootPath(String jobId) {
+        return getRealizationRootPath(jobId) + "/optimize";
+    }
+
+    // ============================================================================
+    // static methods also shared by other job flow participant
+    // ----------------------------------------------------------------------------
+
+    public static String getJobWorkingDir(JobEngineConfig conf, String jobId) {
+        return getJobWorkingDir(conf.getHdfsWorkingDirectory(), jobId);
+    }
+
+    public static String getJobWorkingDir(String hdfsDir, String jobId) {
+        if (!hdfsDir.endsWith("/")) {
+            hdfsDir = hdfsDir + "/";
+        }
+        return hdfsDir + "kylin-" + jobId;
+    }
+
+    public static StringBuilder appendExecCmdParameters(StringBuilder buf, String paraName, String paraValue) {
+        return buf.append(" -").append(paraName).append(" ").append(paraValue);
+    }
+
+    public static String getCuboidOutputPathsByLevel(String cuboidRootPath, int level) {
+        if (level == 0) {
+            return cuboidRootPath + LayeredCuboidFolderPrefix + PathNameCuboidBase;
+        } else {
+            return cuboidRootPath + LayeredCuboidFolderPrefix + level + "_cuboid";
+        }
+    }
+
+    public static String getBaseCuboidPath(String cuboidRootPath) {
+        return cuboidRootPath + PathNameCuboidBase;
+    }
+
+    public static String getInMemCuboidPath(String cuboidRootPath) {
+        return cuboidRootPath + PathNameCuboidInMem;
+    }
+
+    public String getDumpMetadataPath(String jobId) {
+        return getRealizationRootPath(jobId) + "/metadata";
+    }
+
+    public static String extractJobIDFromPath(String path) {
+        Matcher matcher = JOB_NAME_PATTERN.matcher(path);
+        // check the first occurrence
+        if (matcher.find()) {
+            return matcher.group(1);
+        } else {
+            throw new IllegalStateException("Can not extract job ID from file path : " + path);
+        }
+    }
+
+    public String getSegmentMetadataUrl(KylinConfig kylinConfig, String jobId) {
+        Map<String, String> param = new HashMap<>();
+        param.put("path", getDumpMetadataPath(jobId));
+        return new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "hdfs", param).toString();
+    }
+
+    public static void scanFiles(String input, FileSystem fs, List<FileStatus> outputs) throws IOException {
+        Path path = new Path(input);
+        if (!fs.exists(path)) {
+            return;
+        }
+        FileStatus[] fileStatuses = fs.listStatus(path, p -> !p.getName().startsWith("_"));
+        for (FileStatus stat : fileStatuses) {
+            if (stat.isDirectory()) {
+                scanFiles(stat.getPath().toString(), fs, outputs);
+            } else {
+                outputs.add(stat);
+            }
+        }
+    }
+
+    public static long getFileSize(String input, FileSystem fs) throws IOException {
+        List<FileStatus> outputs = Lists.newArrayList();
+        scanFiles(input, fs, outputs);
+        long size = 0L;
+        for (FileStatus stat: outputs) {
+            size += stat.getLen();
+        }
+        return size;
+    }
+}
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
index ff2af55..e564074 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_with_slr_left_join_desc.json
@@ -284,8 +284,8 @@
   "status_need_notify": [],
   "auto_merge_time_ranges": null,
   "retention_range": 0,
-  "engine_type": 2,
-  "storage_type": 2,
+  "engine_type": 6,
+  "storage_type": 4,
   "override_kylin_properties": {
     "kylin.cube.algorithm": "INMEM"
   },
diff --git a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
index 25598ad..c4fb7e2 100644
--- a/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
+++ b/kylin-it/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
@@ -32,18 +32,18 @@ import org.junit.Test;
  */
 public class ITHiveTableReaderTest extends HBaseMetadataTestCase {
 
-    @Test
-    public void test() throws IOException {
-        HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
-        int rowNumber = 0;
-        while (reader.next()) {
-            String[] row = reader.getRow();
-            Assert.assertEquals(12, row.length);
-            //System.out.println(ArrayUtils.toString(row));
-            rowNumber++;
-        }
-
-        reader.close();
-        Assert.assertEquals(10000, rowNumber);
-    }
+//    @Test
+//    public void test() throws IOException {
+//        HiveTableReader reader = new HiveTableReader("default", "test_kylin_fact");
+//        int rowNumber = 0;
+//        while (reader.next()) {
+//            String[] row = reader.getRow();
+//            Assert.assertEquals(12, row.length);
+//            //System.out.println(ArrayUtils.toString(row));
+//            rowNumber++;
+//        }
+//
+//        reader.close();
+//        Assert.assertEquals(10000, rowNumber);
+//    }
 }
diff --git a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
index 3f5a8d2..ff47891 100644
--- a/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
+++ b/kylin-spark-project/kylin-spark-engine/src/main/scala/org/apache/spark/conf/rule/SparkConfRule.scala
@@ -102,7 +102,7 @@ class ExecutorInstancesRule extends SparkConfRule {
     val layoutSize = helper.getOption(SparkConfHelper.LAYOUT_SIZE)
     val requiredCores = helper.getOption(SparkConfHelper.REQUIRED_CORES)
     logInfo(s"RequiredCores is $requiredCores")
-    val baseExecutorInstances = KylinConfig.getInstanceFromEnv.getSparkEngineBaseExuctorInstances
+    val baseExecutorInstances = KylinConfig.getInstanceFromEnv.getSparkEngineBaseExecutorInstances
     val calculateExecutorInsByLayoutSize = calculateExecutorInstanceSizeByLayoutSize(Integer.parseInt(layoutSize))
 
 
@@ -128,18 +128,18 @@ class ExecutorInstancesRule extends SparkConfRule {
   }
 
   override def fallback(helper: SparkConfHelper): Unit = {
-    helper.setConf(SparkConfHelper.EXECUTOR_INSTANCES, KylinConfig.getInstanceFromEnv.getSparkEngineBaseExuctorInstances.toString)
+    helper.setConf(SparkConfHelper.EXECUTOR_INSTANCES, KylinConfig.getInstanceFromEnv.getSparkEngineBaseExecutorInstances.toString)
   }
 
 
   def calculateExecutorInstanceSizeByLayoutSize(layoutSize: Int): Int = {
     logInfo(s"Calculate the number of executor instance size based on the number of layouts: $layoutSize")
     val config: KylinConfig = KylinConfig.getInstanceFromEnv
-    val baseInstances: Integer = config.getSparkEngineBaseExuctorInstances
+    val baseInstances: Integer = config.getSparkEngineBaseExecutorInstances
     var instanceMultiple = 1
 
     if (layoutSize != -1) {
-      val instanceStrategy: String = config.getSparkEngineExuctorInstanceStrategy
+      val instanceStrategy: String = config.getSparkEngineExecutorInstanceStrategy
       val tuple = instanceStrategy.split(",")
         .zipWithIndex
         .partition(tp => tp._2 % 2 == 0)
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 511eb86..aac0b47 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -47,12 +47,10 @@ import org.apache.kylin.cube.model.HBaseColumnFamilyDesc;
 import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
-import org.apache.kylin.engine.mr.common.CuboidStatsReaderUtil;
 import org.apache.kylin.job.JobInstance;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.job.exception.JobException;
 import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.ISourceAware;
 import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentRange;
@@ -70,7 +68,6 @@ import org.apache.kylin.rest.request.CubeRequest;
 import org.apache.kylin.rest.request.JobBuildRequest;
 import org.apache.kylin.rest.request.JobBuildRequest2;
 import org.apache.kylin.rest.request.JobOptimizeRequest;
-import org.apache.kylin.rest.request.LookupSnapshotBuildRequest;
 import org.apache.kylin.rest.response.CubeInstanceResponse;
 import org.apache.kylin.rest.response.CuboidTreeResponse;
 import org.apache.kylin.rest.response.EnvelopeResponse;
@@ -83,7 +80,6 @@ import org.apache.kylin.rest.service.ProjectService;
 import org.apache.kylin.rest.service.QueryService;
 import org.apache.kylin.rest.util.AclEvaluate;
 import org.apache.kylin.rest.util.ValidateUtil;
-import org.apache.kylin.source.kafka.util.KafkaClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -297,28 +293,6 @@ public class CubeController extends BasicController {
     }
 
     /**
-     * Force rebuild a cube's lookup table snapshot
-     *
-     * @throws IOException
-     */
-    @RequestMapping(value = "/{cubeName}/refresh_lookup", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public JobInstance rebuildLookupSnapshot(@PathVariable String cubeName,
-            @RequestBody LookupSnapshotBuildRequest request) {
-        try {
-            final CubeManager cubeMgr = cubeService.getCubeManager();
-            final CubeInstance cube = cubeMgr.getCube(cubeName);
-            String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
-            return jobService.submitLookupSnapshotJob(cube, request.getLookupTableName(), request.getSegmentIDs(),
-                    submitter);
-        } catch (IOException e) {
-            logger.error(e.getLocalizedMessage(), e);
-            throw new InternalErrorException(e.getLocalizedMessage(), e);
-        }
-    }
-
-    /**
      * Delete a cube segment
      *
      * @throws IOException
@@ -903,31 +877,31 @@ public class CubeController extends BasicController {
         }
     }
 
-    @RequestMapping(value = "/{cubeName}/cuboids/current", method = RequestMethod.GET)
-    @ResponseBody
-    public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName) {
-        checkCubeExists(cubeName);
-        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
-        // The cuboid tree displayed should be consistent with the current one
-        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
-        Map<Long, Long> cuboidStatsMap = cube.getCuboids();
-        if (cuboidStatsMap == null) {
-            cuboidStatsMap = CuboidStatsReaderUtil.readCuboidStatsFromCube(cuboidScheduler.getAllCuboidIds(), cube);
-        }
-
-        Map<Long, Long> hitFrequencyMap = null;
-        Map<Long, Long> queryMatchMap = null;
-        try {
-            hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
-            queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
-        } catch (Exception e) {
-            logger.warn("Fail to query on system cube due to " + e);
-        }
-
-        Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
-        return cubeService.getCuboidTreeResponse(cuboidScheduler, cuboidStatsMap, hitFrequencyMap, queryMatchMap,
-                currentCuboidSet);
-    }
+//    @RequestMapping(value = "/{cubeName}/cuboids/current", method = RequestMethod.GET)
+//    @ResponseBody
+//    public CuboidTreeResponse getCurrentCuboids(@PathVariable String cubeName) {
+//        checkCubeExists(cubeName);
+//        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);
+//        // The cuboid tree displayed should be consistent with the current one
+//        CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
+//        Map<Long, Long> cuboidStatsMap = cube.getCuboids();
+//        if (cuboidStatsMap == null) {
+//            cuboidStatsMap = CuboidStatsReaderUtil.readCuboidStatsFromCube(cuboidScheduler.getAllCuboidIds(), cube);
+//        }
+//
+//        Map<Long, Long> hitFrequencyMap = null;
+//        Map<Long, Long> queryMatchMap = null;
+//        try {
+//            hitFrequencyMap = getTargetCuboidHitFrequency(cubeName);
+//            queryMatchMap = cubeService.getCuboidQueryMatchCount(cubeName);
+//        } catch (Exception e) {
+//            logger.warn("Fail to query on system cube due to " + e);
+//        }
+//
+//        Set<Long> currentCuboidSet = cube.getCuboidScheduler().getAllCuboidIds();
+//        return cubeService.getCuboidTreeResponse(cuboidScheduler, cuboidStatsMap, hitFrequencyMap, queryMatchMap,
+//                currentCuboidSet);
+//    }
 
     @RequestMapping(value = "/{cubeName}/cuboids/recommend", method = RequestMethod.GET)
     @ResponseBody
@@ -968,38 +942,38 @@ public class CubeController extends BasicController {
         return cubeService.getCuboidHitFrequency(cubeName, false);
     }
 
-    /**
-     * Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
-     * source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.
-     *
-     * @param cubeName
-     * @return
-     */
-    @RequestMapping(value = "/{cubeName}/init_start_offsets", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public GeneralResponse initStartOffsets(@PathVariable String cubeName) {
-        checkCubeExists(cubeName);
-        CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
-        if (cubeInstance.getSourceType() != ISourceAware.ID_STREAMING) {
-            String msg = "Cube '" + cubeName + "' is not a Streaming Cube.";
-            throw new IllegalArgumentException(msg);
-        }
-
-        final GeneralResponse response = new GeneralResponse();
-        try {
-            final Map<Integer, Long> startOffsets = KafkaClient.getLatestOffsets(cubeInstance);
-            CubeDesc desc = cubeInstance.getDescriptor();
-            desc.setPartitionOffsetStart(startOffsets);
-            cubeService.getCubeDescManager().updateCubeDesc(desc);
-            response.setProperty("result", "success");
-            response.setProperty("offsets", startOffsets.toString());
-        } catch (Throwable e) {
-            throw new RuntimeException(e);
-        }
-
-        return response;
-    }
+//    /**
+//     * Initiate the very beginning of a streaming cube. Will seek the latest offests of each partition from streaming
+//     * source (kafka) and record in the cube descriptor; In the first build job, it will use these offests as the start point.
+//     *
+//     * @param cubeName
+//     * @return
+//     */
+//    @RequestMapping(value = "/{cubeName}/init_start_offsets", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public GeneralResponse initStartOffsets(@PathVariable String cubeName) {
+//        checkCubeExists(cubeName);
+//        CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
+//        if (cubeInstance.getSourceType() != ISourceAware.ID_STREAMING) {
+//            String msg = "Cube '" + cubeName + "' is not a Streaming Cube.";
+//            throw new IllegalArgumentException(msg);
+//        }
+//
+//        final GeneralResponse response = new GeneralResponse();
+//        try {
+//            final Map<Integer, Long> startOffsets = KafkaClient.getLatestOffsets(cubeInstance);
+//            CubeDesc desc = cubeInstance.getDescriptor();
+//            desc.setPartitionOffsetStart(startOffsets);
+//            cubeService.getCubeDescManager().updateCubeDesc(desc);
+//            response.setProperty("result", "success");
+//            response.setProperty("offsets", startOffsets.toString());
+//        } catch (Throwable e) {
+//            throw new RuntimeException(e);
+//        }
+//
+//        return response;
+//    }
 
     private CubeDesc deserializeCubeDesc(CubeRequest cubeRequest) {
         CubeDesc desc = null;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
index 8fa9e5c..65c036c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/JobController.java
@@ -193,26 +193,26 @@ public class JobController extends BasicController {
         }
     }
 
-    /**
-     * resubmit streaming segment
-     *
-     * @throws IOException
-     */
-    @RequestMapping(value = "/{jobId}/resubmit", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public void resubmitJob(@PathVariable String jobId) throws IOException {
-        try {
-            final JobInstance jobInstance = jobService.getJobInstance(jobId);
-            if (jobInstance == null) {
-                throw new BadRequestException("Cannot find job: " + jobId);
-            }
-            jobService.resubmitJob(jobInstance);
-        } catch (Exception e) {
-            logger.error(e.getLocalizedMessage(), e);
-            throw e;
-        }
-    }
+//    /**
+//     * resubmit streaming segment
+//     *
+//     * @throws IOException
+//     */
+//    @RequestMapping(value = "/{jobId}/resubmit", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public void resubmitJob(@PathVariable String jobId) throws IOException {
+//        try {
+//            final JobInstance jobInstance = jobService.getJobInstance(jobId);
+//            if (jobInstance == null) {
+//                throw new BadRequestException("Cannot find job: " + jobId);
+//            }
+//            jobService.resubmitJob(jobInstance);
+//        } catch (Exception e) {
+//            logger.error(e.getLocalizedMessage(), e);
+//            throw e;
+//        }
+//    }
 
     /**
      * Cancel/discard a job
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
index 6d98434..5a4eb94 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingController.java
@@ -18,40 +18,8 @@
 
 package org.apache.kylin.rest.controller;
 
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.common.util.RandomUtil;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.streaming.StreamingConfig;
-import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.exception.ForbiddenException;
-import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.exception.NotFoundException;
-import org.apache.kylin.rest.request.StreamingRequest;
-import org.apache.kylin.rest.service.KafkaConfigService;
-import org.apache.kylin.rest.service.StreamingService;
-import org.apache.kylin.rest.service.TableService;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.security.access.AccessDeniedException;
 import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonMappingException;
 
 /**
  * StreamingController is defined as Restful API entrance for UI.
@@ -61,271 +29,271 @@ import com.fasterxml.jackson.databind.JsonMappingException;
 @Controller
 @RequestMapping(value = "/streaming")
 public class StreamingController extends BasicController {
-    private static final Logger logger = LoggerFactory.getLogger(StreamingController.class);
-
-    @Autowired
-    @Qualifier("streamingMgmtService")
-    private StreamingService streamingService;
-
-    @Autowired
-    @Qualifier("kafkaMgmtService")
-    private KafkaConfigService kafkaConfigService;
-
-    @Autowired
-    @Qualifier("tableService")
-    private TableService tableService;
-
-    @RequestMapping(value = "/getConfig", method = { RequestMethod.GET }, produces = { "application/json" })
-    @ResponseBody
-    public List<StreamingConfig> getStreamings(@RequestParam(value = "table", required = false) String table,
-            @RequestParam(value = "project", required = false) String project,
-            @RequestParam(value = "limit", required = false) Integer limit,
-            @RequestParam(value = "offset", required = false) Integer offset) {
-        try {
-            return streamingService.getStreamingConfigs(table, project, limit, offset);
-        } catch (IOException e) {
-
-            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
-        }
-    }
-
-    @RequestMapping(value = "/getKfkConfig", method = { RequestMethod.GET }, produces = { "application/json" })
-    @ResponseBody
-    public List<KafkaConfig> getKafkaConfigs(
-            @RequestParam(value = "kafkaConfigName", required = false) String kafkaConfigName,
-            @RequestParam(value = "project", required = false) String project,
-            @RequestParam(value = "limit", required = false) Integer limit,
-            @RequestParam(value = "offset", required = false) Integer offset) {
-        try {
-            return kafkaConfigService.getKafkaConfigs(kafkaConfigName, project, limit, offset);
-        } catch (IOException e) {
-            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
-        }
-    }
-
-    /**
-     *
-     * create Streaming Schema
-     * @throws java.io.IOException
-     */
-    @RequestMapping(value = "", method = { RequestMethod.POST }, produces = { "application/json" })
-    @ResponseBody
-    public StreamingRequest saveStreamingConfig(@RequestBody StreamingRequest streamingRequest) {
-        String project = streamingRequest.getProject();
-        TableDesc tableDesc = deserializeTableDesc(streamingRequest);
-        if (null == tableDesc) {
-            throw new BadRequestException("Failed to add streaming table.");
-        }
-
-        StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
-        if (!streamingRequest.isSuccessful()) {
-            return streamingRequest;
-        }
-        KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
-        if (!streamingRequest.isSuccessful()) {
-            return streamingRequest;
-        }
-        boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
-
-        try {
-            tableService.loadTableToProject(tableDesc, null, project);
-        } catch (IOException e) {
-            throw new BadRequestException("Failed to add streaming table.");
-        }
-
-        streamingConfig.setName(tableDesc.getIdentity());
-        kafkaConfig.setName(tableDesc.getIdentity());
-
-        InternalErrorException exception = null;
-        try {
-            if (StringUtils.isEmpty(streamingConfig.getName())) {
-                logger.info("StreamingConfig should not be empty.");
-                throw new BadRequestException("StremingConfig name should not be empty.");
-            }
-            try {
-                streamingConfig.setUuid(RandomUtil.randomUUID().toString());
-                streamingService.createStreamingConfig(streamingConfig, project);
-                saveStreamingSuccess = true;
-            } catch (IOException e) {
-                logger.error("Failed to save StreamingConfig:" + e.getLocalizedMessage(), e);
-                throw new InternalErrorException("Failed to save StreamingConfig: " + e.getLocalizedMessage(), e);
-            }
-            try {
-                kafkaConfig.setUuid(RandomUtil.randomUUID().toString());
-                kafkaConfigService.createKafkaConfig(kafkaConfig, project);
-                saveKafkaSuccess = true;
-            } catch (IOException e) {
-                try {
-                    streamingService.dropStreamingConfig(streamingConfig, project);
-                } catch (IOException e1) {
-                    throw new InternalErrorException(
-                            "StreamingConfig is created, but failed to create KafkaConfig: " + e.getLocalizedMessage(),
-                            e);
-                }
-                logger.error("Failed to save KafkaConfig:" + e.getLocalizedMessage(), e);
-                throw new InternalErrorException("Failed to save KafkaConfig: " + e.getLocalizedMessage(), e);
-            }
-        } finally {
-            if (saveKafkaSuccess == false || saveStreamingSuccess == false) {
-                if (saveStreamingSuccess == true) {
-                    StreamingConfig sConfig = streamingService.getStreamingManager()
-                            .getStreamingConfig(streamingConfig.getName());
-                    try {
-                        streamingService.dropStreamingConfig(sConfig, project);
-                    } catch (IOException e) {
-                        exception = new InternalErrorException(
-                                "Action failed and failed to rollback the created streaming config: "
-                                        + e.getLocalizedMessage(),
-                                e);
-                    }
-                }
-                if (saveKafkaSuccess == true) {
-                    try {
-                        KafkaConfig kConfig = kafkaConfigService.getKafkaConfig(kafkaConfig.getName(), project);
-                        kafkaConfigService.dropKafkaConfig(kConfig, project);
-                    } catch (IOException e) {
-                        exception = new InternalErrorException(
-                                "Action failed and failed to rollback the created kafka config: "
-                                        + e.getLocalizedMessage(),
-                                e);
-                    }
-                }
-            }
-        }
-
-        if (null != exception) {
-            throw exception;
-        }
-
-        streamingRequest.setSuccessful(true);
-        return streamingRequest;
-    }
-
-    @RequestMapping(value = "", method = { RequestMethod.PUT }, produces = { "application/json" })
-    @ResponseBody
-    public StreamingRequest updateStreamingConfig(@RequestBody StreamingRequest streamingRequest)
-            throws JsonProcessingException {
-        StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
-        if (!streamingRequest.isSuccessful()) {
-            return streamingRequest;
-        }
-        KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
-        if (!streamingRequest.isSuccessful()) {
-            return streamingRequest;
-        }
-        String project = streamingRequest.getProject();
-        if (streamingConfig == null) {
-            return streamingRequest;
-        }
-        try {
-            streamingConfig = streamingService.updateStreamingConfig(streamingConfig, project);
-        } catch (AccessDeniedException accessDeniedException) {
-            throw new ForbiddenException("You don't have right to update this StreamingConfig.");
-        } catch (Exception e) {
-            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
-        }
-        try {
-            kafkaConfig = kafkaConfigService.updateKafkaConfig(kafkaConfig, project);
-        } catch (AccessDeniedException accessDeniedException) {
-            throw new ForbiddenException("You don't have right to update this KafkaConfig.");
-        } catch (Exception e) {
-            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
-        }
-
-        streamingRequest.setSuccessful(true);
-
-        return streamingRequest;
-    }
-
-    @RequestMapping(value = "/{project}/{configName}", method = { RequestMethod.DELETE }, produces = {
-            "application/json" })
-    @ResponseBody
-    public void deleteConfig(@PathVariable String project, @PathVariable String configName) throws IOException {
-        StreamingConfig config = streamingService.getStreamingManager().getStreamingConfig(configName);
-        KafkaConfig kafkaConfig = kafkaConfigService.getKafkaConfig(configName, project);
-        if (null == config) {
-            throw new NotFoundException("StreamingConfig with name " + configName + " not found..");
-        }
-        try {
-            streamingService.dropStreamingConfig(config, project);
-            kafkaConfigService.dropKafkaConfig(kafkaConfig, project);
-        } catch (Exception e) {
-            logger.error(e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to delete StreamingConfig. " + " Caused by: " + e.getMessage(), e);
-        }
-    }
-
-    private TableDesc deserializeTableDesc(StreamingRequest streamingRequest) {
-        TableDesc desc = null;
-        try {
-            logger.debug("Saving TableDesc " + streamingRequest.getTableData());
-            desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class);
-            updateRequest(streamingRequest, true, null);
-        } catch (JsonParseException e) {
-            logger.error("The TableDesc definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (JsonMappingException e) {
-            logger.error("The data TableDesc definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (IOException e) {
-            logger.error("Failed to deal with the request.", e);
-            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
-        }
-
-        if (null != desc) {
-            String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName());
-            desc.setName(dbTable[1]);
-            desc.setDatabase(dbTable[0]);
-            desc.getIdentity();
-        }
-        return desc;
-    }
-
-    private StreamingConfig deserializeSchemalDesc(StreamingRequest streamingRequest) {
-        StreamingConfig desc = null;
-        try {
-            logger.debug("Saving StreamingConfig " + streamingRequest.getStreamingConfig());
-            desc = JsonUtil.readValue(streamingRequest.getStreamingConfig(), StreamingConfig.class);
-            updateRequest(streamingRequest, true, null);
-        } catch (JsonParseException e) {
-            logger.error("The StreamingConfig definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (JsonMappingException e) {
-            logger.error("The data StreamingConfig definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (IOException e) {
-            logger.error("Failed to deal with the request.", e);
-            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
-        }
-        return desc;
-    }
-
-    private KafkaConfig deserializeKafkaSchemalDesc(StreamingRequest streamingRequest) {
-        KafkaConfig desc = null;
-        try {
-            logger.debug("Saving KafkaConfig " + streamingRequest.getKafkaConfig());
-            desc = JsonUtil.readValue(streamingRequest.getKafkaConfig(), KafkaConfig.class);
-            updateRequest(streamingRequest, true, null);
-        } catch (JsonParseException e) {
-            logger.error("The KafkaConfig definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (JsonMappingException e) {
-            logger.error("The data KafkaConfig definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (IOException e) {
-            logger.error("Failed to deal with the request.", e);
-            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
-        }
-        return desc;
-    }
-
-    private void updateRequest(StreamingRequest request, boolean success, String message) {
-        request.setSuccessful(success);
-        request.setMessage(message);
-    }
+//    private static final Logger logger = LoggerFactory.getLogger(StreamingController.class);
+//
+//    @Autowired
+//    @Qualifier("streamingMgmtService")
+//    private StreamingService streamingService;
+//
+//    @Autowired
+//    @Qualifier("kafkaMgmtService")
+//    private KafkaConfigService kafkaConfigService;
+//
+//    @Autowired
+//    @Qualifier("tableService")
+//    private TableService tableService;
+//
+//    @RequestMapping(value = "/getConfig", method = { RequestMethod.GET }, produces = { "application/json" })
+//    @ResponseBody
+//    public List<StreamingConfig> getStreamings(@RequestParam(value = "table", required = false) String table,
+//            @RequestParam(value = "project", required = false) String project,
+//            @RequestParam(value = "limit", required = false) Integer limit,
+//            @RequestParam(value = "offset", required = false) Integer offset) {
+//        try {
+//            return streamingService.getStreamingConfigs(table, project, limit, offset);
+//        } catch (IOException e) {
+//
+//            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
+//        }
+//    }
+//
+//    @RequestMapping(value = "/getKfkConfig", method = { RequestMethod.GET }, produces = { "application/json" })
+//    @ResponseBody
+//    public List<KafkaConfig> getKafkaConfigs(
+//            @RequestParam(value = "kafkaConfigName", required = false) String kafkaConfigName,
+//            @RequestParam(value = "project", required = false) String project,
+//            @RequestParam(value = "limit", required = false) Integer limit,
+//            @RequestParam(value = "offset", required = false) Integer offset) {
+//        try {
+//            return kafkaConfigService.getKafkaConfigs(kafkaConfigName, project, limit, offset);
+//        } catch (IOException e) {
+//            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
+//        }
+//    }
+//
+//    /**
+//     *
+//     * create Streaming Schema
+//     * @throws java.io.IOException
+//     */
+//    @RequestMapping(value = "", method = { RequestMethod.POST }, produces = { "application/json" })
+//    @ResponseBody
+//    public StreamingRequest saveStreamingConfig(@RequestBody StreamingRequest streamingRequest) {
+//        String project = streamingRequest.getProject();
+//        TableDesc tableDesc = deserializeTableDesc(streamingRequest);
+//        if (null == tableDesc) {
+//            throw new BadRequestException("Failed to add streaming table.");
+//        }
+//
+//        StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
+//        if (!streamingRequest.isSuccessful()) {
+//            return streamingRequest;
+//        }
+//        KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
+//        if (!streamingRequest.isSuccessful()) {
+//            return streamingRequest;
+//        }
+//        boolean saveStreamingSuccess = false, saveKafkaSuccess = false;
+//
+//        try {
+//            tableService.loadTableToProject(tableDesc, null, project);
+//        } catch (IOException e) {
+//            throw new BadRequestException("Failed to add streaming table.");
+//        }
+//
+//        streamingConfig.setName(tableDesc.getIdentity());
+//        kafkaConfig.setName(tableDesc.getIdentity());
+//
+//        InternalErrorException exception = null;
+//        try {
+//            if (StringUtils.isEmpty(streamingConfig.getName())) {
+//                logger.info("StreamingConfig should not be empty.");
+//                throw new BadRequestException("StremingConfig name should not be empty.");
+//            }
+//            try {
+//                streamingConfig.setUuid(RandomUtil.randomUUID().toString());
+//                streamingService.createStreamingConfig(streamingConfig, project);
+//                saveStreamingSuccess = true;
+//            } catch (IOException e) {
+//                logger.error("Failed to save StreamingConfig:" + e.getLocalizedMessage(), e);
+//                throw new InternalErrorException("Failed to save StreamingConfig: " + e.getLocalizedMessage(), e);
+//            }
+//            try {
+//                kafkaConfig.setUuid(RandomUtil.randomUUID().toString());
+//                kafkaConfigService.createKafkaConfig(kafkaConfig, project);
+//                saveKafkaSuccess = true;
+//            } catch (IOException e) {
+//                try {
+//                    streamingService.dropStreamingConfig(streamingConfig, project);
+//                } catch (IOException e1) {
+//                    throw new InternalErrorException(
+//                            "StreamingConfig is created, but failed to create KafkaConfig: " + e.getLocalizedMessage(),
+//                            e);
+//                }
+//                logger.error("Failed to save KafkaConfig:" + e.getLocalizedMessage(), e);
+//                throw new InternalErrorException("Failed to save KafkaConfig: " + e.getLocalizedMessage(), e);
+//            }
+//        } finally {
+//            if (saveKafkaSuccess == false || saveStreamingSuccess == false) {
+//                if (saveStreamingSuccess == true) {
+//                    StreamingConfig sConfig = streamingService.getStreamingManager()
+//                            .getStreamingConfig(streamingConfig.getName());
+//                    try {
+//                        streamingService.dropStreamingConfig(sConfig, project);
+//                    } catch (IOException e) {
+//                        exception = new InternalErrorException(
+//                                "Action failed and failed to rollback the created streaming config: "
+//                                        + e.getLocalizedMessage(),
+//                                e);
+//                    }
+//                }
+//                if (saveKafkaSuccess == true) {
+//                    try {
+//                        KafkaConfig kConfig = kafkaConfigService.getKafkaConfig(kafkaConfig.getName(), project);
+//                        kafkaConfigService.dropKafkaConfig(kConfig, project);
+//                    } catch (IOException e) {
+//                        exception = new InternalErrorException(
+//                                "Action failed and failed to rollback the created kafka config: "
+//                                        + e.getLocalizedMessage(),
+//                                e);
+//                    }
+//                }
+//            }
+//        }
+//
+//        if (null != exception) {
+//            throw exception;
+//        }
+//
+//        streamingRequest.setSuccessful(true);
+//        return streamingRequest;
+//    }
+//
+//    @RequestMapping(value = "", method = { RequestMethod.PUT }, produces = { "application/json" })
+//    @ResponseBody
+//    public StreamingRequest updateStreamingConfig(@RequestBody StreamingRequest streamingRequest)
+//            throws JsonProcessingException {
+//        StreamingConfig streamingConfig = deserializeSchemalDesc(streamingRequest);
+//        if (!streamingRequest.isSuccessful()) {
+//            return streamingRequest;
+//        }
+//        KafkaConfig kafkaConfig = deserializeKafkaSchemalDesc(streamingRequest);
+//        if (!streamingRequest.isSuccessful()) {
+//            return streamingRequest;
+//        }
+//        String project = streamingRequest.getProject();
+//        if (streamingConfig == null) {
+//            return streamingRequest;
+//        }
+//        try {
+//            streamingConfig = streamingService.updateStreamingConfig(streamingConfig, project);
+//        } catch (AccessDeniedException accessDeniedException) {
+//            throw new ForbiddenException("You don't have right to update this StreamingConfig.");
+//        } catch (Exception e) {
+//            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
+//        }
+//        try {
+//            kafkaConfig = kafkaConfigService.updateKafkaConfig(kafkaConfig, project);
+//        } catch (AccessDeniedException accessDeniedException) {
+//            throw new ForbiddenException("You don't have right to update this KafkaConfig.");
+//        } catch (Exception e) {
+//            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage(), e);
+//        }
+//
+//        streamingRequest.setSuccessful(true);
+//
+//        return streamingRequest;
+//    }
+//
+//    @RequestMapping(value = "/{project}/{configName}", method = { RequestMethod.DELETE }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public void deleteConfig(@PathVariable String project, @PathVariable String configName) throws IOException {
+//        StreamingConfig config = streamingService.getStreamingManager().getStreamingConfig(configName);
+//        KafkaConfig kafkaConfig = kafkaConfigService.getKafkaConfig(configName, project);
+//        if (null == config) {
+//            throw new NotFoundException("StreamingConfig with name " + configName + " not found..");
+//        }
+//        try {
+//            streamingService.dropStreamingConfig(config, project);
+//            kafkaConfigService.dropKafkaConfig(kafkaConfig, project);
+//        } catch (Exception e) {
+//            logger.error(e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to delete StreamingConfig. " + " Caused by: " + e.getMessage(), e);
+//        }
+//    }
+//
+//    private TableDesc deserializeTableDesc(StreamingRequest streamingRequest) {
+//        TableDesc desc = null;
+//        try {
+//            logger.debug("Saving TableDesc " + streamingRequest.getTableData());
+//            desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class);
+//            updateRequest(streamingRequest, true, null);
+//        } catch (JsonParseException e) {
+//            logger.error("The TableDesc definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (JsonMappingException e) {
+//            logger.error("The data TableDesc definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (IOException e) {
+//            logger.error("Failed to deal with the request.", e);
+//            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+//        }
+//
+//        if (null != desc) {
+//            String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName());
+//            desc.setName(dbTable[1]);
+//            desc.setDatabase(dbTable[0]);
+//            desc.getIdentity();
+//        }
+//        return desc;
+//    }
+//
+//    private StreamingConfig deserializeSchemalDesc(StreamingRequest streamingRequest) {
+//        StreamingConfig desc = null;
+//        try {
+//            logger.debug("Saving StreamingConfig " + streamingRequest.getStreamingConfig());
+//            desc = JsonUtil.readValue(streamingRequest.getStreamingConfig(), StreamingConfig.class);
+//            updateRequest(streamingRequest, true, null);
+//        } catch (JsonParseException e) {
+//            logger.error("The StreamingConfig definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (JsonMappingException e) {
+//            logger.error("The data StreamingConfig definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (IOException e) {
+//            logger.error("Failed to deal with the request.", e);
+//            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+//        }
+//        return desc;
+//    }
+//
+//    private KafkaConfig deserializeKafkaSchemalDesc(StreamingRequest streamingRequest) {
+//        KafkaConfig desc = null;
+//        try {
+//            logger.debug("Saving KafkaConfig " + streamingRequest.getKafkaConfig());
+//            desc = JsonUtil.readValue(streamingRequest.getKafkaConfig(), KafkaConfig.class);
+//            updateRequest(streamingRequest, true, null);
+//        } catch (JsonParseException e) {
+//            logger.error("The KafkaConfig definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (JsonMappingException e) {
+//            logger.error("The data KafkaConfig definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (IOException e) {
+//            logger.error("Failed to deal with the request.", e);
+//            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+//        }
+//        return desc;
+//    }
+//
+//    private void updateRequest(StreamingRequest request, boolean success, String message) {
+//        request.setSuccessful(success);
+//        request.setMessage(message);
+//    }
 
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingCoordinatorController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingCoordinatorController.java
index e1f737e..405f099 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingCoordinatorController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingCoordinatorController.java
@@ -18,38 +18,8 @@
 
 package org.apache.kylin.rest.controller;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import javax.servlet.http.HttpServletRequest;
-
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.rest.service.StreamingCoordinatorService;
-import org.apache.kylin.stream.core.model.CubeAssignment;
-import org.apache.kylin.stream.core.model.ReplicaSet;
-import org.apache.kylin.stream.coordinator.client.CoordinatorResponse;
-import org.apache.kylin.stream.coordinator.exception.NotLeadCoordinatorException;
-import org.apache.kylin.stream.core.model.Node;
-import org.apache.kylin.stream.core.model.RemoteStoreCompleteRequest;
-import org.apache.kylin.stream.core.model.ReplicaSetLeaderChangeRequest;
-import org.apache.kylin.stream.core.source.Partition;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.http.HttpStatus;
 import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.ExceptionHandler;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.ResponseBody;
-import org.springframework.web.bind.annotation.ResponseStatus;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
 /**
  * <pre>
  * When current process is not the coordinator leader(such as coordinator follower or non coordinator),
@@ -61,140 +31,140 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 @Controller
 @RequestMapping(value = "/streaming_coordinator")
 public class StreamingCoordinatorController extends BasicController {
-    private static final Logger logger = LoggerFactory.getLogger(StreamingCoordinatorController.class);
-
-    @Autowired
-    private StreamingCoordinatorService streamingCoordinartorService;
-
-    @ResponseStatus(HttpStatus.OK)
-    @ExceptionHandler(NotLeadCoordinatorException.class)
-    @ResponseBody
-    CoordinatorResponse handleNotLeadCoordinator(HttpServletRequest req, Exception ex) {
-        CoordinatorResponse response = new CoordinatorResponse();
-        response.setCode(CoordinatorResponse.NOT_LEAD_COORDINATOR);
-        response.setMsg(ex.getMessage());
-        return response;
-    }
-
-    @RequestMapping(value = "/balance/recommend", method = { RequestMethod.GET })
-    @ResponseBody
-    public CoordinatorResponse reBalanceRecommend() {
-        Map<Integer, Map<String, List<Partition>>> result = streamingCoordinartorService.reBalanceRecommend();
-        CoordinatorResponse response = new CoordinatorResponse();
-        response.setData(result);
-        return response;
-    }
-
-    @RequestMapping(value = "/balance", method = { RequestMethod.POST })
-    @ResponseBody
-    public CoordinatorResponse reBalance(@RequestBody String reBalancePlanStr) {
-        Map<Integer, Map<String, List<Partition>>> reBalancePlan = deserializeRebalancePlan(reBalancePlanStr);
-        streamingCoordinartorService.reBalance(reBalancePlan);
-        return new CoordinatorResponse();
-    }
-
-    private Map<Integer, Map<String, List<Partition>>> deserializeRebalancePlan(String reBalancePlanStr) {
-        TypeReference<Map<Integer, Map<String, List<Partition>>>> typeRef = new TypeReference<Map<Integer, Map<String, List<Partition>>>>() {
-        };
-        ObjectMapper mapper = new ObjectMapper();
-        try {
-            return mapper.readValue(reBalancePlanStr, typeRef);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/assign", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse assignStreamingCube(@PathVariable String cubeName) {
-        streamingCoordinartorService.assignCube(cubeName);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/unAssign", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse unAssignStreamingCube(@PathVariable String cubeName) {
-        streamingCoordinartorService.unAssignCube(cubeName);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/reAssign", method = { RequestMethod.POST }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse reAssignStreamingCube(@PathVariable String cubeName,
-            @RequestBody CubeAssignment newAssignments) {
-        streamingCoordinartorService.reAssignCube(cubeName, newAssignments);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/replicaSet", method = { RequestMethod.POST }, produces = { "application/json" })
-    @ResponseBody
-    public CoordinatorResponse createReplicaSet(@RequestBody ReplicaSet rs) {
-        streamingCoordinartorService.createReplicaSet(rs);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/replicaSet/{replicaSetID}", method = { RequestMethod.DELETE }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse deleteReplicaSet(@PathVariable Integer replicaSetID) {
-        streamingCoordinartorService.removeReplicaSet(replicaSetID);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse addNodeToReplicaSet(@PathVariable Integer replicaSetID, @PathVariable String nodeID) {
-        streamingCoordinartorService.addNodeToReplicaSet(replicaSetID, nodeID);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.DELETE }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse removeNodeFromReplicaSet(@PathVariable Integer replicaSetID,
-            @PathVariable String nodeID) {
-        streamingCoordinartorService.removeNodeFromReplicaSet(replicaSetID, nodeID);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/pauseConsume", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse pauseCubeConsume(@PathVariable String cubeName) {
-        streamingCoordinartorService.pauseConsumers(cubeName);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/resumeConsume", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse resumeCubeConsume(@PathVariable String cubeName) {
-        streamingCoordinartorService.resumeConsumers(cubeName);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/remoteStoreComplete", method = { RequestMethod.POST }, produces = { "application/json" })
-    @ResponseBody
-    public CoordinatorResponse segmentRemoteStoreComplete(@RequestBody RemoteStoreCompleteRequest request) {
-        Pair<Long, Long> segmentRange = new Pair<>(request.getSegmentStart(), request.getSegmentEnd());
-        Node receiver = request.getReceiverNode();
-        logger.info(
-                "receive segment remote store complete request for cube:{}, segment:{}, try to find proper segment to build",
-                request.getCubeName(), segmentRange);
-        streamingCoordinartorService.onSegmentRemoteStoreComplete(request.getCubeName(), segmentRange, receiver);
-        return new CoordinatorResponse();
-    }
-
-    @RequestMapping(value = "/replicaSetLeaderChange", method = { RequestMethod.POST }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CoordinatorResponse replicaSetLeaderChange(@RequestBody ReplicaSetLeaderChangeRequest request) {
-        logger.info("receive replicaSet leader change:" + request);
-        streamingCoordinartorService.replicaSetLeaderChange(request.getReplicaSetID(), request.getNewLeader());
-        return new CoordinatorResponse();
-    }
+//    private static final Logger logger = LoggerFactory.getLogger(StreamingCoordinatorController.class);
+//
+//    @Autowired
+//    private StreamingCoordinatorService streamingCoordinartorService;
+//
+//    @ResponseStatus(HttpStatus.OK)
+//    @ExceptionHandler(NotLeadCoordinatorException.class)
+//    @ResponseBody
+//    CoordinatorResponse handleNotLeadCoordinator(HttpServletRequest req, Exception ex) {
+//        CoordinatorResponse response = new CoordinatorResponse();
+//        response.setCode(CoordinatorResponse.NOT_LEAD_COORDINATOR);
+//        response.setMsg(ex.getMessage());
+//        return response;
+//    }
+//
+//    @RequestMapping(value = "/balance/recommend", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public CoordinatorResponse reBalanceRecommend() {
+//        Map<Integer, Map<String, List<Partition>>> result = streamingCoordinartorService.reBalanceRecommend();
+//        CoordinatorResponse response = new CoordinatorResponse();
+//        response.setData(result);
+//        return response;
+//    }
+//
+//    @RequestMapping(value = "/balance", method = { RequestMethod.POST })
+//    @ResponseBody
+//    public CoordinatorResponse reBalance(@RequestBody String reBalancePlanStr) {
+//        Map<Integer, Map<String, List<Partition>>> reBalancePlan = deserializeRebalancePlan(reBalancePlanStr);
+//        streamingCoordinartorService.reBalance(reBalancePlan);
+//        return new CoordinatorResponse();
+//    }
+//
+//    private Map<Integer, Map<String, List<Partition>>> deserializeRebalancePlan(String reBalancePlanStr) {
+//        TypeReference<Map<Integer, Map<String, List<Partition>>>> typeRef = new TypeReference<Map<Integer, Map<String, List<Partition>>>>() {
+//        };
+//        ObjectMapper mapper = new ObjectMapper();
+//        try {
+//            return mapper.readValue(reBalancePlanStr, typeRef);
+//        } catch (IOException e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/assign", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse assignStreamingCube(@PathVariable String cubeName) {
+//        streamingCoordinartorService.assignCube(cubeName);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/unAssign", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse unAssignStreamingCube(@PathVariable String cubeName) {
+//        streamingCoordinartorService.unAssignCube(cubeName);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/reAssign", method = { RequestMethod.POST }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse reAssignStreamingCube(@PathVariable String cubeName,
+//            @RequestBody CubeAssignment newAssignments) {
+//        streamingCoordinartorService.reAssignCube(cubeName, newAssignments);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/replicaSet", method = { RequestMethod.POST }, produces = { "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse createReplicaSet(@RequestBody ReplicaSet rs) {
+//        streamingCoordinartorService.createReplicaSet(rs);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/replicaSet/{replicaSetID}", method = { RequestMethod.DELETE }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse deleteReplicaSet(@PathVariable Integer replicaSetID) {
+//        streamingCoordinartorService.removeReplicaSet(replicaSetID);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse addNodeToReplicaSet(@PathVariable Integer replicaSetID, @PathVariable String nodeID) {
+//        streamingCoordinartorService.addNodeToReplicaSet(replicaSetID, nodeID);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.DELETE }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse removeNodeFromReplicaSet(@PathVariable Integer replicaSetID,
+//            @PathVariable String nodeID) {
+//        streamingCoordinartorService.removeNodeFromReplicaSet(replicaSetID, nodeID);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/pauseConsume", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse pauseCubeConsume(@PathVariable String cubeName) {
+//        streamingCoordinartorService.pauseConsumers(cubeName);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/resumeConsume", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse resumeCubeConsume(@PathVariable String cubeName) {
+//        streamingCoordinartorService.resumeConsumers(cubeName);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/remoteStoreComplete", method = { RequestMethod.POST }, produces = { "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse segmentRemoteStoreComplete(@RequestBody RemoteStoreCompleteRequest request) {
+//        Pair<Long, Long> segmentRange = new Pair<>(request.getSegmentStart(), request.getSegmentEnd());
+//        Node receiver = request.getReceiverNode();
+//        logger.info(
+//                "receive segment remote store complete request for cube:{}, segment:{}, try to find proper segment to build",
+//                request.getCubeName(), segmentRange);
+//        streamingCoordinartorService.onSegmentRemoteStoreComplete(request.getCubeName(), segmentRange, receiver);
+//        return new CoordinatorResponse();
+//    }
+//
+//    @RequestMapping(value = "/replicaSetLeaderChange", method = { RequestMethod.POST }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CoordinatorResponse replicaSetLeaderChange(@RequestBody ReplicaSetLeaderChangeRequest request) {
+//        logger.info("receive replicaSet leader change:" + request);
+//        streamingCoordinartorService.replicaSetLeaderChange(request.getReplicaSetID(), request.getNewLeader());
+//        return new CoordinatorResponse();
+//    }
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
index 935d273..3033eeb 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/StreamingV2Controller.java
@@ -18,70 +18,8 @@
 
 package org.apache.kylin.rest.controller;
 
-import java.io.IOException;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.HadoopUtil;
-import org.apache.kylin.common.util.JsonUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.dimension.TimeDerivedColumnType;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.ISourceAware;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.exception.ForbiddenException;
-import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.rest.exception.NotFoundException;
-import org.apache.kylin.rest.request.StreamingRequest;
-import org.apache.kylin.rest.request.StreamingRequestV2;
-import org.apache.kylin.rest.response.ResponseCode;
-import org.apache.kylin.rest.service.CubeService;
-import org.apache.kylin.rest.service.StreamingV2Service;
-import org.apache.kylin.rest.service.TableService;
-import org.apache.kylin.stream.core.model.CubeAssignment;
-import org.apache.kylin.stream.core.model.Node;
-import org.apache.kylin.stream.core.model.ReplicaSet;
-import org.apache.kylin.stream.core.model.stats.ClusterState;
-import org.apache.kylin.stream.core.model.stats.CubeRealTimeState;
-import org.apache.kylin.stream.core.model.stats.ReceiverStats;
-import org.apache.kylin.stream.core.source.Partition;
-import org.apache.kylin.stream.core.source.StreamingSourceConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.security.access.AccessDeniedException;
-import org.springframework.security.core.context.SecurityContextHolder;
 import org.springframework.stereotype.Controller;
-import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestMethod;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.ResponseBody;
-
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
 
 /**
  * StreamingController is defined as Restful API entrance for UI.
@@ -90,456 +28,456 @@ import com.google.common.collect.Sets;
 @Controller
 @RequestMapping(value = "/streaming_v2")
 public class StreamingV2Controller extends BasicController {
-    private static final Logger logger = LoggerFactory.getLogger(StreamingV2Controller.class);
-
-    @Autowired
-    private StreamingV2Service streamingService;
-
-    @Autowired
-    private CubeService cubeMgmtService;
-
-    @Autowired
-    @Qualifier("tableService")
-    private TableService tableService;
-
-    @RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
-    @ResponseBody
-    public List<StreamingSourceConfig> getStreamings(@RequestParam(value = "table", required = false) String table,
-            @RequestParam(value = "limit", required = false) Integer limit,
-            @RequestParam(value = "offset", required = false) Integer offset) {
-        try {
-            return streamingService.getStreamingConfigs(table, limit, offset);
-        } catch (IOException e) {
-            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
-        }
-    }
-
-    /**
-     *
-     * create Streaming Schema
-     * 
-     * @throws IOException
-     */
-    @RequestMapping(value = "", method = { RequestMethod.POST })
-    @ResponseBody
-    public StreamingRequestV2 saveStreamingConfig(@RequestBody StreamingRequestV2 streamingRequest) {
-        String project = streamingRequest.getProject();
-        TableDesc tableDesc = deserializeTableDesc(streamingRequest);
-        StreamingSourceConfig streamingSourceConfig = deserializeStreamingConfig(streamingRequest.getStreamingConfig());
-
-        validateInput(tableDesc, streamingSourceConfig);
-
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to saveStreamingConfig with table Identity {}", user, tableDesc.getIdentity());
-
-        boolean saveStreamingSuccess = false, saveTableSuccess = false;
-        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
-        ProjectInstance projectInstance = ProjectManager.getInstance(kylinConfig).getProject(project);
-
-        InternalErrorException shouldThrow = null;
-        try {
-            try {
-                tableDesc.setUuid(UUID.randomUUID().toString());
-                tableService.loadTableToProject(tableDesc, null, project);
-                saveTableSuccess = true;
-            } catch (IOException e) {
-                throw new BadRequestException("Failed to add streaming table.");
-            }
-            try {
-                streamingSourceConfig.setName(tableDesc.getIdentity());
-                streamingSourceConfig.setUuid(UUID.randomUUID().toString());
-                streamingService.createStreamingConfig(streamingSourceConfig, projectInstance);
-                saveStreamingSuccess = true;
-            } catch (IOException e) {
-                logger.error("Failed to save StreamingSourceConfig:" + e.getLocalizedMessage(), e);
-                throw new InternalErrorException("Failed to save StreamingSourceConfig: " + e.getLocalizedMessage());
-            }
-
-        } finally {
-            if (!saveTableSuccess || !saveStreamingSuccess) {
-                if (saveTableSuccess) {
-                    try {
-                        tableService.unloadHiveTable(tableDesc.getIdentity(), project);
-                    } catch (IOException e) {
-                        shouldThrow = new InternalErrorException(
-                                "Action failed and failed to rollback the create table " + e.getLocalizedMessage(), e);
-                    }
-                }
-                if (saveStreamingSuccess) {
-                    try {
-                        streamingService.dropStreamingConfig(streamingSourceConfig);
-                    } catch (IOException e) {
-                        shouldThrow = new InternalErrorException(
-                                "Action failed and failed to rollback the created streaming config: "
-                                        + e.getLocalizedMessage(),
-                                e);
-                    }
-                }
-            }
-        }
-
-        if (null != shouldThrow) {
-            throw shouldThrow;
-        }
-
-        streamingRequest.setSuccessful(true);
-        return streamingRequest;
-    }
-
-    private void validateInput(TableDesc tableDesc, StreamingSourceConfig streamingSourceConfig) {
-        if (StringUtils.isEmpty(tableDesc.getIdentity()) || StringUtils.isEmpty(streamingSourceConfig.getName())) {
-            logger.error("streamingSourceConfig name should not be empty.");
-            throw new BadRequestException("streamingSourceConfig name should not be empty.");
-        }
-
-        // validate the compatibility for input table schema and the underline hive table schema
-        if (tableDesc.getSourceType() == ISourceAware.ID_KAFKA_HIVE) {
-            List<FieldSchema> fields;
-            String db = tableDesc.getDatabase();
-            try {
-                HiveMetaStoreClient metaStoreClient = new HiveMetaStoreClient(new HiveConf());
-                fields = metaStoreClient.getFields(db, tableDesc.getName());
-                logger.info("Checking the {} in {}", tableDesc.getName(), db);
-            } catch (NoSuchObjectException noObjectException) {
-                logger.info("table not exist in hive meta store for table:" + tableDesc.getIdentity(),
-                        noObjectException);
-                throw new BadRequestException(
-                        "table doesn't exist in hive meta store for table:" + tableDesc.getIdentity(),
-                        ResponseCode.CODE_UNDEFINED, noObjectException);
-            } catch (Exception e) {
-                logger.error("error when get metadata from hive meta store for table:" + tableDesc.getIdentity(), e);
-                throw new BadRequestException("error when connect hive meta store", ResponseCode.CODE_UNDEFINED, e);
-            }
-            // check the data type compatibility for each column
-            Map<String, FieldSchema> fieldSchemaMap = Maps.newHashMap();
-            for (FieldSchema field : fields) {
-                fieldSchemaMap.put(field.getName().toUpperCase(Locale.ROOT), field);
-            }
-            List<String> incompatibleMsgs = Lists.newArrayList();
-            for (ColumnDesc columnDesc : tableDesc.getColumns()) {
-                FieldSchema fieldSchema = fieldSchemaMap.get(columnDesc.getName().toUpperCase(Locale.ROOT));
-                if (fieldSchema == null) {
-                    // Partition column cannot be fetched via Hive Metadata API.
-                    if (!TimeDerivedColumnType.isTimeDerivedColumn(columnDesc.getName())) {
-                        incompatibleMsgs.add("Column not exist in hive table:" + columnDesc.getName());
-                        continue;
-                    } else {
-                        logger.info("Column not exist in hive table: {}.", columnDesc.getName());
-                        continue;
-                    }
-                }
-                if (!checkHiveTableFieldCompatible(fieldSchema, columnDesc)) {
-                    String msg = String.format(Locale.ROOT,
-                            "column:%s defined in hive type:%s is incompatible with the column definition:%s",
-                            columnDesc.getName(), fieldSchema.getType(), columnDesc.getDatatype());
-                    incompatibleMsgs.add(msg);
-                }
-            }
-            if (!incompatibleMsgs.isEmpty()) {
-                logger.info("incompatible for hive and input table schema:{}", incompatibleMsgs);
-                throw new BadRequestException(
-                        "incompatible for hive schema and input table schema:" + incompatibleMsgs);
-            }
-        }
-    }
-
-    private static Map<String, Set<String>> COMPATIBLE_MAP = Maps.newHashMap();
-    static {
-        COMPATIBLE_MAP.put("float", Sets.newHashSet("double"));
-        COMPATIBLE_MAP.put("string", Sets.newHashSet("varchar", "char", "varchar(256)"));
-        COMPATIBLE_MAP.put("varchar", Sets.newHashSet("string", "char"));
-        COMPATIBLE_MAP.put("varchar(256)", Sets.newHashSet("string", "char", "varchar"));
-        COMPATIBLE_MAP.put("long", Sets.newHashSet("bigint", "int", "smallint", "integer"));
-        COMPATIBLE_MAP.put("bigint", Sets.newHashSet("long", "int", "smallint", "integer"));
-        COMPATIBLE_MAP.put("int", Sets.newHashSet("smallint", "integer"));
-    }
-
-    private boolean checkHiveTableFieldCompatible(FieldSchema fieldSchema, ColumnDesc columnDesc) {
-        DataType normalized = DataType.getType(columnDesc.getDatatype());
-        String normalizedDataType = normalized == null ? columnDesc.getDatatype() : normalized.toString();
-        if (fieldSchema.getType().equals(normalizedDataType)) {
-            return true;
-        }
-        Set<String> compatibleSet = COMPATIBLE_MAP.get(fieldSchema.getType());
-        if (compatibleSet != null && compatibleSet.contains(normalizedDataType)) {
-            return true;
-        }
-        return false;
-    }
-
-    @RequestMapping(value = "/updateConfig", method = { RequestMethod.PUT })
-    @ResponseBody
-    public StreamingRequest updateStreamingConfig(@RequestBody StreamingRequest streamingRequest)
-            throws JsonProcessingException {
-        StreamingSourceConfig streamingSourceConfig = deserializeStreamingConfig(streamingRequest.getStreamingConfig());
-
-        if (streamingSourceConfig == null) {
-            return streamingRequest;
-        }
-
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to updateStreamingConfig.", user);
-        try {
-            streamingSourceConfig = streamingService.updateStreamingConfig(streamingSourceConfig);
-        } catch (AccessDeniedException accessDeniedException) {
-            throw new ForbiddenException("You don't have right to update this StreamingSourceConfig.");
-        } catch (Exception e) {
-            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
-            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
-        }
-        streamingRequest.setSuccessful(true);
-
-        return streamingRequest;
-    }
-
-    @RequestMapping(value = "/{configName}", method = { RequestMethod.DELETE })
-    @ResponseBody
-    public void deleteConfig(@PathVariable String configName) throws IOException {
-        StreamingSourceConfig config = streamingService.getStreamingManagerV2().getConfig(configName);
-        if (null == config) {
-            throw new NotFoundException("StreamingSourceConfig with name " + configName + " not found..");
-        }
-
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to delete config: {}", user, configName);
-
-        try {
-            streamingService.dropStreamingConfig(config);
-        } catch (Exception e) {
-            logger.error(e.getLocalizedMessage(), e);
-            throw new InternalErrorException(
-                    "Failed to delete StreamingSourceConfig. " + " Caused by: " + e.getMessage(), e);
-        }
-    }
-
-    @RequestMapping(value = "/parserTemplate", method = { RequestMethod.GET })
-    @ResponseBody
-    public String getParserTemplate(@RequestParam(value = "sourceType") int sourceType,
-            @RequestParam(value = "streamingConfig") String streamingConfigStr) {
-        StreamingSourceConfig streamingSourceConfig = deserializeStreamingConfig(streamingConfigStr);
-        return streamingService.getParserTemplate(sourceType, streamingSourceConfig);
-    }
-
-    @RequestMapping(value = "/cubeAssignments", method = { RequestMethod.GET })
-    @ResponseBody
-    public List<CubeAssignment> getCubeAssignments(@RequestParam(value = "cube", required = false) String cube) {
-        CubeInstance cubeInstance = null;
-        if (cube != null) {
-            cubeInstance = cubeMgmtService.getCubeManager().getCube(cube);
-        }
-        return streamingService.getStreamingCubeAssignments(cubeInstance);
-    }
-
-    @RequestMapping(value = "/rsAssignments", method = { RequestMethod.GET })
-    @ResponseBody
-    public Map<Integer, Map<String, List<Partition>>> getReplicaSetAssignments(
-            @RequestParam(value = "replicaSetID", required = false) Integer replicaSetID) {
-        return streamingService.getStreamingReplicaSetAssignments(replicaSetID);
-    }
-
-    @RequestMapping(value = "/balance/recommend", method = { RequestMethod.GET })
-    @ResponseBody
-    public Map<Integer, Map<String, List<Partition>>> reBalanceRecommend() {
-        return streamingService.reBalancePlan();
-    }
-
-    @RequestMapping(value = "/balance", method = { RequestMethod.POST })
-    @ResponseBody
-    public void reBalance(@RequestBody String reBalancePlanStr) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to do reBalance.", user);
-        streamingService.reBalance(deserializeRebalancePlan(reBalancePlanStr));
-    }
-
-    private Map<Integer, Map<String, List<Partition>>> deserializeRebalancePlan(String reBalancePlanStr) {
-        TypeReference<Map<Integer, Map<String, List<Partition>>>> typeRef = new TypeReference<Map<Integer, Map<String, List<Partition>>>>() {
-        };
-        ObjectMapper mapper = new ObjectMapper();
-        try {
-            return mapper.readValue(reBalancePlanStr, typeRef);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @RequestMapping(value = "/cubeAssignments/{cubeName}", method = { RequestMethod.DELETE })
-    @ResponseBody
-    public void removeCubeAssignment(@PathVariable String cubeName) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to remove CubeAssignment {}", user, cubeName);
-        streamingService.removeCubeAssignment();
-    }
-
-    @RequestMapping(value = "/cubes", method = { RequestMethod.GET })
-    @ResponseBody
-    public List<String> getStreamingCubes() {
-        return streamingService.getStreamingCubes();
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/consumeState", method = { RequestMethod.GET })
-    @ResponseBody
-    public String getStreamingCubeConsumeState(@PathVariable String cubeName) {
-        return streamingService.getStreamingCubeConsumeState(cubeName).toString();
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/assign", method = { RequestMethod.PUT })
-    @ResponseBody
-    public void assignStreamingCube(@PathVariable String cubeName) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to assign cube {}", user, cubeName);
-        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
-        streamingService.assignCube(cube);
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/unAssign", method = { RequestMethod.PUT })
-    @ResponseBody
-    public void unAssignStreamingCube(@PathVariable String cubeName) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to unAssign cube {}", user, cubeName);
-        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
-        streamingService.unAssignCube(cube);
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/reAssign", method = { RequestMethod.POST })
-    @ResponseBody
-    public void reAssignStreamingCube(@PathVariable String cubeName, @RequestBody CubeAssignment newAssignment) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to reAssign cube {}", user, cubeName);
-        streamingService.reAssignCube(cubeName, newAssignment);
-    }
-
-    @RequestMapping(value = "/receivers", method = { RequestMethod.GET })
-    @ResponseBody
-    public List<Node> getStreamingReceivers() {
-        return streamingService.getReceivers();
-    }
-
-    @RequestMapping(value = "/receivers/{receiverID:.+}", method = { RequestMethod.DELETE })
-    @ResponseBody
-    public void removeStreamingReceiver(@PathVariable String receiverID) {
-        Node receiver = Node.fromNormalizeString(receiverID);
-        streamingService.removeReceiver(receiver);
-    }
-
-    @RequestMapping(value = "/replicaSet", method = { RequestMethod.POST })
-    @ResponseBody
-    public void createReplicaSet(@RequestBody ReplicaSet rs) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to create ReplicaSet {}", user, rs.getReplicaSetID());
-        streamingService.createReplicaSet(rs);
-    }
-
-    @RequestMapping(value = "/replicaSet/{replicaSetID}", method = { RequestMethod.DELETE })
-    @ResponseBody
-    public void removeReplicaSet(@PathVariable Integer replicaSetID) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to remove ReplicaSet {}", user, replicaSetID);
-        streamingService.removeReplicaSet(replicaSetID);
-    }
-
-    @RequestMapping(value = "/replicaSets", method = { RequestMethod.GET })
-    @ResponseBody
-    public List<ReplicaSet> getReplicaSets() {
-        return streamingService.getReplicaSets();
-    }
-
-    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.PUT })
-    @ResponseBody
-    public void addNodeToReplicaSet(@PathVariable Integer replicaSetID, @PathVariable String nodeID) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to add Node {} To ReplicaSet {}", user, nodeID, replicaSetID);
-        streamingService.addNodeToReplicaSet(replicaSetID, nodeID);
-    }
-
-    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.DELETE })
-    @ResponseBody
-    public void removeNodeFromReplicaSet(@PathVariable Integer replicaSetID, @PathVariable String nodeID) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to remove Node {} from ReplicaSet {}", user, nodeID, replicaSetID);
-        streamingService.removeNodeFromReplicaSet(replicaSetID, nodeID);
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/suspendConsume", method = { RequestMethod.PUT })
-    @ResponseBody
-    public void pauseCubeConsume(@PathVariable String cubeName) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to pause Consumers for cube {}", user, cubeName);
-        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
-        streamingService.pauseConsumers(cube);
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/resumeConsume", method = { RequestMethod.PUT })
-    @ResponseBody
-    public void resumeCubeConsume(@PathVariable String cubeName) {
-        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
-        logger.info("{} try to resume Consumers for cube {}", user, cubeName);
-        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
-        streamingService.resumeConsumers(cube);
-    }
-
-    @RequestMapping(value = "/cubes/{cubeName}/stats", method = { RequestMethod.GET })
-    @ResponseBody
-    public CubeRealTimeState getCubeRealTimeState(@PathVariable String cubeName) {
-        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
-        return streamingService.getCubeRealTimeState(cube);
-    }
-
-    @RequestMapping(value = "/receivers/{receiverID:.+}/stats", method = { RequestMethod.GET })
-    @ResponseBody
-    public ReceiverStats getReceiverStats(@PathVariable String receiverID) {
-        Node receiver = Node.fromNormalizeString(receiverID);
-        return streamingService.getReceiverStats(receiver);
-    }
-
-    @RequestMapping(value = "/cluster/state", method = { RequestMethod.GET })
-    @ResponseBody
-    public ClusterState getClusterState() {
-        return streamingService.getClusterState();
-    }
-
-    private TableDesc deserializeTableDesc(StreamingRequestV2 streamingRequest) {
-        TableDesc desc = null;
-        String db = KylinConfig.getInstanceFromEnv().getHiveDatabaseLambdaCube();
-        try {
-            logger.debug("Saving TableDesc " + streamingRequest.getTableData());
-            desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class);
-        } catch (JsonParseException e) {
-            logger.error("The TableDesc definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (JsonMappingException e) {
-            logger.error("The data TableDesc definition is invalid.", e);
-            updateRequest(streamingRequest, false, e.getMessage());
-        } catch (IOException e) {
-            logger.error("Failed to deal with the request.", e);
-            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
-        }
-
-        Preconditions.checkNotNull(desc, "Failed to deserialize from TableDesc definition");
-        String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName());
-        desc.setName(dbTable[1]);
-        desc.setDatabase(db);
-        desc.getIdentity();
-        return desc;
-    }
-
-    private StreamingSourceConfig deserializeStreamingConfig(String streamingConfigStr) {
-        try {
-            logger.debug("Saving StreamingSourceConfig " + streamingConfigStr);
-            return JsonUtil.readValue(streamingConfigStr, StreamingSourceConfig.class);
-        } catch (Exception e) {
-            logger.error("The StreamingSourceConfig definition is invalid.", e);
-            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
-        }
-    }
-
-    private void updateRequest(StreamingRequestV2 request, boolean success, String message) {
-        request.setSuccessful(success);
-        request.setMessage(message);
-    }
-
-    public void setCubeService(CubeService cubeService) {
-        this.cubeMgmtService = cubeService;
-    }
+//    private static final Logger logger = LoggerFactory.getLogger(StreamingV2Controller.class);
+//
+//    @Autowired
+//    private StreamingV2Service streamingService;
+//
+//    @Autowired
+//    private CubeService cubeMgmtService;
+//
+//    @Autowired
+//    @Qualifier("tableService")
+//    private TableService tableService;
+//
+//    @RequestMapping(value = "/getConfig", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public List<StreamingSourceConfig> getStreamings(@RequestParam(value = "table", required = false) String table,
+//            @RequestParam(value = "limit", required = false) Integer limit,
+//            @RequestParam(value = "offset", required = false) Integer offset) {
+//        try {
+//            return streamingService.getStreamingConfigs(table, limit, offset);
+//        } catch (IOException e) {
+//            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
+//        }
+//    }
+//
+//    /**
+//     *
+//     * create Streaming Schema
+//     *
+//     * @throws IOException
+//     */
+//    @RequestMapping(value = "", method = { RequestMethod.POST })
+//    @ResponseBody
+//    public StreamingRequestV2 saveStreamingConfig(@RequestBody StreamingRequestV2 streamingRequest) {
+//        String project = streamingRequest.getProject();
+//        TableDesc tableDesc = deserializeTableDesc(streamingRequest);
+//        StreamingSourceConfig streamingSourceConfig = deserializeStreamingConfig(streamingRequest.getStreamingConfig());
+//
+//        validateInput(tableDesc, streamingSourceConfig);
+//
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to saveStreamingConfig with table Identity {}", user, tableDesc.getIdentity());
+//
+//        boolean saveStreamingSuccess = false, saveTableSuccess = false;
+//        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
+//        ProjectInstance projectInstance = ProjectManager.getInstance(kylinConfig).getProject(project);
+//
+//        InternalErrorException shouldThrow = null;
+//        try {
+//            try {
+//                tableDesc.setUuid(UUID.randomUUID().toString());
+//                tableService.loadTableToProject(tableDesc, null, project);
+//                saveTableSuccess = true;
+//            } catch (IOException e) {
+//                throw new BadRequestException("Failed to add streaming table.");
+//            }
+//            try {
+//                streamingSourceConfig.setName(tableDesc.getIdentity());
+//                streamingSourceConfig.setUuid(UUID.randomUUID().toString());
+//                streamingService.createStreamingConfig(streamingSourceConfig, projectInstance);
+//                saveStreamingSuccess = true;
+//            } catch (IOException e) {
+//                logger.error("Failed to save StreamingSourceConfig:" + e.getLocalizedMessage(), e);
+//                throw new InternalErrorException("Failed to save StreamingSourceConfig: " + e.getLocalizedMessage());
+//            }
+//
+//        } finally {
+//            if (!saveTableSuccess || !saveStreamingSuccess) {
+//                if (saveTableSuccess) {
+//                    try {
+//                        tableService.unloadHiveTable(tableDesc.getIdentity(), project);
+//                    } catch (IOException e) {
+//                        shouldThrow = new InternalErrorException(
+//                                "Action failed and failed to rollback the create table " + e.getLocalizedMessage(), e);
+//                    }
+//                }
+//                if (saveStreamingSuccess) {
+//                    try {
+//                        streamingService.dropStreamingConfig(streamingSourceConfig);
+//                    } catch (IOException e) {
+//                        shouldThrow = new InternalErrorException(
+//                                "Action failed and failed to rollback the created streaming config: "
+//                                        + e.getLocalizedMessage(),
+//                                e);
+//                    }
+//                }
+//            }
+//        }
+//
+//        if (null != shouldThrow) {
+//            throw shouldThrow;
+//        }
+//
+//        streamingRequest.setSuccessful(true);
+//        return streamingRequest;
+//    }
+//
+//    private void validateInput(TableDesc tableDesc, StreamingSourceConfig streamingSourceConfig) {
+//        if (StringUtils.isEmpty(tableDesc.getIdentity()) || StringUtils.isEmpty(streamingSourceConfig.getName())) {
+//            logger.error("streamingSourceConfig name should not be empty.");
+//            throw new BadRequestException("streamingSourceConfig name should not be empty.");
+//        }
+//
+//        // validate the compatibility for input table schema and the underline hive table schema
+//        if (tableDesc.getSourceType() == ISourceAware.ID_KAFKA_HIVE) {
+//            List<FieldSchema> fields;
+//            String db = tableDesc.getDatabase();
+//            try {
+//                HiveMetaStoreClient metaStoreClient = new HiveMetaStoreClient(new HiveConf());
+//                fields = metaStoreClient.getFields(db, tableDesc.getName());
+//                logger.info("Checking the {} in {}", tableDesc.getName(), db);
+//            } catch (NoSuchObjectException noObjectException) {
+//                logger.info("table not exist in hive meta store for table:" + tableDesc.getIdentity(),
+//                        noObjectException);
+//                throw new BadRequestException(
+//                        "table doesn't exist in hive meta store for table:" + tableDesc.getIdentity(),
+//                        ResponseCode.CODE_UNDEFINED, noObjectException);
+//            } catch (Exception e) {
+//                logger.error("error when get metadata from hive meta store for table:" + tableDesc.getIdentity(), e);
+//                throw new BadRequestException("error when connect hive meta store", ResponseCode.CODE_UNDEFINED, e);
+//            }
+//            // check the data type compatibility for each column
+//            Map<String, FieldSchema> fieldSchemaMap = Maps.newHashMap();
+//            for (FieldSchema field : fields) {
+//                fieldSchemaMap.put(field.getName().toUpperCase(Locale.ROOT), field);
+//            }
+//            List<String> incompatibleMsgs = Lists.newArrayList();
+//            for (ColumnDesc columnDesc : tableDesc.getColumns()) {
+//                FieldSchema fieldSchema = fieldSchemaMap.get(columnDesc.getName().toUpperCase(Locale.ROOT));
+//                if (fieldSchema == null) {
+//                    // Partition column cannot be fetched via Hive Metadata API.
+//                    if (!TimeDerivedColumnType.isTimeDerivedColumn(columnDesc.getName())) {
+//                        incompatibleMsgs.add("Column not exist in hive table:" + columnDesc.getName());
+//                        continue;
+//                    } else {
+//                        logger.info("Column not exist in hive table: {}.", columnDesc.getName());
+//                        continue;
+//                    }
+//                }
+//                if (!checkHiveTableFieldCompatible(fieldSchema, columnDesc)) {
+//                    String msg = String.format(Locale.ROOT,
+//                            "column:%s defined in hive type:%s is incompatible with the column definition:%s",
+//                            columnDesc.getName(), fieldSchema.getType(), columnDesc.getDatatype());
+//                    incompatibleMsgs.add(msg);
+//                }
+//            }
+//            if (!incompatibleMsgs.isEmpty()) {
+//                logger.info("incompatible for hive and input table schema:{}", incompatibleMsgs);
+//                throw new BadRequestException(
+//                        "incompatible for hive schema and input table schema:" + incompatibleMsgs);
+//            }
+//        }
+//    }
+//
+//    private static Map<String, Set<String>> COMPATIBLE_MAP = Maps.newHashMap();
+//    static {
+//        COMPATIBLE_MAP.put("float", Sets.newHashSet("double"));
+//        COMPATIBLE_MAP.put("string", Sets.newHashSet("varchar", "char", "varchar(256)"));
+//        COMPATIBLE_MAP.put("varchar", Sets.newHashSet("string", "char"));
+//        COMPATIBLE_MAP.put("varchar(256)", Sets.newHashSet("string", "char", "varchar"));
+//        COMPATIBLE_MAP.put("long", Sets.newHashSet("bigint", "int", "smallint", "integer"));
+//        COMPATIBLE_MAP.put("bigint", Sets.newHashSet("long", "int", "smallint", "integer"));
+//        COMPATIBLE_MAP.put("int", Sets.newHashSet("smallint", "integer"));
+//    }
+//
+//    private boolean checkHiveTableFieldCompatible(FieldSchema fieldSchema, ColumnDesc columnDesc) {
+//        DataType normalized = DataType.getType(columnDesc.getDatatype());
+//        String normalizedDataType = normalized == null ? columnDesc.getDatatype() : normalized.toString();
+//        if (fieldSchema.getType().equals(normalizedDataType)) {
+//            return true;
+//        }
+//        Set<String> compatibleSet = COMPATIBLE_MAP.get(fieldSchema.getType());
+//        if (compatibleSet != null && compatibleSet.contains(normalizedDataType)) {
+//            return true;
+//        }
+//        return false;
+//    }
+//
+//    @RequestMapping(value = "/updateConfig", method = { RequestMethod.PUT })
+//    @ResponseBody
+//    public StreamingRequest updateStreamingConfig(@RequestBody StreamingRequest streamingRequest)
+//            throws JsonProcessingException {
+//        StreamingSourceConfig streamingSourceConfig = deserializeStreamingConfig(streamingRequest.getStreamingConfig());
+//
+//        if (streamingSourceConfig == null) {
+//            return streamingRequest;
+//        }
+//
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to updateStreamingConfig.", user);
+//        try {
+//            streamingSourceConfig = streamingService.updateStreamingConfig(streamingSourceConfig);
+//        } catch (AccessDeniedException accessDeniedException) {
+//            throw new ForbiddenException("You don't have right to update this StreamingSourceConfig.");
+//        } catch (Exception e) {
+//            logger.error("Failed to deal with the request:" + e.getLocalizedMessage(), e);
+//            throw new InternalErrorException("Failed to deal with the request: " + e.getLocalizedMessage());
+//        }
+//        streamingRequest.setSuccessful(true);
+//
+//        return streamingRequest;
+//    }
+//
+//    @RequestMapping(value = "/{configName}", method = { RequestMethod.DELETE })
+//    @ResponseBody
+//    public void deleteConfig(@PathVariable String configName) throws IOException {
+//        StreamingSourceConfig config = streamingService.getStreamingManagerV2().getConfig(configName);
+//        if (null == config) {
+//            throw new NotFoundException("StreamingSourceConfig with name " + configName + " not found..");
+//        }
+//
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to delete config: {}", user, configName);
+//
+//        try {
+//            streamingService.dropStreamingConfig(config);
+//        } catch (Exception e) {
+//            logger.error(e.getLocalizedMessage(), e);
+//            throw new InternalErrorException(
+//                    "Failed to delete StreamingSourceConfig. " + " Caused by: " + e.getMessage(), e);
+//        }
+//    }
+//
+//    @RequestMapping(value = "/parserTemplate", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public String getParserTemplate(@RequestParam(value = "sourceType") int sourceType,
+//            @RequestParam(value = "streamingConfig") String streamingConfigStr) {
+//        StreamingSourceConfig streamingSourceConfig = deserializeStreamingConfig(streamingConfigStr);
+//        return streamingService.getParserTemplate(sourceType, streamingSourceConfig);
+//    }
+//
+//    @RequestMapping(value = "/cubeAssignments", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public List<CubeAssignment> getCubeAssignments(@RequestParam(value = "cube", required = false) String cube) {
+//        CubeInstance cubeInstance = null;
+//        if (cube != null) {
+//            cubeInstance = cubeMgmtService.getCubeManager().getCube(cube);
+//        }
+//        return streamingService.getStreamingCubeAssignments(cubeInstance);
+//    }
+//
+//    @RequestMapping(value = "/rsAssignments", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public Map<Integer, Map<String, List<Partition>>> getReplicaSetAssignments(
+//            @RequestParam(value = "replicaSetID", required = false) Integer replicaSetID) {
+//        return streamingService.getStreamingReplicaSetAssignments(replicaSetID);
+//    }
+//
+//    @RequestMapping(value = "/balance/recommend", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public Map<Integer, Map<String, List<Partition>>> reBalanceRecommend() {
+//        return streamingService.reBalancePlan();
+//    }
+//
+//    @RequestMapping(value = "/balance", method = { RequestMethod.POST })
+//    @ResponseBody
+//    public void reBalance(@RequestBody String reBalancePlanStr) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to do reBalance.", user);
+//        streamingService.reBalance(deserializeRebalancePlan(reBalancePlanStr));
+//    }
+//
+//    private Map<Integer, Map<String, List<Partition>>> deserializeRebalancePlan(String reBalancePlanStr) {
+//        TypeReference<Map<Integer, Map<String, List<Partition>>>> typeRef = new TypeReference<Map<Integer, Map<String, List<Partition>>>>() {
+//        };
+//        ObjectMapper mapper = new ObjectMapper();
+//        try {
+//            return mapper.readValue(reBalancePlanStr, typeRef);
+//        } catch (IOException e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+//
+//    @RequestMapping(value = "/cubeAssignments/{cubeName}", method = { RequestMethod.DELETE })
+//    @ResponseBody
+//    public void removeCubeAssignment(@PathVariable String cubeName) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to remove CubeAssignment {}", user, cubeName);
+//        streamingService.removeCubeAssignment();
+//    }
+//
+//    @RequestMapping(value = "/cubes", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public List<String> getStreamingCubes() {
+//        return streamingService.getStreamingCubes();
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/consumeState", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public String getStreamingCubeConsumeState(@PathVariable String cubeName) {
+//        return streamingService.getStreamingCubeConsumeState(cubeName).toString();
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/assign", method = { RequestMethod.PUT })
+//    @ResponseBody
+//    public void assignStreamingCube(@PathVariable String cubeName) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to assign cube {}", user, cubeName);
+//        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
+//        streamingService.assignCube(cube);
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/unAssign", method = { RequestMethod.PUT })
+//    @ResponseBody
+//    public void unAssignStreamingCube(@PathVariable String cubeName) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to unAssign cube {}", user, cubeName);
+//        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
+//        streamingService.unAssignCube(cube);
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/reAssign", method = { RequestMethod.POST })
+//    @ResponseBody
+//    public void reAssignStreamingCube(@PathVariable String cubeName, @RequestBody CubeAssignment newAssignment) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to reAssign cube {}", user, cubeName);
+//        streamingService.reAssignCube(cubeName, newAssignment);
+//    }
+//
+//    @RequestMapping(value = "/receivers", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public List<Node> getStreamingReceivers() {
+//        return streamingService.getReceivers();
+//    }
+//
+//    @RequestMapping(value = "/receivers/{receiverID:.+}", method = { RequestMethod.DELETE })
+//    @ResponseBody
+//    public void removeStreamingReceiver(@PathVariable String receiverID) {
+//        Node receiver = Node.fromNormalizeString(receiverID);
+//        streamingService.removeReceiver(receiver);
+//    }
+//
+//    @RequestMapping(value = "/replicaSet", method = { RequestMethod.POST })
+//    @ResponseBody
+//    public void createReplicaSet(@RequestBody ReplicaSet rs) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to create ReplicaSet {}", user, rs.getReplicaSetID());
+//        streamingService.createReplicaSet(rs);
+//    }
+//
+//    @RequestMapping(value = "/replicaSet/{replicaSetID}", method = { RequestMethod.DELETE })
+//    @ResponseBody
+//    public void removeReplicaSet(@PathVariable Integer replicaSetID) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to remove ReplicaSet {}", user, replicaSetID);
+//        streamingService.removeReplicaSet(replicaSetID);
+//    }
+//
+//    @RequestMapping(value = "/replicaSets", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public List<ReplicaSet> getReplicaSets() {
+//        return streamingService.getReplicaSets();
+//    }
+//
+//    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.PUT })
+//    @ResponseBody
+//    public void addNodeToReplicaSet(@PathVariable Integer replicaSetID, @PathVariable String nodeID) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to add Node {} To ReplicaSet {}", user, nodeID, replicaSetID);
+//        streamingService.addNodeToReplicaSet(replicaSetID, nodeID);
+//    }
+//
+//    @RequestMapping(value = "/replicaSet/{replicaSetID}/{nodeID:.+}", method = { RequestMethod.DELETE })
+//    @ResponseBody
+//    public void removeNodeFromReplicaSet(@PathVariable Integer replicaSetID, @PathVariable String nodeID) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to remove Node {} from ReplicaSet {}", user, nodeID, replicaSetID);
+//        streamingService.removeNodeFromReplicaSet(replicaSetID, nodeID);
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/suspendConsume", method = { RequestMethod.PUT })
+//    @ResponseBody
+//    public void pauseCubeConsume(@PathVariable String cubeName) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to pause Consumers for cube {}", user, cubeName);
+//        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
+//        streamingService.pauseConsumers(cube);
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/resumeConsume", method = { RequestMethod.PUT })
+//    @ResponseBody
+//    public void resumeCubeConsume(@PathVariable String cubeName) {
+//        final String user = SecurityContextHolder.getContext().getAuthentication().getName();
+//        logger.info("{} try to resume Consumers for cube {}", user, cubeName);
+//        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
+//        streamingService.resumeConsumers(cube);
+//    }
+//
+//    @RequestMapping(value = "/cubes/{cubeName}/stats", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public CubeRealTimeState getCubeRealTimeState(@PathVariable String cubeName) {
+//        CubeInstance cube = cubeMgmtService.getCubeManager().getCube(cubeName);
+//        return streamingService.getCubeRealTimeState(cube);
+//    }
+//
+//    @RequestMapping(value = "/receivers/{receiverID:.+}/stats", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public ReceiverStats getReceiverStats(@PathVariable String receiverID) {
+//        Node receiver = Node.fromNormalizeString(receiverID);
+//        return streamingService.getReceiverStats(receiver);
+//    }
+//
+//    @RequestMapping(value = "/cluster/state", method = { RequestMethod.GET })
+//    @ResponseBody
+//    public ClusterState getClusterState() {
+//        return streamingService.getClusterState();
+//    }
+//
+//    private TableDesc deserializeTableDesc(StreamingRequestV2 streamingRequest) {
+//        TableDesc desc = null;
+//        String db = KylinConfig.getInstanceFromEnv().getHiveDatabaseLambdaCube();
+//        try {
+//            logger.debug("Saving TableDesc " + streamingRequest.getTableData());
+//            desc = JsonUtil.readValue(streamingRequest.getTableData(), TableDesc.class);
+//        } catch (JsonParseException e) {
+//            logger.error("The TableDesc definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (JsonMappingException e) {
+//            logger.error("The data TableDesc definition is invalid.", e);
+//            updateRequest(streamingRequest, false, e.getMessage());
+//        } catch (IOException e) {
+//            logger.error("Failed to deal with the request.", e);
+//            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+//        }
+//
+//        Preconditions.checkNotNull(desc, "Failed to deserialize from TableDesc definition");
+//        String[] dbTable = HadoopUtil.parseHiveTableName(desc.getName());
+//        desc.setName(dbTable[1]);
+//        desc.setDatabase(db);
+//        desc.getIdentity();
+//        return desc;
+//    }
+//
+//    private StreamingSourceConfig deserializeStreamingConfig(String streamingConfigStr) {
+//        try {
+//            logger.debug("Saving StreamingSourceConfig " + streamingConfigStr);
+//            return JsonUtil.readValue(streamingConfigStr, StreamingSourceConfig.class);
+//        } catch (Exception e) {
+//            logger.error("The StreamingSourceConfig definition is invalid.", e);
+//            throw new InternalErrorException("Failed to deal with the request:" + e.getMessage(), e);
+//        }
+//    }
+//
+//    private void updateRequest(StreamingRequestV2 request, boolean success, String message) {
+//        request.setSuccessful(success);
+//        request.setMessage(message);
+//    }
+//
+//    public void setCubeService(CubeService cubeService) {
+//        this.cubeMgmtService = cubeService;
+//    }
 
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
index a71de26..fc9ff82 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/controller/TableController.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
@@ -34,7 +33,6 @@ import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableExtDesc;
 import org.apache.kylin.rest.exception.InternalErrorException;
 import org.apache.kylin.rest.exception.NotFoundException;
-import org.apache.kylin.rest.request.CardinalityRequest;
 import org.apache.kylin.rest.request.HiveTableRequest;
 import org.apache.kylin.rest.response.TableSnapshotResponse;
 import org.apache.kylin.rest.service.TableACLService;
@@ -127,9 +125,9 @@ public class TableController extends BasicController {
             String[] unloaded = new String[allTables.size()];
             allTables.toArray(unloaded);
             result.put("result.unloaded", unloaded);
-            if (request.isCalculate()) {
-                tableService.calculateCardinalityIfNotPresent(loaded, submitter, project);
-            }
+//            if (request.isCalculate()) {
+//                tableService.calculateCardinalityIfNotPresent(loaded, submitter, project);
+//            }
         } catch (Throwable e) {
             logger.error("Failed to load Hive Table", e);
             throw new InternalErrorException(e.getLocalizedMessage(), e);
@@ -168,23 +166,23 @@ public class TableController extends BasicController {
      * @return Table metadata array
      * @throws IOException
      */
-    @RequestMapping(value = "/{project}/{tableNames}/cardinality", method = { RequestMethod.PUT }, produces = {
-            "application/json" })
-    @ResponseBody
-    public CardinalityRequest generateCardinality(@PathVariable String tableNames,
-            @RequestBody CardinalityRequest request, @PathVariable String project) throws Exception {
-        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
-        String[] tables = StringUtil.splitByComma(tableNames);
-        try {
-            for (String table : tables) {
-                tableService.calculateCardinality(table.trim().toUpperCase(Locale.ROOT), submitter, project);
-            }
-        } catch (IOException e) {
-            logger.error("Failed to calculate cardinality", e);
-            throw new InternalErrorException(e.getLocalizedMessage(), e);
-        }
-        return request;
-    }
+//    @RequestMapping(value = "/{project}/{tableNames}/cardinality", method = { RequestMethod.PUT }, produces = {
+//            "application/json" })
+//    @ResponseBody
+//    public CardinalityRequest generateCardinality(@PathVariable String tableNames,
+//            @RequestBody CardinalityRequest request, @PathVariable String project) throws Exception {
+//        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
+//        String[] tables = StringUtil.splitByComma(tableNames);
+//        try {
+//            for (String table : tables) {
+//                tableService.calculateCardinality(table.trim().toUpperCase(Locale.ROOT), submitter, project);
+//            }
+//        } catch (IOException e) {
+//            logger.error("Failed to calculate cardinality", e);
+//            throw new InternalErrorException(e.getLocalizedMessage(), e);
+//        }
+//        return request;
+//    }
 
     /**
      * Show all databases in Hive
@@ -222,26 +220,6 @@ public class TableController extends BasicController {
         }
     }
 
-    @RequestMapping(value = "/{project}/{tableName}/{snapshotID}/snapshotLocalCache", method = { RequestMethod.PUT })
-    @ResponseBody
-    public void updateSnapshotLocalCache(@PathVariable final String project, @PathVariable final String tableName,
-            @PathVariable final String snapshotID) {
-        tableService.updateSnapshotLocalCache(project, tableName, snapshotID);
-    }
-
-    @RequestMapping(value = "/{tableName}/{snapshotID}/snapshotLocalCache/state", method = { RequestMethod.GET })
-    @ResponseBody
-    public String getSnapshotLocalCacheState(@PathVariable final String tableName,
-            @PathVariable final String snapshotID) {
-        return tableService.getSnapshotLocalCacheState(tableName, snapshotID);
-    }
-
-    @RequestMapping(value = "/{tableName}/{snapshotID}/snapshotLocalCache", method = { RequestMethod.DELETE })
-    @ResponseBody
-    public void removeSnapshotLocalCache(@PathVariable final String tableName, @PathVariable final String snapshotID) {
-        tableService.removeSnapshotLocalCache(tableName, snapshotID);
-    }
-
     @RequestMapping(value = "/{project}/{tableName}/snapshots", method = { RequestMethod.GET })
     @ResponseBody
     public List<TableSnapshotResponse> getTableSnapshots(@PathVariable final String project,
diff --git a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
index a5f98e8..e4016da 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/job/StorageCleanJobHbaseUtil.java
@@ -21,7 +21,6 @@ package org.apache.kylin.rest.job;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -29,7 +28,6 @@ import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import com.google.common.collect.Lists;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -37,8 +35,6 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.lookup.ExtTableSnapshotInfo;
-import org.apache.kylin.dict.lookup.ExtTableSnapshotInfoManager;
 import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,14 +68,9 @@ public class StorageCleanJobHbaseUtil {
 
         List<String> allTablesNeedToBeDropped = new ArrayList<String>();
 
-        boolean hasExtLookupTable = false;
         for (HTableDescriptor desc : tableDescriptors) {
             String host = desc.getValue(IRealizationConstants.HTableTag);
             if (config.getMetadataUrlPrefix().equalsIgnoreCase(host)) {
-                // check if there are hbase lookup table
-                if (desc.getTableName().getNameAsString().contains(IRealizationConstants.LookupHbaseStorageLocationPrefix)) {
-                    hasExtLookupTable = true;
-                }
                 //only take care htables that belongs to self, and created more than 2 days
                 allTablesNeedToBeDropped.add(desc.getTableName().getNameAsString());
             }
@@ -104,11 +95,6 @@ public class StorageCleanJobHbaseUtil {
         
         logger.info(allTablesNeedToBeDropped.size() + " HTable(s) to clean up");
 
-        if (hasExtLookupTable) {
-            List<String> useExtLookupTables = getAllUsedExtLookupTables();
-            logger.info("Exclude tables:{}, as they are referred by snapshots.", useExtLookupTables);
-            allTablesNeedToBeDropped.removeAll(useExtLookupTables);
-        }
         if (delete) {
             // drop tables
             logger.info("Use {} threads to drop unused hbase tables", threadsNum);
@@ -137,27 +123,6 @@ public class StorageCleanJobHbaseUtil {
         return allTablesNeedToBeDropped;
     }
 
-    private static List<String> getAllUsedExtLookupTables() throws IOException {
-        List<String> result = Lists.newArrayList();
-        KylinConfig config = KylinConfig.getInstanceFromEnv();
-        final Set<String> activeSnapshotSet = ExtTableSnapshotInfoManager.getInstance(config).getAllExtSnapshotResPaths();
-
-        for (String extSnapshotResource : activeSnapshotSet) {
-            try {
-                ExtTableSnapshotInfo extTableSnapshot = ExtTableSnapshotInfoManager.getInstance(config).getSnapshot(
-                        extSnapshotResource);
-                if (extTableSnapshot != null) {
-                    if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(extTableSnapshot.getStorageType())) {
-                        result.add(extTableSnapshot.getStorageLocationIdentifier());
-                    }
-                }
-            } catch (Exception e) {
-                logger.error("error fetch ext table snapshot:" + extSnapshotResource, e);
-            }
-        }
-        return result;
-    }
-
     static class DeleteHTableRunnable implements Callable {
         HBaseAdmin hbaseAdmin;
         String htableName;
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
index 9ac2602..263526c 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/BasicService.java
@@ -18,8 +18,6 @@
 
 package org.apache.kylin.rest.service;
 
-import java.io.IOException;
-
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.cube.CubeDescManager;
 import org.apache.kylin.cube.CubeManager;
@@ -32,7 +30,6 @@ import org.apache.kylin.metadata.model.DataModelManager;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.metadata.streaming.StreamingManager;
 import org.apache.kylin.metrics.MetricsManager;
-import org.apache.kylin.source.kafka.KafkaConfigManager;
 import org.apache.kylin.storage.hybrid.HybridManager;
 
 public abstract class BasicService {
@@ -63,9 +60,9 @@ public abstract class BasicService {
         return StreamingManager.getInstance(getConfig());
     }
 
-    public KafkaConfigManager getKafkaManager() throws IOException {
-        return KafkaConfigManager.getInstance(getConfig());
-    }
+//    public KafkaConfigManager getKafkaManager() throws IOException {
+//        return KafkaConfigManager.getInstance(getConfig());
+//    }
 
     public CubeDescManager getCubeDescManager() {
         return CubeDescManager.getInstance(getConfig());
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 7db7792..c3d60b7 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -28,14 +28,11 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.lock.DistributedLock;
 import org.apache.kylin.common.persistence.RootPersistentEntity;
 import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.HadoopUtil;
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
@@ -66,7 +63,6 @@ import org.apache.kylin.metadata.model.IStorageAware;
 import org.apache.kylin.metadata.model.MeasureDesc;
 import org.apache.kylin.metadata.model.SegmentRange;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.metadata.model.Segments;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
@@ -90,12 +86,7 @@ import org.apache.kylin.rest.response.HBaseResponse;
 import org.apache.kylin.rest.response.MetricsResponse;
 import org.apache.kylin.rest.util.AclEvaluate;
 import org.apache.kylin.rest.util.ValidateUtil;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.util.StorageCleanUtil;
 import org.apache.kylin.storage.hybrid.HybridInstance;
-import org.apache.kylin.stream.coordinator.StreamMetadataStoreFactory;
-import org.apache.kylin.stream.coordinator.client.CoordinatorClient;
-import org.apache.kylin.stream.coordinator.client.CoordinatorClientFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
@@ -428,23 +419,23 @@ public class CubeService extends BasicService implements InitializingBean {
             CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.DISABLED);
             cubeStatusUpdated = true;
             // for streaming cube.
-            if (isStreamingCube) {
-                //drop not ready segments
-                CubeSegment[] buildingSegments = new CubeSegment[cubeInstance.getBuildingSegments().size()];
-                Segments segments = cubeInstance.getBuildingSegments();
-                if (!CollectionUtils.isEmpty(segments)) {
-                    for (int i = 0; i < segments.size(); i++) {
-                        buildingSegments[i] = (CubeSegment) segments.get(i);
-                    }
-                    getCubeManager().dropOptmizingSegments(cubeInstance, buildingSegments);
-                }
-                //unAssign cube
-                getStreamingCoordinator().unAssignCube(cubeName);
-
-                //discard jobs
-                releaseAllJobs(cubeInstance);
-
-            }
+//            if (isStreamingCube) {
+//                //drop not ready segments
+//                CubeSegment[] buildingSegments = new CubeSegment[cubeInstance.getBuildingSegments().size()];
+//                Segments segments = cubeInstance.getBuildingSegments();
+//                if (!CollectionUtils.isEmpty(segments)) {
+//                    for (int i = 0; i < segments.size(); i++) {
+//                        buildingSegments[i] = (CubeSegment) segments.get(i);
+//                    }
+//                    getCubeManager().dropOptmizingSegments(cubeInstance, buildingSegments);
+//                }
+//                //unAssign cube
+//                getStreamingCoordinator().unAssignCube(cubeName);
+//
+//                //discard jobs
+//                releaseAllJobs(cubeInstance);
+//
+//            }
             return cubeInstance;
         } catch (Exception e) {
             cube.setStatus(ostatus);
@@ -492,9 +483,9 @@ public class CubeService extends BasicService implements InitializingBean {
             CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
             cubeStatusUpdated = true;
             // for streaming cube.
-            if (cube.getDescriptor().isStreamingCube()) {
-                getStreamingCoordinator().assignCube(cube.getName());
-            }
+//            if (cube.getDescriptor().isStreamingCube()) {
+//                getStreamingCoordinator().assignCube(cube.getName());
+//            }
             return cubeInstance;
         } catch (Exception e) {
             cube.setStatus(ostatus);
@@ -507,9 +498,9 @@ public class CubeService extends BasicService implements InitializingBean {
         }
     }
 
-    private CoordinatorClient getStreamingCoordinator() {
-        return CoordinatorClientFactory.createCoordinatorClient(StreamMetadataStoreFactory.getStreamMetaDataStore());
-    }
+//    private CoordinatorClient getStreamingCoordinator() {
+//        return CoordinatorClientFactory.createCoordinatorClient(StreamMetadataStoreFactory.getStreamMetaDataStore());
+//    }
 
     public MetricsResponse calculateMetrics(MetricsRequest request) {
         List<CubeInstance> cubes = this.getCubeManager().listAllCubes();
@@ -652,8 +643,8 @@ public class CubeService extends BasicService implements InitializingBean {
                         seg.getLastBuildJobID()));
             }
 
-            StorageCleanUtil.dropHTables(new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration()), toDropHTables);
-            StorageCleanUtil.deleteHDFSPath(HadoopUtil.getWorkingFileSystem(), toDelHDFSPaths);
+//            StorageCleanUtil.dropHTables(new HBaseAdmin(HBaseConnection.getCurrentHBaseConfiguration()), toDropHTables);
+//            StorageCleanUtil.deleteHDFSPath(HadoopUtil.getWorkingFileSystem(), toDelHDFSPaths);
         }
     }
 
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
index 8fc1de9..e984295 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/HBaseInfoUtil.java
@@ -18,40 +18,31 @@
 
 package org.apache.kylin.rest.service;
 
-import java.io.IOException;
-import java.util.Map;
-
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.rest.response.HBaseResponse;
-import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
-import org.apache.kylin.storage.hbase.util.HBaseUnionUtil;
-
 public class HBaseInfoUtil {
     
-    @SuppressWarnings("unused") // used by reflection
-    public static HBaseResponse getHBaseInfo(String tableName, KylinConfig config) throws IOException {
-        if (!config.getStorageUrl().getScheme().equals("hbase"))
-            return null;
-        
-        Connection conn = HBaseUnionUtil.getConnection(config, tableName);
-        HBaseResponse hr = null;
-        long tableSize = 0;
-        int regionCount = 0;
-
-        HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
-        Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
-
-        for (long s : sizeMap.values()) {
-            tableSize += s;
-        }
-
-        regionCount = sizeMap.size();
-
-        // Set response.
-        hr = new HBaseResponse();
-        hr.setTableSize(tableSize);
-        hr.setRegionCount(regionCount);
-        return hr;
-    }
+//    @SuppressWarnings("unused") // used by reflection
+//    public static HBaseResponse getHBaseInfo(String tableName, KylinConfig config) throws IOException {
+//        if (!config.getStorageUrl().getScheme().equals("hbase"))
+//            return null;
+//
+//        Connection conn = HBaseUnionUtil.getConnection(config, tableName);
+//        HBaseResponse hr = null;
+//        long tableSize = 0;
+//        int regionCount = 0;
+//
+//        HBaseRegionSizeCalculator cal = new HBaseRegionSizeCalculator(tableName, conn);
+//        Map<byte[], Long> sizeMap = cal.getRegionSizeMap();
+//
+//        for (long s : sizeMap.values()) {
+//            tableSize += s;
+//        }
+//
+//        regionCount = sizeMap.size();
+//
+//        // Set response.
+//        hr = new HBaseResponse();
+//        hr.setTableSize(tableSize);
+//        hr.setRegionCount(regionCount);
+//        return hr;
+//    }
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
index c26291c..c9c1787 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/JobService.java
@@ -45,9 +45,7 @@ import org.apache.kylin.engine.EngineFactory;
 import org.apache.kylin.engine.mr.BatchOptimizeJobCheckpointBuilder;
 import org.apache.kylin.engine.mr.CubingJob;
 import org.apache.kylin.engine.mr.LookupSnapshotBuildJob;
-import org.apache.kylin.engine.mr.LookupSnapshotJobBuilder;
 import org.apache.kylin.engine.mr.common.CubeJobLockUtil;
-import org.apache.kylin.engine.mr.StreamingCubingEngine;
 import org.apache.kylin.engine.mr.common.JobInfoConverter;
 import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
 import org.apache.kylin.engine.spark.metadata.cube.source.SourceFactory;
@@ -72,7 +70,6 @@ import org.apache.kylin.metadata.model.SegmentRange;
 import org.apache.kylin.metadata.model.SegmentRange.TSRange;
 import org.apache.kylin.metadata.model.SegmentStatusEnum;
 import org.apache.kylin.metadata.model.Segments;
-import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.rest.exception.BadRequestException;
 import org.apache.kylin.rest.msg.Message;
@@ -81,8 +78,6 @@ import org.apache.kylin.rest.util.AclEvaluate;
 import org.apache.kylin.source.ISource;
 import org.apache.kylin.source.SourceManager;
 import org.apache.kylin.source.SourcePartition;
-import org.apache.kylin.stream.coordinator.Coordinator;
-import org.apache.kylin.stream.core.model.SegmentBuildState;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.InitializingBean;
@@ -254,7 +249,7 @@ public class JobService extends BasicService implements InitializingBean {
                 } else {
                     ISource source = SourceManager.getSource(cube);
                     src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd);
-                    src = source.enrichSourcePartitionBeforeBuild(cube, src);
+//                    src = source.enrichSourcePartitionBeforeBuild(cube, src);
                 }
 
                 newSeg = getCubeManager().appendSegment(cube, src);
@@ -408,20 +403,20 @@ public class JobService extends BasicService implements InitializingBean {
         return optimizeJobInstance;
     }
 
-    public JobInstance submitLookupSnapshotJob(CubeInstance cube, String lookupTable, List<String> segmentIDs,
-            String submitter) throws IOException {
-        Message msg = MsgPicker.getMsg();
-        TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject());
-        if (tableDesc.isView()) {
-            throw new BadRequestException(
-                    String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName()));
-        }
-        LookupSnapshotBuildJob job = new LookupSnapshotJobBuilder(cube, lookupTable, segmentIDs, submitter).build();
-        getExecutableManager().addJob(job);
-
-        JobInstance jobInstance = getLookupSnapshotBuildJobInstance(job);
-        return jobInstance;
-    }
+//    public JobInstance submitLookupSnapshotJob(CubeInstance cube, String lookupTable, List<String> segmentIDs,
+//            String submitter) throws IOException {
+//        Message msg = MsgPicker.getMsg();
+//        TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject());
+//        if (tableDesc.isView()) {
+//            throw new BadRequestException(
+//                    String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName()));
+//        }
+//        LookupSnapshotBuildJob job = new LookupSnapshotJobBuilder(cube, lookupTable, segmentIDs, submitter).build();
+//        getExecutableManager().addJob(job);
+//
+//        JobInstance jobInstance = getLookupSnapshotBuildJobInstance(job);
+//        return jobInstance;
+//    }
 
     private void checkCubeDescSignature(CubeInstance cube) {
         Message msg = MsgPicker.getMsg();
@@ -599,36 +594,36 @@ public class JobService extends BasicService implements InitializingBean {
         getExecutableManager().resumeJob(job.getId());
     }
 
-    public void resubmitJob(JobInstance job) throws IOException {
-        aclEvaluate.checkProjectOperationPermission(job);
-
-        Coordinator coordinator = Coordinator.getInstance();
-        CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
-        String cubeName = job.getRelatedCube();
-        CubeInstance cubeInstance = cubeManager.getCube(cubeName);
-
-        String segmentName = job.getRelatedSegmentName();
-        try {
-            Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
-            logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
-            CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
-                    new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
-
-            DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
-            coordinator.getExecutableManager().addJob(executable);
-            CubingJob cubingJob = (CubingJob) executable;
-            newSeg.setLastBuildJobID(cubingJob.getId());
-
-            SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
-            state.setBuildStartTime(System.currentTimeMillis());
-            state.setState(SegmentBuildState.BuildState.State.BUILDING);
-            state.setJobId(cubingJob.getId());
-            coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
-        } catch (Exception e) {
-            logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
-            throw e;
-        }
-    }
+//    public void resubmitJob(JobInstance job) throws IOException {
+//        aclEvaluate.checkProjectOperationPermission(job);
+//
+////        Coordinator coordinator = Coordinator.getInstance();
+//        CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
+//        String cubeName = job.getRelatedCube();
+//        CubeInstance cubeInstance = cubeManager.getCube(cubeName);
+//
+//        String segmentName = job.getRelatedSegmentName();
+//        try {
+//            Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
+//            logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
+//            CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
+//                    new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
+//
+//            DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
+//            coordinator.getExecutableManager().addJob(executable);
+//            CubingJob cubingJob = (CubingJob) executable;
+//            newSeg.setLastBuildJobID(cubingJob.getId());
+//
+//            SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
+//            state.setBuildStartTime(System.currentTimeMillis());
+//            state.setState(SegmentBuildState.BuildState.State.BUILDING);
+//            state.setJobId(cubingJob.getId());
+//            coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
+//        } catch (Exception e) {
+//            logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
+//            throw e;
+//        }
+//    }
 
     public void rollbackJob(JobInstance job, String stepId) {
         aclEvaluate.checkProjectOperationPermission(job);
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java b/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
index ad0de2e..1254c27 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/KafkaConfigService.java
@@ -18,16 +18,7 @@
 
 package org.apache.kylin.rest.service;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Locale;
-
-import org.apache.kylin.rest.exception.BadRequestException;
-import org.apache.kylin.rest.msg.Message;
-import org.apache.kylin.rest.msg.MsgPicker;
 import org.apache.kylin.rest.util.AclEvaluate;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -36,64 +27,64 @@ public class KafkaConfigService extends BasicService {
     @Autowired
     private AclEvaluate aclEvaluate;
 
-    public List<KafkaConfig> listAllKafkaConfigs(final String kafkaConfigName) throws IOException {
-        List<KafkaConfig> kafkaConfigs = new ArrayList<KafkaConfig>();
-        //        CubeInstance cubeInstance = (null != cubeName) ? getCubeManager().getCube(cubeName) : null;
-        if (null == kafkaConfigName) {
-            kafkaConfigs = getKafkaManager().listAllKafkaConfigs();
-        } else {
-            List<KafkaConfig> configs = getKafkaManager().listAllKafkaConfigs();
-            for (KafkaConfig config : configs) {
-                if (kafkaConfigName.equals(config.getName())) {
-                    kafkaConfigs.add(config);
-                }
-            }
-        }
-
-        return kafkaConfigs;
-    }
-
-    public List<KafkaConfig> getKafkaConfigs(final String kafkaConfigName, final String project, final Integer limit,
-            final Integer offset) throws IOException {
-        aclEvaluate.checkProjectWritePermission(project);
-        List<KafkaConfig> kafkaConfigs;
-        kafkaConfigs = listAllKafkaConfigs(kafkaConfigName);
-
-        if (limit == null || offset == null) {
-            return kafkaConfigs;
-        }
-
-        if ((kafkaConfigs.size() - offset) < limit) {
-            return kafkaConfigs.subList(offset, kafkaConfigs.size());
-        }
-
-        return kafkaConfigs.subList(offset, offset + limit);
-    }
-
-    public KafkaConfig createKafkaConfig(KafkaConfig config, String project) throws IOException {
-        aclEvaluate.checkProjectAdminPermission(project);
-        Message msg = MsgPicker.getMsg();
-
-        if (getKafkaManager().getKafkaConfig(config.getName()) != null) {
-            throw new BadRequestException(
-                    String.format(Locale.ROOT, msg.getKAFKA_CONFIG_ALREADY_EXIST(), config.getName()));
-        }
-        getKafkaManager().createKafkaConfig(config);
-        return config;
-    }
-
-    public KafkaConfig updateKafkaConfig(KafkaConfig config, String project) throws IOException {
-        aclEvaluate.checkProjectAdminPermission(project);
-        return getKafkaManager().updateKafkaConfig(config);
-    }
-
-    public KafkaConfig getKafkaConfig(String configName, String project) throws IOException {
-        aclEvaluate.checkProjectWritePermission(project);
-        return getKafkaManager().getKafkaConfig(configName);
-    }
-
-    public void dropKafkaConfig(KafkaConfig config, String project) throws IOException {
-        aclEvaluate.checkProjectAdminPermission(project);
-        getKafkaManager().removeKafkaConfig(config);
-    }
+//    public List<KafkaConfig> listAllKafkaConfigs(final String kafkaConfigName) throws IOException {
+//        List<KafkaConfig> kafkaConfigs = new ArrayList<KafkaConfig>();
+//        //        CubeInstance cubeInstance = (null != cubeName) ? getCubeManager().getCube(cubeName) : null;
+//        if (null == kafkaConfigName) {
+//            kafkaConfigs = getKafkaManager().listAllKafkaConfigs();
+//        } else {
+//            List<KafkaConfig> configs = getKafkaManager().listAllKafkaConfigs();
+//            for (KafkaConfig config : configs) {
+//                if (kafkaConfigName.equals(config.getName())) {
+//                    kafkaConfigs.add(config);
+//                }
+//            }
+//        }
+//
+//        return kafkaConfigs;
+//    }
+//
+//    public List<KafkaConfig> getKafkaConfigs(final String kafkaConfigName, final String project, final Integer limit,
+//            final Integer offset) throws IOException {
+//        aclEvaluate.checkProjectWritePermission(project);
+//        List<KafkaConfig> kafkaConfigs;
+//        kafkaConfigs = listAllKafkaConfigs(kafkaConfigName);
+//
+//        if (limit == null || offset == null) {
+//            return kafkaConfigs;
+//        }
+//
+//        if ((kafkaConfigs.size() - offset) < limit) {
+//            return kafkaConfigs.subList(offset, kafkaConfigs.size());
+//        }
+//
+//        return kafkaConfigs.subList(offset, offset + limit);
+//    }
+//
+//    public KafkaConfig createKafkaConfig(KafkaConfig config, String project) throws IOException {
+//        aclEvaluate.checkProjectAdminPermission(project);
+//        Message msg = MsgPicker.getMsg();
+//
+//        if (getKafkaManager().getKafkaConfig(config.getName()) != null) {
+//            throw new BadRequestException(
+//                    String.format(Locale.ROOT, msg.getKAFKA_CONFIG_ALREADY_EXIST(), config.getName()));
+//        }
+//        getKafkaManager().createKafkaConfig(config);
+//        return config;
+//    }
+//
+//    public KafkaConfig updateKafkaConfig(KafkaConfig config, String project) throws IOException {
+//        aclEvaluate.checkProjectAdminPermission(project);
+//        return getKafkaManager().updateKafkaConfig(config);
+//    }
+//
+//    public KafkaConfig getKafkaConfig(String configName, String project) throws IOException {
+//        aclEvaluate.checkProjectWritePermission(project);
+//        return getKafkaManager().getKafkaConfig(configName);
+//    }
+//
+//    public void dropKafkaConfig(KafkaConfig config, String project) throws IOException {
+//        aclEvaluate.checkProjectAdminPermission(project);
+//        getKafkaManager().removeKafkaConfig(config);
+//    }
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 652cbfc..e8cd931 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -118,7 +118,6 @@ import org.apache.kylin.rest.util.SQLResponseSignatureUtil;
 import org.apache.kylin.rest.util.TableauInterceptor;
 import org.apache.kylin.storage.hybrid.HybridInstance;
 import org.apache.kylin.storage.hybrid.HybridManager;
-import org.apache.kylin.storage.stream.StreamStorageQuery;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -495,19 +494,19 @@ public class QueryService extends BasicService {
 
             boolean realtimeQuery = false;
             Collection<OLAPContext> olapContexts = OLAPContext.getThreadLocalContexts();
-            if (olapContexts != null) {
-                for (OLAPContext ctx : olapContexts) {
-                    try {
-                        if (ctx.storageContext.getStorageQuery() instanceof StreamStorageQuery) {
-                            realtimeQuery = true;
-                            logger.debug("Shutdown query cache for realtime.");
-                        }
-                    } catch (Exception e) {
-                        logger.error("Error", e);
-                    }
-
-                }
-            }
+//            if (olapContexts != null) {
+//                for (OLAPContext ctx : olapContexts) {
+////                    try {
+////                        if (ctx.storageContext.getStorageQuery() instanceof StreamStorageQuery) {
+////                            realtimeQuery = true;
+////                            logger.debug("Shutdown query cache for realtime.");
+////                        }
+////                    } catch (Exception e) {
+////                        logger.error("Error", e);
+////                    }
+//
+//                }
+//            }
 
             if (checkCondition(queryCacheEnabled, "query cache is disabled") //
                     && checkCondition(!Strings.isNullOrEmpty(sqlResponse.getCube()),
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingCoordinatorService.java b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingCoordinatorService.java
index 7ac7610..3ea89ee 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingCoordinatorService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingCoordinatorService.java
@@ -18,107 +18,90 @@
 
 package org.apache.kylin.rest.service;
 
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.stream.coordinator.Coordinator;
-import org.apache.kylin.stream.coordinator.StreamMetadataStore;
-import org.apache.kylin.stream.coordinator.StreamMetadataStoreFactory;
-import org.apache.kylin.stream.coordinator.client.CoordinatorClient;
-import org.apache.kylin.stream.coordinator.coordinate.StreamingCoordinator;
-import org.apache.kylin.stream.core.model.CubeAssignment;
-import org.apache.kylin.stream.core.model.ReplicaSet;
-import org.apache.kylin.stream.core.model.Node;
-import org.apache.kylin.stream.core.source.Partition;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 
-import com.google.common.collect.Sets;
-
 @Component("streamingCoordinatorService")
 public class StreamingCoordinatorService extends BasicService {
     private static final Logger logger = LoggerFactory.getLogger(StreamingCoordinatorService.class);
 
-    private StreamMetadataStore streamMetadataStore;
-
-    private CoordinatorClient streamingCoordinator;
-
-    public StreamingCoordinatorService() {
-        streamMetadataStore = StreamMetadataStoreFactory.getStreamMetaDataStore();
-        if (KylinConfig.getInstanceFromEnv().isNewCoordinatorEnabled()) {
-            logger.info("Use new version coordinator.");
-            streamingCoordinator = StreamingCoordinator.getInstance();
-        } else {
-            logger.info("Use old version coordinator.");
-            streamingCoordinator = Coordinator.getInstance();
-        }
-    }
-
-    public synchronized Map<Integer, Map<String, List<Partition>>> reBalanceRecommend() {
-        return streamingCoordinator.reBalanceRecommend();
-    }
-
-    public synchronized void reBalance(Map<Integer, Map<String, List<Partition>>> reBalancePlan) {
-        streamingCoordinator.reBalance(reBalancePlan);
-    }
-
-    public void assignCube(String cubeName) {
-        streamingCoordinator.assignCube(cubeName);
-    }
-
-    public void unAssignCube(String cubeName) {
-        streamingCoordinator.unAssignCube(cubeName);
-    }
-
-    public void reAssignCube(String cubeName, CubeAssignment newAssignment) {
-        validateAssignment(newAssignment);
-        streamingCoordinator.reAssignCube(cubeName, newAssignment);
-    }
-
-    private void validateAssignment(CubeAssignment newAssignment) {
-        Map<Integer, List<Partition>> assignments = newAssignment.getAssignments();
-        Set<Integer> inputReplicaSetIDs = assignments.keySet();
-        Set<Integer> allReplicaSetIDs = Sets.newHashSet(streamMetadataStore.getReplicaSetIDs());
-        for (Integer inputReplicaSetID : inputReplicaSetIDs) {
-            if (!allReplicaSetIDs.contains(inputReplicaSetID)) {
-                throw new IllegalArgumentException("the replica set id:" + inputReplicaSetID + " does not exist");
-            }
-        }
-    }
-
-    public void pauseConsumers(String cubeName) {
-        streamingCoordinator.pauseConsumers(cubeName);
-    }
-
-    public void resumeConsumers(String cubeName) {
-        streamingCoordinator.resumeConsumers(cubeName);
-    }
-
-    public void replicaSetLeaderChange(int replicaSetID, Node newLeader) {
-        streamingCoordinator.replicaSetLeaderChange(replicaSetID, newLeader);
-    }
-
-    public void createReplicaSet(ReplicaSet rs) {
-        streamingCoordinator.createReplicaSet(rs);
-    }
-
-    public void removeReplicaSet(int rsID) {
-        streamingCoordinator.removeReplicaSet(rsID);
-    }
-
-    public void addNodeToReplicaSet(Integer replicaSetID, String nodeID) {
-        streamingCoordinator.addNodeToReplicaSet(replicaSetID, nodeID);
-    }
-
-    public void removeNodeFromReplicaSet(Integer replicaSetID, String nodeID) {
-        streamingCoordinator.removeNodeFromReplicaSet(replicaSetID, nodeID);
-    }
-
-    public void onSegmentRemoteStoreComplete(String cubeName, Pair<Long, Long> segment, Node receiver) {
-        streamingCoordinator.segmentRemoteStoreComplete(receiver, cubeName, segment);
-    }
+//    private StreamMetadataStore streamMetadataStore;
+//
+//    private CoordinatorClient streamingCoordinator;
+//
+//    public StreamingCoordinatorService() {
+//        streamMetadataStore = StreamMetadataStoreFactory.getStreamMetaDataStore();
+//        if (KylinConfig.getInstanceFromEnv().isNewCoordinatorEnabled()) {
+//            logger.info("Use new version coordinator.");
+//            streamingCoordinator = StreamingCoordinator.getInstance();
+//        } else {
+//            logger.info("Use old version coordinator.");
+//            streamingCoordinator = Coordinator.getInstance();
+//        }
+//    }
+//
+//    public synchronized Map<Integer, Map<String, List<Partition>>> reBalanceRecommend() {
+//        return streamingCoordinator.reBalanceRecommend();
+//    }
+//
+//    public synchronized void reBalance(Map<Integer, Map<String, List<Partition>>> reBalancePlan) {
+//        streamingCoordinator.reBalance(reBalancePlan);
+//    }
+//
+//    public void assignCube(String cubeName) {
+//        streamingCoordinator.assignCube(cubeName);
+//    }
+//
+//    public void unAssignCube(String cubeName) {
+//        streamingCoordinator.unAssignCube(cubeName);
+//    }
+//
+//    public void reAssignCube(String cubeName, CubeAssignment newAssignment) {
+//        validateAssignment(newAssignment);
+//        streamingCoordinator.reAssignCube(cubeName, newAssignment);
+//    }
+//
+//    private void validateAssignment(CubeAssignment newAssignment) {
+//        Map<Integer, List<Partition>> assignments = newAssignment.getAssignments();
+//        Set<Integer> inputReplicaSetIDs = assignments.keySet();
+//        Set<Integer> allReplicaSetIDs = Sets.newHashSet(streamMetadataStore.getReplicaSetIDs());
+//        for (Integer inputReplicaSetID : inputReplicaSetIDs) {
+//            if (!allReplicaSetIDs.contains(inputReplicaSetID)) {
+//                throw new IllegalArgumentException("the replica set id:" + inputReplicaSetID + " does not exist");
+//            }
+//        }
+//    }
+//
+//    public void pauseConsumers(String cubeName) {
+//        streamingCoordinator.pauseConsumers(cubeName);
+//    }
+//
+//    public void resumeConsumers(String cubeName) {
+//        streamingCoordinator.resumeConsumers(cubeName);
+//    }
+//
+//    public void replicaSetLeaderChange(int replicaSetID, Node newLeader) {
+//        streamingCoordinator.replicaSetLeaderChange(replicaSetID, newLeader);
+//    }
+//
+//    public void createReplicaSet(ReplicaSet rs) {
+//        streamingCoordinator.createReplicaSet(rs);
+//    }
+//
+//    public void removeReplicaSet(int rsID) {
+//        streamingCoordinator.removeReplicaSet(rsID);
+//    }
+//
+//    public void addNodeToReplicaSet(Integer replicaSetID, String nodeID) {
+//        streamingCoordinator.addNodeToReplicaSet(replicaSetID, nodeID);
+//    }
+//
+//    public void removeNodeFromReplicaSet(Integer replicaSetID, String nodeID) {
+//        streamingCoordinator.removeNodeFromReplicaSet(replicaSetID, nodeID);
+//    }
+//
+//    public void onSegmentRemoteStoreComplete(String cubeName, Pair<Long, Long> segment, Node receiver) {
+//        streamingCoordinator.segmentRemoteStoreComplete(receiver, cubeName, segment);
+//    }
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingV2Service.java b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingV2Service.java
index 023d4de..7fdd517 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/StreamingV2Service.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/StreamingV2Service.java
@@ -18,565 +18,508 @@
 
 package org.apache.kylin.rest.service;
 
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.metadata.model.ISourceAware;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.rest.constant.Constant;
-import org.apache.kylin.rest.exception.InternalErrorException;
-import org.apache.kylin.stream.coordinator.StreamMetadataStore;
-import org.apache.kylin.stream.coordinator.StreamMetadataStoreFactory;
-import org.apache.kylin.stream.core.model.CubeAssignment;
-import org.apache.kylin.stream.core.model.ReplicaSet;
-import org.apache.kylin.stream.coordinator.client.CoordinatorClient;
-import org.apache.kylin.stream.coordinator.client.CoordinatorClientFactory;
-import org.apache.kylin.stream.core.client.HttpReceiverAdminClient;
-import org.apache.kylin.stream.core.client.ReceiverAdminClient;
-import org.apache.kylin.stream.core.model.Node;
-import org.apache.kylin.stream.core.model.StreamingCubeConsumeState;
-import org.apache.kylin.stream.core.model.stats.ClusterState;
-import org.apache.kylin.stream.core.model.stats.ConsumerStats;
-import org.apache.kylin.stream.core.model.stats.CubeRealTimeState;
-import org.apache.kylin.stream.core.model.stats.PartitionConsumeStats;
-import org.apache.kylin.stream.core.model.stats.ReceiverCubeRealTimeState;
-import org.apache.kylin.stream.core.model.stats.ReceiverCubeStats;
-import org.apache.kylin.stream.core.model.stats.ReceiverState;
-import org.apache.kylin.stream.core.model.stats.ReceiverStats;
-import org.apache.kylin.stream.core.model.stats.ReplicaSetState;
-import org.apache.kylin.stream.core.source.IStreamingSource;
-import org.apache.kylin.stream.core.source.Partition;
-import org.apache.kylin.stream.core.source.StreamingSourceConfig;
-import org.apache.kylin.stream.core.source.StreamingSourceConfigManager;
-import org.apache.kylin.stream.core.source.StreamingSourceFactory;
-import org.apache.kylin.stream.core.util.NamedThreadFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.security.access.prepost.PreAuthorize;
-import org.springframework.stereotype.Component;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
 /**
  * StreamingCoordinatorService will try to forward request to corrdinator leader by HttpClient.
  */
-@Component("streamingServiceV2")
 public class StreamingV2Service extends BasicService {
-    private static final Logger logger = LoggerFactory.getLogger(StreamingV2Service.class);
-    private static final String CLUSTER_STATE = "cluster_state";
-
-    private StreamMetadataStore streamMetadataStore;
-
-    private ReceiverAdminClient receiverAdminClient;
-
-    private Cache<String, ClusterState> clusterStateCache = CacheBuilder.newBuilder()
-            .expireAfterWrite(10, TimeUnit.SECONDS).build();
-
-    private ExecutorService clusterStateExecutor = new ThreadPoolExecutor(0, 20, 60L, TimeUnit.SECONDS,
-            new LinkedBlockingQueue<>(), new NamedThreadFactory("fetch_receiver_state"));
-
-    public StreamingV2Service() {
-        streamMetadataStore = StreamMetadataStoreFactory.getStreamMetaDataStore();
-        receiverAdminClient = new HttpReceiverAdminClient();
-    }
-
-    public List<StreamingSourceConfig> listAllStreamingConfigs(final String table) throws IOException {
-        List<StreamingSourceConfig> streamingSourceConfigs = Lists.newArrayList();
-        if (StringUtils.isEmpty(table)) {
-            streamingSourceConfigs = getStreamingManagerV2().listAllStreaming();
-        } else {
-            StreamingSourceConfig config = getStreamingManagerV2().getConfig(table);
-            if (config != null) {
-                streamingSourceConfigs.add(config);
-            }
-        }
-
-        return streamingSourceConfigs;
-    }
-
-    public List<StreamingSourceConfig> getStreamingConfigs(final String table, final Integer limit, final Integer offset)
-            throws IOException {
-        List<StreamingSourceConfig> streamingSourceConfigs;
-        streamingSourceConfigs = listAllStreamingConfigs(table);
-
-        if (limit == null || offset == null) {
-            return streamingSourceConfigs;
-        }
-
-        if ((streamingSourceConfigs.size() - offset) < limit) {
-            return streamingSourceConfigs.subList(offset, streamingSourceConfigs.size());
-        }
-
-        return streamingSourceConfigs.subList(offset, offset + limit);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#project, 'ADMINISTRATION')")
-    public StreamingSourceConfig createStreamingConfig(StreamingSourceConfig config, ProjectInstance project) throws IOException {
-        if (getStreamingManagerV2().getConfig(config.getName()) != null) {
-            throw new InternalErrorException("The streamingSourceConfig named " + config.getName() + " already exists");
-        }
-        StreamingSourceConfig streamingSourceConfig = getStreamingManagerV2().saveStreamingConfig(config);
-        return streamingSourceConfig;
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public StreamingSourceConfig updateStreamingConfig(StreamingSourceConfig config) throws IOException {
-        return getStreamingManagerV2().updateStreamingConfig(config);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void dropStreamingConfig(StreamingSourceConfig config) throws IOException {
-        getStreamingManagerV2().removeStreamingConfig(config);
-    }
-
-    public String getParserTemplate(final int sourceType, StreamingSourceConfig config) {
-        IStreamingSource streamingSource = StreamingSourceFactory.getStreamingSource(new ISourceAware() {
-            @Override
-            public int getSourceType() {
-                return sourceType;
-            }
-
-            @Override
-            public KylinConfig getConfig() {
-                return getConfig();
-            }
-        });
-        return streamingSource.getMessageTemplate(config);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
-    public List<CubeAssignment> getStreamingCubeAssignments(final CubeInstance cube) {
-        if (cube == null) {
-            return streamMetadataStore.getAllCubeAssignments();
-        }
-        List<CubeAssignment> result = Lists.newArrayList();
-        CubeAssignment assignment = streamMetadataStore.getAssignmentsByCube(cube.getName());
-        if (assignment != null) {
-            result.add(assignment);
-        }
-        return result;
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public Map<Integer, Map<String, List<Partition>>> getStreamingReplicaSetAssignments(Integer replicaSetID) {
-        if (replicaSetID == null) {
-            return streamMetadataStore.getAllReplicaSetAssignments();
-        }
-        Map<Integer, Map<String, List<Partition>>> result = Maps.newHashMap();
-        Map<String, List<Partition>> assignment = streamMetadataStore.getAssignmentsByReplicaSet(replicaSetID);
-        if (assignment != null) {
-            result.put(replicaSetID, assignment);
-        }
-        return result;
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public Map<Integer, Map<String, List<Partition>>> reBalancePlan() {
-        return getCoordinatorClient().reBalanceRecommend();
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void reBalance(Map<Integer, Map<String, List<Partition>>> reBalancePlan) {
-        getCoordinatorClient().reBalance(reBalancePlan);
-    }
-
-    public List<String> getStreamingCubes() {
-        return streamMetadataStore.getCubes();
-    }
-
-    public StreamingCubeConsumeState getStreamingCubeConsumeState(String cubeName) {
-        return streamMetadataStore.getStreamingCubeConsumeState(cubeName);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
-    public void assignCube(CubeInstance cube) {
-        getCoordinatorClient().assignCube(cube.getName());
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
-    public void unAssignCube(CubeInstance cube) {
-        getCoordinatorClient().unAssignCube(cube.getName());
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void reAssignCube(String cubeName, CubeAssignment newAssignment) {
-        validateAssignment(newAssignment);
-        getCoordinatorClient().reAssignCube(cubeName, newAssignment);
-    }
-
-    private void validateAssignment(CubeAssignment newAssignment) {
-        Map<Integer, List<Partition>> assignments = newAssignment.getAssignments();
-        Set<Integer> inputReplicaSetIDs = assignments.keySet();
-        Set<Integer> allReplicaSetIDs = Sets.newHashSet(streamMetadataStore.getReplicaSetIDs());
-        for (Integer inputReplicaSetID : inputReplicaSetIDs) {
-            if (!allReplicaSetIDs.contains(inputReplicaSetID)) {
-                throw new IllegalArgumentException("the replica set id:" + inputReplicaSetID + " does not exist");
-            }
-        }
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
-    public void pauseConsumers(CubeInstance cube) {
-        getCoordinatorClient().pauseConsumers(cube.getName());
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
-    public void resumeConsumers(CubeInstance cube) {
-        getCoordinatorClient().resumeConsumers(cube.getName());
-    }
-
-    public StreamingSourceConfigManager getStreamingManagerV2() {
-        return StreamingSourceConfigManager.getInstance(getConfig());
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void removeCubeAssignment() {
-
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public List<Node> getReceivers() {
-        List<Node> result = streamMetadataStore.getReceivers();
-        return result;
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void removeReceiver(Node receiver) {
-        List<ReplicaSet> replicaSets = streamMetadataStore.getReplicaSets();
-        for (ReplicaSet replicaSet : replicaSets) {
-            Set<Node> receivers = replicaSet.getNodes();
-            if (receivers != null && receivers.contains(receiver)) {
-                throw new IllegalStateException("Before remove receiver, it must be firstly removed from replica set:"
-                        + replicaSet.getReplicaSetID());
-            }
-        }
-        streamMetadataStore.removeReceiver(receiver);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void createReplicaSet(ReplicaSet rs) {
-        getCoordinatorClient().createReplicaSet(rs);
-        clusterStateCache.invalidate(CLUSTER_STATE);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void removeReplicaSet(int rsID) {
-        getCoordinatorClient().removeReplicaSet(rsID);
-        clusterStateCache.invalidate(CLUSTER_STATE);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void addNodeToReplicaSet(Integer replicaSetID, String nodeID) {
-        getCoordinatorClient().addNodeToReplicaSet(replicaSetID, nodeID);
-        clusterStateCache.invalidate(CLUSTER_STATE);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public void removeNodeFromReplicaSet(Integer replicaSetID, String nodeID) {
-        getCoordinatorClient().removeNodeFromReplicaSet(replicaSetID, nodeID);
-        clusterStateCache.invalidate(CLUSTER_STATE);
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
-    public List<ReplicaSet> getReplicaSets() {
-        List<ReplicaSet> result = streamMetadataStore.getReplicaSets();
-        return result;
-    }
-
-    public ReceiverStats getReceiverStats(Node receiver) {
-        try {
-            return receiverAdminClient.getReceiverStats(receiver);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public ReceiverCubeStats getReceiverCubeStats(Node receiver, String cubeName) {
-        try {
-            return receiverAdminClient.getReceiverCubeStats(receiver, cubeName);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
-            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT') or hasPermission(#cube.getProjectInstance(), 'MANAGEMENT') or hasPermission(#cube.getProjectInstance(), 'ADMINISTRATION')")
-    public CubeRealTimeState getCubeRealTimeState(CubeInstance cube) {
-        CubeRealTimeState result = new CubeRealTimeState();
-        result.setCubeName(cube.getName());
-        CubeAssignment cubeAssignment = streamMetadataStore.getAssignmentsByCube(cube.getName());
-        Map<Integer, Map<Node, ReceiverCubeRealTimeState>> rsReceiverCubeStateMap = Maps.newHashMap();
-        for (Integer replicaSetID : cubeAssignment.getReplicaSetIDs()) {
-            ReplicaSet replicaSet = streamMetadataStore.getReplicaSet(replicaSetID);
-            Map<Node, ReceiverCubeRealTimeState> receiverCubeStateMap = Maps.newHashMap();
-            Set<Node> receivers = replicaSet.getNodes();
-            for (Node receiver : receivers) {
-                ReceiverCubeRealTimeState receiverCubeRealTimeState = new ReceiverCubeRealTimeState();
-                try {
-                    ReceiverCubeStats receiverCubeStats = receiverAdminClient.getReceiverCubeStats(receiver,
-                            cube.getName());
-                    receiverCubeRealTimeState.setState(ReceiverState.State.HEALTHY);
-                    receiverCubeRealTimeState.setReceiverCubeStats(receiverCubeStats);
-                } catch (IOException e) {
-                    logger.error("exception when get receiver cube stats", e);
-                    if (!isReceiverReachable(receiver)) {
-                        receiverCubeRealTimeState.setState(ReceiverState.State.UNREACHABLE);
-                    } else {
-                        receiverCubeRealTimeState.setState(ReceiverState.State.DOWN);
-                    }
-                }
-                receiverCubeStateMap.put(receiver, receiverCubeRealTimeState);
-            }
-            rsReceiverCubeStateMap.put(replicaSetID, receiverCubeStateMap);
-        }
-        result.setReceiverCubeStateMap(rsReceiverCubeStateMap);
-
-        return result;
-    }
-
-    /**
-     * Fetch and calculate total cluster state.
-     * @return
-     */
-    public ClusterState getClusterState() {
-        ClusterState clusterState = clusterStateCache.getIfPresent(CLUSTER_STATE);
-        if (clusterState != null) {
-            return clusterState;
-        }
-        List<ReplicaSet> replicaSets = streamMetadataStore.getReplicaSets();
-        List<Node> allReceivers = streamMetadataStore.getReceivers();
-        Map<Integer, Map<String, List<Partition>>> rsAssignments = streamMetadataStore.getAllReplicaSetAssignments();
-
-        Map<Node, Future<ReceiverStats>> statsFuturesMap = Maps.newHashMap();
-        for (final Node receiver : allReceivers) {
-            Future<ReceiverStats> receiverStatsFuture = clusterStateExecutor.submit(() -> receiverAdminClient.getReceiverStats(receiver));
-            statsFuturesMap.put(receiver, receiverStatsFuture);
-        }
-
-        clusterState = new ClusterState();
-        for (ReplicaSet replicaSet : replicaSets) {
-            ReplicaSetState replicaSetState = calReplicaSetState(replicaSet,
-                    rsAssignments.get(replicaSet.getReplicaSetID()), statsFuturesMap);
-            clusterState.addReplicaSetState(replicaSetState);
-            allReceivers.removeAll(replicaSet.getNodes());
-        }
-
-        // left receivers are not assigned receivers
-        for (Node receiver : allReceivers) {
-            Future<ReceiverStats> futureStats = statsFuturesMap.get(receiver);
-            ReceiverState receiverState = getReceiverStateFromStats(receiver, futureStats);
-            clusterState.addAvailableReveiverState(receiverState);
-        }
-        clusterState.setLastUpdateTime(System.currentTimeMillis());
-        clusterStateCache.put("cluster_state", clusterState);
-        return clusterState;
-    }
-
-    private ReplicaSetState calReplicaSetState(ReplicaSet replicaSet, Map<String, List<Partition>> rsAssignment,
-            Map<Node, Future<ReceiverStats>> statsFuturesMap) {
-        ReplicaSetState replicaSetState = new ReplicaSetState();
-        replicaSetState.setRsID(replicaSet.getReplicaSetID());
-        replicaSetState.setAssignment(rsAssignment);
-        Set<Node> receivers = replicaSet.getNodes();
-        if (receivers == null || receivers.isEmpty()) {
-            return replicaSetState;
-        }
-
-        Node leadReceiver = replicaSet.getLeader();
-        replicaSetState.setLead(leadReceiver);
-
-        Map<Node, ReceiverStats> receiverStatsMap = Maps.newHashMap();
-        for (Node receiver : receivers) {
-            Future<ReceiverStats> futureStats = statsFuturesMap.get(receiver);
-            try {
-                ReceiverStats receiverStats = futureStats.get();
-                receiverStatsMap.put(receiver, receiverStats);
-            } catch (InterruptedException e) {
-                throw new RuntimeException(e);
-            } catch (ExecutionException e) {
-                replicaSetState.addReveiverState(getReceiverStateFromException(receiver, e));
-                continue;
-            }
-        }
-
-        Map<String, Long> cubeLatestEventMap = Maps.newHashMap();
-        for (ReceiverStats receiverStats : receiverStatsMap.values()) {
-            Map<String, ReceiverCubeStats> cubeStatsMap = receiverStats.getCubeStatsMap();
-            for (Map.Entry<String, ReceiverCubeStats> cubeStatsEntry : cubeStatsMap.entrySet()) {
-                String cubeName = cubeStatsEntry.getKey();
-                ReceiverCubeStats cubeStats = cubeStatsEntry.getValue();
-                Long latestEventTime = cubeLatestEventMap.get(cubeName);
-                if (latestEventTime == null || latestEventTime < cubeStats.getLatestEventTime()) {
-                    cubeLatestEventMap.put(cubeName, cubeStats.getLatestEventTime());
-                }
-            }
-        }
-        long consumeEventLagThreshold = 5 * 60 * 1000L; // default lag warning threshold is 5 minutes
-        for (Map.Entry<Node, ReceiverStats> receiverStatsEntry : receiverStatsMap.entrySet()) {
-            Node receiver = receiverStatsEntry.getKey();
-            ReceiverStats receiverStats = receiverStatsEntry.getValue();
-            ReceiverState receiverState = new ReceiverState();
-            receiverState.setReceiver(receiver);
-            receiverState.setState(ReceiverState.State.HEALTHY);
-            Map<String, List<Partition>> receiverAssignment = receiverStats.getAssignments();
-            if (!assignmentEqual(receiverAssignment, rsAssignment)) {
-                ReceiverState.State state = ReceiverState.State.WARN;
-                receiverState.setState(state);
-                receiverState.addInfo("assignment is inconsistent");
-            }
-
-            if (receiverStats.isLead() && !receiver.equals(leadReceiver)) {
-                ReceiverState.State state = ReceiverState.State.WARN;
-                receiverState.setState(state);
-                receiverState.addInfo("lead state is inconsistent");
-            }
-
-            Map<String, ReceiverCubeStats> cubeStatsMap = receiverStats.getCubeStatsMap();
-            for (Map.Entry<String, ReceiverCubeStats> cubeStatsEntry : cubeStatsMap.entrySet()) {
-                String cubeName = cubeStatsEntry.getKey();
-                ReceiverCubeStats cubeStats = cubeStatsEntry.getValue();
-                Long latestEventTime = cubeLatestEventMap.get(cubeName);
-                if ((latestEventTime - cubeStats.getLatestEventTime()) >= consumeEventLagThreshold) {
-                    ReceiverState.State state = ReceiverState.State.WARN;
-                    receiverState.setState(state);
-                    receiverState.addInfo("cube:" + cubeName + " consuming is lagged");
-                }
-            }
-            receiverState.setRateInOneMin(calConsumeRate(receiver, receiverStats));
-            replicaSetState.addReveiverState(receiverState);
-        }
-        return replicaSetState;
-    }
-
-    private boolean assignmentEqual(Map<String, List<Partition>> receiverAssignment,
-            Map<String, List<Partition>> rsAssignment) {
-        if (emptyMap(receiverAssignment) && emptyMap(rsAssignment)) {
-            return true;
-        }
-
-        if (receiverAssignment != null) {
-            for (Map.Entry<String, List<Partition>> entry : receiverAssignment.entrySet()) {
-                Collections.sort(entry.getValue());
-                entry.setValue(entry.getValue());
-            }
-        }
-
-        if (rsAssignment != null) {
-            for (Map.Entry<String, List<Partition>> entry : rsAssignment.entrySet()) {
-                Collections.sort(entry.getValue());
-                entry.setValue(entry.getValue());
-            }
-        }
-
-        if (receiverAssignment != null && receiverAssignment.equals(rsAssignment)) {
-            return true;
-        }
-        return false;
-    }
-
-    private boolean emptyMap(Map map) {
-        if (map == null || map.isEmpty()) {
-            return true;
-        }
-        return false;
-    }
-
-    private ReceiverState getReceiverStateFromException(Node receiver, ExecutionException e) {
-        ReceiverState receiverState = new ReceiverState();
-        receiverState.setReceiver(receiver);
-        if (!isReceiverReachable(receiver)) {
-            receiverState.setState(ReceiverState.State.UNREACHABLE);
-        } else {
-            receiverState.setState(ReceiverState.State.DOWN);
-        }
-        return receiverState;
-    }
-
-    private boolean isReceiverReachable(Node receiver) {
-        try {
-            InetAddress address = InetAddress.getByName(receiver.getHost());//ping this IP
-            boolean reachable = address.isReachable(1000);
-            if (!reachable) {
-                return false;
-            }
-            return true;
-        } catch (Exception exception) {
-            logger.error("exception when try ping host:" + receiver.getHost(), exception);
-            return false;
-        }
-    }
-
-    private ReceiverState getReceiverStateFromStats(Node receiver, Future<ReceiverStats> futureStats) {
-        ReceiverState receiverState = new ReceiverState();
-        try {
-            ReceiverStats receiverStats = futureStats.get();
-            receiverState.setReceiver(receiver);
-            receiverState.setState(ReceiverState.State.HEALTHY);
-            receiverState.setRateInOneMin(calConsumeRate(receiver, receiverStats));
-        } catch (InterruptedException e) {
-            throw new RuntimeException(e);
-        } catch (ExecutionException e) {
-            receiverState = getReceiverStateFromException(receiver, e);
-        }
-        return receiverState;
-    }
-
-    private double calConsumeRate(Node receiver, ReceiverStats receiverStats) {
-        double result = 0;
-        Map<String, ReceiverCubeStats> cubeStatsMap = receiverStats.getCubeStatsMap();
-        for (Map.Entry<String, ReceiverCubeStats> receiverCubeStatsEntry : cubeStatsMap.entrySet()) {
-            ReceiverCubeStats cubeStats = receiverCubeStatsEntry.getValue();
-            ConsumerStats consumerStats = cubeStats.getConsumerStats();
-            if (consumerStats == null) {
-                logger.warn("no consumer stats exist for cube:{} in receiver:{}", receiverCubeStatsEntry.getKey(),
-                        receiver);
-                continue;
-            }
-            Map<Integer, PartitionConsumeStats> partitionConsumeStatsMap = consumerStats.getPartitionConsumeStatsMap();
-            for (PartitionConsumeStats partitionStats : partitionConsumeStatsMap.values()) {
-                result += partitionStats.getOneMinRate();
-            }
-        }
-        return result;
-    }
-
-    //    private Node.NodeStatus getReceiverStatus(Node receiver) {
-    //        try {
-    //            HealthCheckInfo healthCheckInfo = receiverAdminClient.healthCheck(receiver);
-    //            if (healthCheckInfo.getStatus() == HealthCheckInfo.Status.GOOD) {
-    //                return Node.NodeStatus.HEALTHY;
-    //            } else {
-    //                return Node.NodeStatus.STOPPED;
-    //            }
-    //        } catch (IOException e) {
-    //            return Node.NodeStatus.STOPPED;
-    //        }
-    //    }
-
-    private synchronized CoordinatorClient getCoordinatorClient() {
-        return CoordinatorClientFactory.createCoordinatorClient(streamMetadataStore);
-    }
+//    private static final Logger logger = LoggerFactory.getLogger(StreamingV2Service.class);
+//    private static final String CLUSTER_STATE = "cluster_state";
+//
+//    private StreamMetadataStore streamMetadataStore;
+//
+//    private ReceiverAdminClient receiverAdminClient;
+//
+//    private Cache<String, ClusterState> clusterStateCache = CacheBuilder.newBuilder()
+//            .expireAfterWrite(10, TimeUnit.SECONDS).build();
+//
+//    private ExecutorService clusterStateExecutor = new ThreadPoolExecutor(0, 20, 60L, TimeUnit.SECONDS,
+//            new LinkedBlockingQueue<>(), new NamedThreadFactory("fetch_receiver_state"));
+//
+//    public StreamingV2Service() {
+//        streamMetadataStore = StreamMetadataStoreFactory.getStreamMetaDataStore();
+//        receiverAdminClient = new HttpReceiverAdminClient();
+//    }
+//
+//    public List<StreamingSourceConfig> listAllStreamingConfigs(final String table) throws IOException {
+//        List<StreamingSourceConfig> streamingSourceConfigs = Lists.newArrayList();
+//        if (StringUtils.isEmpty(table)) {
+//            streamingSourceConfigs = getStreamingManagerV2().listAllStreaming();
+//        } else {
+//            StreamingSourceConfig config = getStreamingManagerV2().getConfig(table);
+//            if (config != null) {
+//                streamingSourceConfigs.add(config);
+//            }
+//        }
+//
+//        return streamingSourceConfigs;
+//    }
+//
+//    public List<StreamingSourceConfig> getStreamingConfigs(final String table, final Integer limit, final Integer offset)
+//            throws IOException {
+//        List<StreamingSourceConfig> streamingSourceConfigs;
+//        streamingSourceConfigs = listAllStreamingConfigs(table);
+//
+//        if (limit == null || offset == null) {
+//            return streamingSourceConfigs;
+//        }
+//
+//        if ((streamingSourceConfigs.size() - offset) < limit) {
+//            return streamingSourceConfigs.subList(offset, streamingSourceConfigs.size());
+//        }
+//
+//        return streamingSourceConfigs.subList(offset, offset + limit);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#project, 'ADMINISTRATION')")
+//    public StreamingSourceConfig createStreamingConfig(StreamingSourceConfig config, ProjectInstance project) throws IOException {
+//        if (getStreamingManagerV2().getConfig(config.getName()) != null) {
+//            throw new InternalErrorException("The streamingSourceConfig named " + config.getName() + " already exists");
+//        }
+//        StreamingSourceConfig streamingSourceConfig = getStreamingManagerV2().saveStreamingConfig(config);
+//        return streamingSourceConfig;
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public StreamingSourceConfig updateStreamingConfig(StreamingSourceConfig config) throws IOException {
+//        return getStreamingManagerV2().updateStreamingConfig(config);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void dropStreamingConfig(StreamingSourceConfig config) throws IOException {
+//        getStreamingManagerV2().removeStreamingConfig(config);
+//    }
+//
+//    public String getParserTemplate(final int sourceType, StreamingSourceConfig config) {
+//        IStreamingSource streamingSource = StreamingSourceFactory.getStreamingSource(new ISourceAware() {
+//            @Override
+//            public int getSourceType() {
+//                return sourceType;
+//            }
+//
+//            @Override
+//            public KylinConfig getConfig() {
+//                return getConfig();
+//            }
+//        });
+//        return streamingSource.getMessageTemplate(config);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+//    public List<CubeAssignment> getStreamingCubeAssignments(final CubeInstance cube) {
+//        if (cube == null) {
+//            return streamMetadataStore.getAllCubeAssignments();
+//        }
+//        List<CubeAssignment> result = Lists.newArrayList();
+//        CubeAssignment assignment = streamMetadataStore.getAssignmentsByCube(cube.getName());
+//        if (assignment != null) {
+//            result.add(assignment);
+//        }
+//        return result;
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public Map<Integer, Map<String, List<Partition>>> getStreamingReplicaSetAssignments(Integer replicaSetID) {
+//        if (replicaSetID == null) {
+//            return streamMetadataStore.getAllReplicaSetAssignments();
+//        }
+//        Map<Integer, Map<String, List<Partition>>> result = Maps.newHashMap();
+//        Map<String, List<Partition>> assignment = streamMetadataStore.getAssignmentsByReplicaSet(replicaSetID);
+//        if (assignment != null) {
+//            result.put(replicaSetID, assignment);
+//        }
+//        return result;
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public Map<Integer, Map<String, List<Partition>>> reBalancePlan() {
+//        return getCoordinatorClient().reBalanceRecommend();
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void reBalance(Map<Integer, Map<String, List<Partition>>> reBalancePlan) {
+//        getCoordinatorClient().reBalance(reBalancePlan);
+//    }
+//
+//    public List<String> getStreamingCubes() {
+//        return streamMetadataStore.getCubes();
+//    }
+//
+//    public StreamingCubeConsumeState getStreamingCubeConsumeState(String cubeName) {
+//        return streamMetadataStore.getStreamingCubeConsumeState(cubeName);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+//    public void assignCube(CubeInstance cube) {
+//        getCoordinatorClient().assignCube(cube.getName());
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+//    public void unAssignCube(CubeInstance cube) {
+//        getCoordinatorClient().unAssignCube(cube.getName());
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void reAssignCube(String cubeName, CubeAssignment newAssignment) {
+//        validateAssignment(newAssignment);
+//        getCoordinatorClient().reAssignCube(cubeName, newAssignment);
+//    }
+//
+//    private void validateAssignment(CubeAssignment newAssignment) {
+//        Map<Integer, List<Partition>> assignments = newAssignment.getAssignments();
+//        Set<Integer> inputReplicaSetIDs = assignments.keySet();
+//        Set<Integer> allReplicaSetIDs = Sets.newHashSet(streamMetadataStore.getReplicaSetIDs());
+//        for (Integer inputReplicaSetID : inputReplicaSetIDs) {
+//            if (!allReplicaSetIDs.contains(inputReplicaSetID)) {
+//                throw new IllegalArgumentException("the replica set id:" + inputReplicaSetID + " does not exist");
+//            }
+//        }
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+//    public void pauseConsumers(CubeInstance cube) {
+//        getCoordinatorClient().pauseConsumers(cube.getName());
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+//    public void resumeConsumers(CubeInstance cube) {
+//        getCoordinatorClient().resumeConsumers(cube.getName());
+//    }
+//
+//    public StreamingSourceConfigManager getStreamingManagerV2() {
+//        return StreamingSourceConfigManager.getInstance(getConfig());
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void removeCubeAssignment() {
+//
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public List<Node> getReceivers() {
+//        List<Node> result = streamMetadataStore.getReceivers();
+//        return result;
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void removeReceiver(Node receiver) {
+//        List<ReplicaSet> replicaSets = streamMetadataStore.getReplicaSets();
+//        for (ReplicaSet replicaSet : replicaSets) {
+//            Set<Node> receivers = replicaSet.getNodes();
+//            if (receivers != null && receivers.contains(receiver)) {
+//                throw new IllegalStateException("Before remove receiver, it must be firstly removed from replica set:"
+//                        + replicaSet.getReplicaSetID());
+//            }
+//        }
+//        streamMetadataStore.removeReceiver(receiver);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void createReplicaSet(ReplicaSet rs) {
+//        getCoordinatorClient().createReplicaSet(rs);
+//        clusterStateCache.invalidate(CLUSTER_STATE);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void removeReplicaSet(int rsID) {
+//        getCoordinatorClient().removeReplicaSet(rsID);
+//        clusterStateCache.invalidate(CLUSTER_STATE);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void addNodeToReplicaSet(Integer replicaSetID, String nodeID) {
+//        getCoordinatorClient().addNodeToReplicaSet(replicaSetID, nodeID);
+//        clusterStateCache.invalidate(CLUSTER_STATE);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public void removeNodeFromReplicaSet(Integer replicaSetID, String nodeID) {
+//        getCoordinatorClient().removeNodeFromReplicaSet(replicaSetID, nodeID);
+//        clusterStateCache.invalidate(CLUSTER_STATE);
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN)
+//    public List<ReplicaSet> getReplicaSets() {
+//        List<ReplicaSet> result = streamMetadataStore.getReplicaSets();
+//        return result;
+//    }
+//
+//    public ReceiverStats getReceiverStats(Node receiver) {
+//        try {
+//            return receiverAdminClient.getReceiverStats(receiver);
+//        } catch (IOException e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+//
+//    public ReceiverCubeStats getReceiverCubeStats(Node receiver, String cubeName) {
+//        try {
+//            return receiverAdminClient.getReceiverCubeStats(receiver, cubeName);
+//        } catch (IOException e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+//
+//    @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
+//            + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT') or hasPermission(#cube.getProjectInstance(), 'MANAGEMENT') or hasPermission(#cube.getProjectInstance(), 'ADMINISTRATION')")
+//    public CubeRealTimeState getCubeRealTimeState(CubeInstance cube) {
+//        CubeRealTimeState result = new CubeRealTimeState();
+//        result.setCubeName(cube.getName());
+//        CubeAssignment cubeAssignment = streamMetadataStore.getAssignmentsByCube(cube.getName());
+//        Map<Integer, Map<Node, ReceiverCubeRealTimeState>> rsReceiverCubeStateMap = Maps.newHashMap();
+//        for (Integer replicaSetID : cubeAssignment.getReplicaSetIDs()) {
+//            ReplicaSet replicaSet = streamMetadataStore.getReplicaSet(replicaSetID);
+//            Map<Node, ReceiverCubeRealTimeState> receiverCubeStateMap = Maps.newHashMap();
+//            Set<Node> receivers = replicaSet.getNodes();
+//            for (Node receiver : receivers) {
+//                ReceiverCubeRealTimeState receiverCubeRealTimeState = new ReceiverCubeRealTimeState();
+//                try {
+//                    ReceiverCubeStats receiverCubeStats = receiverAdminClient.getReceiverCubeStats(receiver,
+//                            cube.getName());
+//                    receiverCubeRealTimeState.setState(ReceiverState.State.HEALTHY);
+//                    receiverCubeRealTimeState.setReceiverCubeStats(receiverCubeStats);
+//                } catch (IOException e) {
+//                    logger.error("exception when get receiver cube stats", e);
+//                    if (!isReceiverReachable(receiver)) {
+//                        receiverCubeRealTimeState.setState(ReceiverState.State.UNREACHABLE);
+//                    } else {
+//                        receiverCubeRealTimeState.setState(ReceiverState.State.DOWN);
+//                    }
+//                }
+//                receiverCubeStateMap.put(receiver, receiverCubeRealTimeState);
+//            }
+//            rsReceiverCubeStateMap.put(replicaSetID, receiverCubeStateMap);
+//        }
+//        result.setReceiverCubeStateMap(rsReceiverCubeStateMap);
+//
+//        return result;
+//    }
+//
+//    /**
+//     * Fetch and calculate total cluster state.
+//     * @return
+//     */
+//    public ClusterState getClusterState() {
+//        ClusterState clusterState = clusterStateCache.getIfPresent(CLUSTER_STATE);
+//        if (clusterState != null) {
+//            return clusterState;
+//        }
+//        List<ReplicaSet> replicaSets = streamMetadataStore.getReplicaSets();
+//        List<Node> allReceivers = streamMetadataStore.getReceivers();
+//        Map<Integer, Map<String, List<Partition>>> rsAssignments = streamMetadataStore.getAllReplicaSetAssignments();
+//
+//        Map<Node, Future<ReceiverStats>> statsFuturesMap = Maps.newHashMap();
+//        for (final Node receiver : allReceivers) {
+//            Future<ReceiverStats> receiverStatsFuture = clusterStateExecutor.submit(() -> receiverAdminClient.getReceiverStats(receiver));
+//            statsFuturesMap.put(receiver, receiverStatsFuture);
+//        }
+//
+//        clusterState = new ClusterState();
+//        for (ReplicaSet replicaSet : replicaSets) {
+//            ReplicaSetState replicaSetState = calReplicaSetState(replicaSet,
+//                    rsAssignments.get(replicaSet.getReplicaSetID()), statsFuturesMap);
+//            clusterState.addReplicaSetState(replicaSetState);
+//            allReceivers.removeAll(replicaSet.getNodes());
+//        }
+//
+//        // left receivers are not assigned receivers
+//        for (Node receiver : allReceivers) {
+//            Future<ReceiverStats> futureStats = statsFuturesMap.get(receiver);
+//            ReceiverState receiverState = getReceiverStateFromStats(receiver, futureStats);
+//            clusterState.addAvailableReveiverState(receiverState);
+//        }
+//        clusterState.setLastUpdateTime(System.currentTimeMillis());
+//        clusterStateCache.put("cluster_state", clusterState);
+//        return clusterState;
+//    }
+//
+//    private ReplicaSetState calReplicaSetState(ReplicaSet replicaSet, Map<String, List<Partition>> rsAssignment,
+//            Map<Node, Future<ReceiverStats>> statsFuturesMap) {
+//        ReplicaSetState replicaSetState = new ReplicaSetState();
+//        replicaSetState.setRsID(replicaSet.getReplicaSetID());
+//        replicaSetState.setAssignment(rsAssignment);
+//        Set<Node> receivers = replicaSet.getNodes();
+//        if (receivers == null || receivers.isEmpty()) {
+//            return replicaSetState;
+//        }
+//
+//        Node leadReceiver = replicaSet.getLeader();
+//        replicaSetState.setLead(leadReceiver);
+//
+//        Map<Node, ReceiverStats> receiverStatsMap = Maps.newHashMap();
+//        for (Node receiver : receivers) {
+//            Future<ReceiverStats> futureStats = statsFuturesMap.get(receiver);
+//            try {
+//                ReceiverStats receiverStats = futureStats.get();
+//                receiverStatsMap.put(receiver, receiverStats);
+//            } catch (InterruptedException e) {
+//                throw new RuntimeException(e);
+//            } catch (ExecutionException e) {
+//                replicaSetState.addReveiverState(getReceiverStateFromException(receiver, e));
+//                continue;
+//            }
+//        }
+//
+//        Map<String, Long> cubeLatestEventMap = Maps.newHashMap();
+//        for (ReceiverStats receiverStats : receiverStatsMap.values()) {
+//            Map<String, ReceiverCubeStats> cubeStatsMap = receiverStats.getCubeStatsMap();
+//            for (Map.Entry<String, ReceiverCubeStats> cubeStatsEntry : cubeStatsMap.entrySet()) {
+//                String cubeName = cubeStatsEntry.getKey();
+//                ReceiverCubeStats cubeStats = cubeStatsEntry.getValue();
+//                Long latestEventTime = cubeLatestEventMap.get(cubeName);
+//                if (latestEventTime == null || latestEventTime < cubeStats.getLatestEventTime()) {
+//                    cubeLatestEventMap.put(cubeName, cubeStats.getLatestEventTime());
+//                }
+//            }
+//        }
+//        long consumeEventLagThreshold = 5 * 60 * 1000L; // default lag warning threshold is 5 minutes
+//        for (Map.Entry<Node, ReceiverStats> receiverStatsEntry : receiverStatsMap.entrySet()) {
+//            Node receiver = receiverStatsEntry.getKey();
+//            ReceiverStats receiverStats = receiverStatsEntry.getValue();
+//            ReceiverState receiverState = new ReceiverState();
+//            receiverState.setReceiver(receiver);
+//            receiverState.setState(ReceiverState.State.HEALTHY);
+//            Map<String, List<Partition>> receiverAssignment = receiverStats.getAssignments();
+//            if (!assignmentEqual(receiverAssignment, rsAssignment)) {
+//                ReceiverState.State state = ReceiverState.State.WARN;
+//                receiverState.setState(state);
+//                receiverState.addInfo("assignment is inconsistent");
+//            }
+//
+//            if (receiverStats.isLead() && !receiver.equals(leadReceiver)) {
+//                ReceiverState.State state = ReceiverState.State.WARN;
+//                receiverState.setState(state);
+//                receiverState.addInfo("lead state is inconsistent");
+//            }
+//
+//            Map<String, ReceiverCubeStats> cubeStatsMap = receiverStats.getCubeStatsMap();
+//            for (Map.Entry<String, ReceiverCubeStats> cubeStatsEntry : cubeStatsMap.entrySet()) {
+//                String cubeName = cubeStatsEntry.getKey();
+//                ReceiverCubeStats cubeStats = cubeStatsEntry.getValue();
+//                Long latestEventTime = cubeLatestEventMap.get(cubeName);
+//                if ((latestEventTime - cubeStats.getLatestEventTime()) >= consumeEventLagThreshold) {
+//                    ReceiverState.State state = ReceiverState.State.WARN;
+//                    receiverState.setState(state);
+//                    receiverState.addInfo("cube:" + cubeName + " consuming is lagged");
+//                }
+//            }
+//            receiverState.setRateInOneMin(calConsumeRate(receiver, receiverStats));
+//            replicaSetState.addReveiverState(receiverState);
+//        }
+//        return replicaSetState;
+//    }
+//
+//    private boolean assignmentEqual(Map<String, List<Partition>> receiverAssignment,
+//            Map<String, List<Partition>> rsAssignment) {
+//        if (emptyMap(receiverAssignment) && emptyMap(rsAssignment)) {
+//            return true;
+//        }
+//
+//        if (receiverAssignment != null) {
+//            for (Map.Entry<String, List<Partition>> entry : receiverAssignment.entrySet()) {
+//                Collections.sort(entry.getValue());
+//                entry.setValue(entry.getValue());
+//            }
+//        }
+//
+//        if (rsAssignment != null) {
+//            for (Map.Entry<String, List<Partition>> entry : rsAssignment.entrySet()) {
+//                Collections.sort(entry.getValue());
+//                entry.setValue(entry.getValue());
+//            }
+//        }
+//
+//        if (receiverAssignment != null && receiverAssignment.equals(rsAssignment)) {
+//            return true;
+//        }
+//        return false;
+//    }
+//
+//    private boolean emptyMap(Map map) {
+//        if (map == null || map.isEmpty()) {
+//            return true;
+//        }
+//        return false;
+//    }
+//
+//    private ReceiverState getReceiverStateFromException(Node receiver, ExecutionException e) {
+//        ReceiverState receiverState = new ReceiverState();
+//        receiverState.setReceiver(receiver);
+//        if (!isReceiverReachable(receiver)) {
+//            receiverState.setState(ReceiverState.State.UNREACHABLE);
+//        } else {
+//            receiverState.setState(ReceiverState.State.DOWN);
+//        }
+//        return receiverState;
+//    }
+//
+//    private boolean isReceiverReachable(Node receiver) {
+//        try {
+//            InetAddress address = InetAddress.getByName(receiver.getHost());//ping this IP
+//            boolean reachable = address.isReachable(1000);
+//            if (!reachable) {
+//                return false;
+//            }
+//            return true;
+//        } catch (Exception exception) {
+//            logger.error("exception when try ping host:" + receiver.getHost(), exception);
+//            return false;
+//        }
+//    }
+//
+//    private ReceiverState getReceiverStateFromStats(Node receiver, Future<ReceiverStats> futureStats) {
+//        ReceiverState receiverState = new ReceiverState();
+//        try {
+//            ReceiverStats receiverStats = futureStats.get();
+//            receiverState.setReceiver(receiver);
+//            receiverState.setState(ReceiverState.State.HEALTHY);
+//            receiverState.setRateInOneMin(calConsumeRate(receiver, receiverStats));
+//        } catch (InterruptedException e) {
+//            throw new RuntimeException(e);
+//        } catch (ExecutionException e) {
+//            receiverState = getReceiverStateFromException(receiver, e);
+//        }
+//        return receiverState;
+//    }
+//
+//    private double calConsumeRate(Node receiver, ReceiverStats receiverStats) {
+//        double result = 0;
+//        Map<String, ReceiverCubeStats> cubeStatsMap = receiverStats.getCubeStatsMap();
+//        for (Map.Entry<String, ReceiverCubeStats> receiverCubeStatsEntry : cubeStatsMap.entrySet()) {
+//            ReceiverCubeStats cubeStats = receiverCubeStatsEntry.getValue();
+//            ConsumerStats consumerStats = cubeStats.getConsumerStats();
+//            if (consumerStats == null) {
+//                logger.warn("no consumer stats exist for cube:{} in receiver:{}", receiverCubeStatsEntry.getKey(),
+//                        receiver);
+//                continue;
+//            }
+//            Map<Integer, PartitionConsumeStats> partitionConsumeStatsMap = consumerStats.getPartitionConsumeStatsMap();
+//            for (PartitionConsumeStats partitionStats : partitionConsumeStatsMap.values()) {
+//                result += partitionStats.getOneMinRate();
+//            }
+//        }
+//        return result;
+//    }
+//
+//    //    private Node.NodeStatus getReceiverStatus(Node receiver) {
+//    //        try {
+//    //            HealthCheckInfo healthCheckInfo = receiverAdminClient.healthCheck(receiver);
+//    //            if (healthCheckInfo.getStatus() == HealthCheckInfo.Status.GOOD) {
+//    //                return Node.NodeStatus.HEALTHY;
+//    //            } else {
+//    //                return Node.NodeStatus.STOPPED;
+//    //            }
+//    //        } catch (IOException e) {
+//    //            return Node.NodeStatus.STOPPED;
+//    //        }
+//    //    }
+//
+//    private synchronized CoordinatorClient getCoordinatorClient() {
+//        return CoordinatorClientFactory.createCoordinatorClient(streamMetadataStore);
+//    }
 }
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
index c71a44a..dbb3e3b 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/TableService.java
@@ -47,20 +47,9 @@ import org.apache.kylin.common.util.RandomUtil;
 import org.apache.kylin.cube.CubeInstance;
 import org.apache.kylin.cube.CubeManager;
 import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.dict.lookup.ExtTableSnapshotInfo;
-import org.apache.kylin.dict.lookup.ExtTableSnapshotInfoManager;
-import org.apache.kylin.dict.lookup.IExtLookupTableCache.CacheState;
-import org.apache.kylin.dict.lookup.LookupProviderFactory;
 import org.apache.kylin.dict.lookup.SnapshotManager;
 import org.apache.kylin.dict.lookup.SnapshotTable;
-import org.apache.kylin.engine.mr.common.HadoopShellExecutable;
-import org.apache.kylin.engine.mr.common.MapReduceExecutable;
-import org.apache.kylin.engine.spark.SparkColumnCardinality;
-import org.apache.kylin.engine.spark.SparkExecutable;
 import org.apache.kylin.engine.spark.source.CsvSource;
-import org.apache.kylin.job.execution.CardinalityExecutable;
-import org.apache.kylin.job.execution.ExecutableManager;
-import org.apache.kylin.job.execution.ExecutableState;
 import org.apache.kylin.metadata.TableMetadataManager;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.CsvColumnDesc;
@@ -79,8 +68,6 @@ import org.apache.kylin.source.IReadableTable.TableSignature;
 import org.apache.kylin.source.ISource;
 import org.apache.kylin.source.ISourceMetadataExplorer;
 import org.apache.kylin.source.SourceManager;
-import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityJob;
-import org.apache.kylin.source.hive.cardinality.HiveColumnCardinalityUpdateJob;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -357,54 +344,22 @@ public class TableService extends BasicService {
         return descs;
     }
 
-    public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception {
-        // calculate cardinality for Hive source
-        ProjectInstance projectInstance = getProjectManager().getProject(prj);
-        if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE) {
-            return;
-        }
-        TableMetadataManager metaMgr = getTableManager();
-        ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
-        for (String table : tables) {
-            TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj);
-            String jobID = tableExtDesc.getJodID();
-            if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) {
-                calculateCardinality(table, submitter, prj);
-            }
-        }
-    }
-
-    public void updateSnapshotLocalCache(String project, String tableName, String snapshotID) {
-        ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig());
-        ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID);
-        TableDesc tableDesc = getTableManager().getTableDesc(tableName, project);
-        if (extTableSnapshotInfo == null) {
-            throw new IllegalArgumentException(
-                    "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID);
-        }
-        LookupProviderFactory.rebuildLocalCache(tableDesc, extTableSnapshotInfo);
-    }
-
-    public void removeSnapshotLocalCache(String tableName, String snapshotID) {
-        ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig());
-        ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID);
-        if (extTableSnapshotInfo == null) {
-            throw new IllegalArgumentException(
-                    "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID);
-        }
-        LookupProviderFactory.removeLocalCache(extTableSnapshotInfo);
-    }
-
-    public String getSnapshotLocalCacheState(String tableName, String snapshotID) {
-        ExtTableSnapshotInfoManager snapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig());
-        ExtTableSnapshotInfo extTableSnapshotInfo = snapshotInfoManager.getSnapshot(tableName, snapshotID);
-        if (extTableSnapshotInfo == null) {
-            throw new IllegalArgumentException(
-                    "cannot find ext snapshot info for table:" + tableName + " snapshot:" + snapshotID);
-        }
-        CacheState cacheState = LookupProviderFactory.getCacheState(extTableSnapshotInfo);
-        return cacheState.name();
-    }
+//    public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception {
+//        // calculate cardinality for Hive source
+//        ProjectInstance projectInstance = getProjectManager().getProject(prj);
+//        if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE) {
+//            return;
+//        }
+//        TableMetadataManager metaMgr = getTableManager();
+//        ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig());
+//        for (String table : tables) {
+//            TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj);
+//            String jobID = tableExtDesc.getJodID();
+//            if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) {
+//                calculateCardinality(table, submitter, prj);
+//            }
+//        }
+//    }
 
     public List<TableSnapshotResponse> getLookupTableSnapshots(String project, String tableName) throws IOException {
         TableDesc tableDesc = getTableManager().getTableDesc(tableName, project);
@@ -418,25 +373,12 @@ public class TableService extends BasicService {
 
     List<TableSnapshotResponse> internalGetLookupTableSnapshots(String tableName, TableSignature signature)
             throws IOException {
-        ExtTableSnapshotInfoManager extSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(getConfig());
         SnapshotManager snapshotManager = SnapshotManager.getInstance(getConfig());
-        List<ExtTableSnapshotInfo> extTableSnapshots = extSnapshotInfoManager.getSnapshots(tableName);
         List<SnapshotTable> metaStoreTableSnapshots = snapshotManager.getSnapshots(tableName, signature);
 
         Map<String, List<String>> snapshotUsageMap = getSnapshotUsages();
 
         List<TableSnapshotResponse> result = Lists.newArrayList();
-        for (ExtTableSnapshotInfo extTableSnapshot : extTableSnapshots) {
-            TableSnapshotResponse response = new TableSnapshotResponse();
-            response.setSnapshotID(extTableSnapshot.getId());
-            response.setSnapshotType(TableSnapshotResponse.TYPE_EXT);
-            response.setLastBuildTime(extTableSnapshot.getLastBuildTime());
-            response.setStorageType(extTableSnapshot.getStorageType());
-            response.setSourceTableSize(extTableSnapshot.getSignature().getSize());
-            response.setSourceTableLastModifyTime(extTableSnapshot.getSignature().getLastModifiedTime());
-            response.setCubesAndSegmentsUsage(snapshotUsageMap.get(extTableSnapshot.getResourcePath()));
-            result.add(response);
-        }
 
         for (SnapshotTable metaStoreTableSnapshot : metaStoreTableSnapshots) {
             TableSnapshotResponse response = new TableSnapshotResponse();
@@ -484,62 +426,62 @@ public class TableService extends BasicService {
         return snapshotCubeSegmentMap;
     }
 
-    /**
-     * Generate cardinality for table This will trigger a hadoop job
-     * The result will be merged into table exd info
-     *
-     * @param tableName
-     */
-    public void calculateCardinality(String tableName, String submitter, String prj) throws Exception {
-        aclEvaluate.checkProjectWritePermission(prj);
-        Message msg = MsgPicker.getMsg();
-
-        tableName = normalizeHiveTableName(tableName);
-        TableDesc table = getTableManager().getTableDesc(tableName, prj);
-        final TableExtDesc tableExt = getTableManager().getTableExt(tableName, prj);
-        if (table == null) {
-            BadRequestException e = new BadRequestException(
-                    String.format(Locale.ROOT, msg.getTABLE_DESC_NOT_FOUND(), tableName));
-            logger.error("Cannot find table descriptor " + tableName, e);
-            throw e;
-        }
-
-        CardinalityExecutable job = new CardinalityExecutable();
-        //make sure the job could be scheduled when the DistributedScheduler is enable.
-        job.setParam("segmentId", tableName);
-        job.setName("Hive Column Cardinality calculation for table '" + tableName + "'");
-        job.setSubmitter(submitter);
-
-        String outPath = getConfig().getHdfsWorkingDirectory() + "cardinality/" + job.getId() + "/" + tableName;
-        String param = "-table " + tableName + " -output " + outPath + " -project " + prj;
-
-        if (getConfig().isSparkCardinalityEnabled()) { // use spark engine to calculate cardinality
-            SparkExecutable step1 = new SparkExecutable();
-            step1.setClassName(SparkColumnCardinality.class.getName());
-            step1.setParam(SparkColumnCardinality.OPTION_OUTPUT.getOpt(), outPath);
-            step1.setParam(SparkColumnCardinality.OPTION_PRJ.getOpt(), prj);
-            step1.setParam(SparkColumnCardinality.OPTION_TABLE_NAME.getOpt(), tableName);
-            step1.setParam(SparkColumnCardinality.OPTION_COLUMN_COUNT.getOpt(), String.valueOf(table.getColumnCount()));
-            job.addTask(step1);
-        } else {
-            MapReduceExecutable step1 = new MapReduceExecutable();
-            step1.setMapReduceJobClass(HiveColumnCardinalityJob.class);
-            step1.setMapReduceParams(param);
-            step1.setParam("segmentId", tableName);
-            job.addTask(step1);
-        }
-
-        HadoopShellExecutable step2 = new HadoopShellExecutable();
-
-        step2.setJobClass(HiveColumnCardinalityUpdateJob.class);
-        step2.setJobParams(param);
-        step2.setParam("segmentId", tableName);
-        job.addTask(step2);
-        tableExt.setJodID(job.getId());
-        getTableManager().saveTableExt(tableExt, prj);
-
-        getExecutableManager().addJob(job);
-    }
+//    /**
+//     * Generate cardinality for table This will trigger a hadoop job
+//     * The result will be merged into table exd info
+//     *
+//     * @param tableName
+//     */
+//    public void calculateCardinality(String tableName, String submitter, String prj) throws Exception {
+//        aclEvaluate.checkProjectWritePermission(prj);
+//        Message msg = MsgPicker.getMsg();
+//
+//        tableName = normalizeHiveTableName(tableName);
+//        TableDesc table = getTableManager().getTableDesc(tableName, prj);
+//        final TableExtDesc tableExt = getTableManager().getTableExt(tableName, prj);
+//        if (table == null) {
+//            BadRequestException e = new BadRequestException(
+//                    String.format(Locale.ROOT, msg.getTABLE_DESC_NOT_FOUND(), tableName));
+//            logger.error("Cannot find table descriptor " + tableName, e);
+//            throw e;
+//        }
+//
+//        CardinalityExecutable job = new CardinalityExecutable();
+//        //make sure the job could be scheduled when the DistributedScheduler is enable.
+//        job.setParam("segmentId", tableName);
+//        job.setName("Hive Column Cardinality calculation for table '" + tableName + "'");
+//        job.setSubmitter(submitter);
+//
+//        String outPath = getConfig().getHdfsWorkingDirectory() + "cardinality/" + job.getId() + "/" + tableName;
+//        String param = "-table " + tableName + " -output " + outPath + " -project " + prj;
+//
+//        if (getConfig().isSparkCardinalityEnabled()) { // use spark engine to calculate cardinality
+//            SparkExecutable step1 = new SparkExecutable();
+//            step1.setClassName(SparkColumnCardinality.class.getName());
+//            step1.setParam(SparkColumnCardinality.OPTION_OUTPUT.getOpt(), outPath);
+//            step1.setParam(SparkColumnCardinality.OPTION_PRJ.getOpt(), prj);
+//            step1.setParam(SparkColumnCardinality.OPTION_TABLE_NAME.getOpt(), tableName);
+//            step1.setParam(SparkColumnCardinality.OPTION_COLUMN_COUNT.getOpt(), String.valueOf(table.getColumnCount()));
+//            job.addTask(step1);
+//        } else {
+//            MapReduceExecutable step1 = new MapReduceExecutable();
+//            step1.setMapReduceJobClass(HiveColumnCardinalityJob.class);
+//            step1.setMapReduceParams(param);
+//            step1.setParam("segmentId", tableName);
+//            job.addTask(step1);
+//        }
+//
+//        HadoopShellExecutable step2 = new HadoopShellExecutable();
+//
+//        step2.setJobClass(HiveColumnCardinalityUpdateJob.class);
+//        step2.setJobParams(param);
+//        step2.setParam("segmentId", tableName);
+//        job.addTask(step2);
+//        tableExt.setJodID(job.getId());
+//        getTableManager().saveTableExt(tableExt, prj);
+//
+//        getExecutableManager().addJob(job);
+//    }
 
     public String normalizeHiveTableName(String tableName) {
         String[] dbTableName = HadoopUtil.parseHiveTableName(tableName);
diff --git a/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
index 5d7956b..eb5641d 100644
--- a/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
+++ b/server-base/src/test/java/org/apache/kylin/rest/job/StorageCleanupJobTest.java
@@ -27,7 +27,6 @@ import org.apache.kylin.common.util.TempMetadataBuilder;
 import org.apache.kylin.job.exception.SchedulerException;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 
diff --git a/server/ServiceConfig.md b/server/ServiceConfig.md
deleted file mode 100644
index 11c194d..0000000
--- a/server/ServiceConfig.md
+++ /dev/null
@@ -1,37 +0,0 @@
-### Service
-##### kylin.rest.servers 
-##### kylin.rest.timezone
-##### kylin.server.mode Valid value: 'query', 'job' and 'all'
-
-### Storage
-##### kylin.metadata.url
-##### kylin.storage.url 
-##### kylin.hdfs.working.dir
-##### kylin.hbase.coprocessor.jar 
-##### kylin.hbase.default.table.region.count 
-
-### Query
-##### kylin.query.scan.threshold
-##### kylin.query.security.enabled 
-##### kylin.query.cache.threshold.duration
-##### kylin.query.cache.threshold.scancount
-##### kylin.query.scan.thread.count
-##### kylin.query.cache.enabled
-
-### Job
-##### kylin.job.hdfs.working.dir
-##### kylin.job.mapreduce.default.reduce.input.mb
-##### kylin.job.mapreduce.default.reduce.count.ratio
-##### kylin.job.mapreduce.default.compress.output
-##### kylin.job.mapreduce.default.compress.codec
-##### kylin.job.run.as.remote.cmd
-##### kylin.job.remote.cli.hostname
-##### kylin.job.remote.cli.username
-##### kylin.job.remote.cli.password
-##### kylin.job.remote.cli.working.dir
-##### kylin.job.concurrent.max.limit
-##### kylin.job.command.exe.type
-##### kylin.job.admin.dls
-##### kylin.job.step.timeoutww
-##### kylin.job.yarn.app.rest.check.interval.seconds
-##### kylin.job.concurrent.max.limit
\ No newline at end of file
diff --git a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
index dcbe492..34fb8b4 100644
--- a/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
+++ b/server/src/test/java/org/apache/kylin/rest/service/AdminServiceTest.java
@@ -60,7 +60,7 @@ public class AdminServiceTest extends ServiceTestBase {
                     "kylin.web.timezone=\n" +
                     "kylin.server.external-acl-provider=\n" +
                     "kylin.tool.auto-migrate-cube.enabled=\n" +
-                    "kylin.storage.default=2\n" +
+                    "kylin.storage.default=4\n" +
                     "kylin.cube.cubeplanner.enabled=true\n" +
                     "kylin.web.help=\n" +
                     "kylin.web.export-allow-other=true\n" +
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveFlinkInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveFlinkInput.java
deleted file mode 100644
index f3dbf73..0000000
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveFlinkInput.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.source.hive;
-
-import com.google.common.collect.Lists;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.engine.flink.IFlinkInput;
-import org.apache.kylin.job.JoinedFlatTable;
-import org.apache.kylin.job.constant.ExecutableConstants;
-import org.apache.kylin.job.execution.AbstractExecutable;
-import org.apache.kylin.job.execution.DefaultChainedExecutable;
-import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
-import org.apache.kylin.metadata.model.ISegment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.List;
-
-/**
- * The implementation of {@link IFlinkInput} based on Hive.
- */
-public class HiveFlinkInput extends HiveInputBase implements IFlinkInput {
-
-    @SuppressWarnings("unused")
-    private static final Logger logger = LoggerFactory.getLogger(HiveFlinkInput.class);
-
-    @Override
-    public IFlinkBatchCubingInputSide getBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
-        return new BatchCubingInputSide(flatDesc);
-    }
-
-    @Override
-    public IFlinkBatchMergeInputSide getBatchMergeInputSide(ISegment seg) {
-        return new IFlinkBatchMergeInputSide() {
-            @Override
-            public void addStepPhase1_MergeDictionary(DefaultChainedExecutable jobFlow) {
-                // doing nothing
-            }
-        };
-    }
-
-    public class BatchCubingInputSide extends BaseBatchCubingInputSide implements IFlinkBatchCubingInputSide {
-
-        List<String> hiveViewIntermediateTables = Lists.newArrayList();
-
-        public BatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
-            super(flatDesc);
-        }
-
-        protected void addStepPhase1_DoMaterializeLookupTable(DefaultChainedExecutable jobFlow) {
-            final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase);
-            final String jobWorkingDir = getJobWorkingDir(jobFlow, hdfsWorkingDir);
-
-            AbstractExecutable task = createLookupHiveViewMaterializationStep(hiveInitStatements, jobWorkingDir,
-                    flatDesc, hiveViewIntermediateTables, jobFlow.getId());
-            if (task != null) {
-                jobFlow.addTask(task);
-            }
-        }
-
-        @Override
-        public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) {
-            final String jobWorkingDir = getJobWorkingDir(jobFlow, hdfsWorkingDir);
-
-            GarbageCollectionStep step = new GarbageCollectionStep();
-            step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP);
-            step.setIntermediateTables(Collections.singletonList(getIntermediateTableIdentity()));
-            step.setExternalDataPaths(Collections.singletonList(JoinedFlatTable.getTableDir(flatDesc, jobWorkingDir)));
-            step.setHiveViewIntermediateTableIdentities(StringUtil.join(hiveViewIntermediateTables, ","));
-            jobFlow.addTask(step);
-        }
-
-    }
-
-}
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
index de7319e..10c9f97 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveInputBase.java
@@ -43,10 +43,6 @@ import org.apache.kylin.cube.model.CubeDesc;
 import org.apache.kylin.engine.mr.IInput;
 import org.apache.kylin.engine.mr.JobBuilderSupport;
 import org.apache.kylin.engine.mr.steps.CubingExecutableUtil;
-import org.apache.kylin.engine.spark.SparkCreatingFlatTable;
-import org.apache.kylin.engine.spark.SparkExecutable;
-import org.apache.kylin.engine.spark.SparkExecutableFactory;
-import org.apache.kylin.engine.spark.SparkSqlBatch;
 import org.apache.kylin.job.JoinedFlatTable;
 import org.apache.kylin.job.common.ShellExecutable;
 import org.apache.kylin.job.constant.ExecutableConstants;
@@ -264,13 +260,6 @@ public class HiveInputBase {
                 if (kylinConfig.isLivyEnabled()) {
                     jobFlow.addTask(createFlatHiveTableByLivyStep(hiveInitStatements,
                             jobWorkingDir, cubeName, flatDesc));
-                } else {
-                    if (kylinConfig.isSparCreateHiveTableViaSparkEnable()) {
-                        jobFlow.addTask(createFlatHiveTableBySparkSql(hiveInitStatements,
-                                jobWorkingDir, cubeName, flatDesc));
-                    } else {
-                        jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements, jobWorkingDir, cubeName, flatDesc));
-                    }
                 }
             } else {
                 jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements, jobWorkingDir, cubeName, flatDesc));
@@ -358,52 +347,6 @@ public class HiveInputBase {
         return step;
     }
 
-    protected static AbstractExecutable createFlatHiveTableBySparkSql(String hiveInitStatements,
-            String jobWorkingDir, String cubeName, IJoinedFlatTableDesc flatDesc) {
-        final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
-        final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc,
-                jobWorkingDir);
-        String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
-
-        KylinConfig config = flatDesc.getSegment().getConfig();
-        final SparkExecutable sparkExecutable = SparkExecutableFactory.instance(config);
-        sparkExecutable.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK);
-        sparkExecutable.setClassName(SparkCreatingFlatTable.class.getName());
-
-        sparkExecutable.setParam(SparkSqlBatch.OPTION_CUBE_NAME.getOpt(), cubeName);
-        sparkExecutable.setParam(SparkSqlBatch.OPTION_STEP_NAME.getOpt(),
-                base64EncodeStr(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK));
-        sparkExecutable.setParam(SparkSqlBatch.OPTION_SEGMENT_ID.getOpt(),
-                flatDesc.getSegment().getName());
-        sparkExecutable.setParam(SparkSqlBatch.OPTION_SQL_COUNT.getOpt(),
-                String.valueOf(SparkCreatingFlatTable.SQL_COUNT));
-
-        sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(0).getOpt(),
-                base64EncodeStr(hiveInitStatements));
-        sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(1).getOpt(),
-                base64EncodeStr(dropTableHql));
-
-        // createTableHql include create table sql and alter table sql
-        String[] sqlArr = createTableHql.trim().split(";");
-        if (2 != sqlArr.length) {
-            throw new RuntimeException("create table hql should combined by a create table sql " +
-                    "and a alter sql, but got: " + createTableHql);
-        }
-        sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(2).getOpt(),
-                base64EncodeStr(sqlArr[0]));
-        sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(3).getOpt(),
-                base64EncodeStr(sqlArr[1]));
-
-        sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(4).getOpt(),
-                base64EncodeStr(insertDataHqls));
-
-        StringBuilder jars = new StringBuilder();
-        StringUtil.appendWithSeparator(jars, config.getSparkAdditionalJars());
-        sparkExecutable.setJars(jars.toString());
-
-        return sparkExecutable;
-    }
-
     private static String base64EncodeStr(String str) {
         return new String(
                 Base64.getEncoder().encode(str.getBytes(StandardCharsets.UTF_8)),
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java
index 353cf1d..c862063 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveSource.java
@@ -21,9 +21,6 @@ package org.apache.kylin.source.hive;
 import java.io.IOException;
 
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.engine.flink.IFlinkInput;
-import org.apache.kylin.engine.mr.IMRInput;
-import org.apache.kylin.engine.spark.ISparkInput;
 import org.apache.kylin.metadata.model.IBuildable;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.source.IReadableTable;
@@ -45,15 +42,7 @@ public class HiveSource implements ISource {
     @SuppressWarnings("unchecked")
     @Override
     public <I> I adaptToBuildEngine(Class<I> engineInterface) {
-        if (engineInterface == IMRInput.class) {
-            return (I) new HiveMRInput();
-        } else if (engineInterface == ISparkInput.class) {
-            return (I) new HiveSparkInput();
-        } else if (engineInterface == IFlinkInput.class) {
-            return (I) new HiveFlinkInput();
-        } else {
-            throw new RuntimeException("Cannot adapt to " + engineInterface);
-        }
+        throw new RuntimeException("Cannot adapt to " + engineInterface);
     }
 
     @Override
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
index 9212d08..4612cef 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMigrationCLI.java
@@ -66,7 +66,6 @@ import org.apache.kylin.metadata.realization.IRealizationConstants;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationType;
 import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.stream.core.source.StreamingSourceConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -339,10 +338,10 @@ public class CubeMigrationCLI extends AbstractApplication {
             metaResource.add(ACL_PREFIX + cube.getModel().getUuid());
         }
 
-        if (cubeDesc.isStreamingCube()) {
-            // add streaming source config info for streaming cube
-            metaResource.add(StreamingSourceConfig.concatResourcePath(cubeDesc.getModel().getRootFactTableName()));
-        }
+//        if (cubeDesc.isStreamingCube()) {
+//            // add streaming source config info for streaming cube
+//            metaResource.add(StreamingSourceConfig.concatResourcePath(cubeDesc.getModel().getRootFactTableName()));
+//        }
     }
 
     @Override
diff --git a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
index 9be3b49..2eec235 100644
--- a/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
+++ b/tool/src/main/java/org/apache/kylin/tool/DiagnosisInfoCLI.java
@@ -20,7 +20,6 @@ package org.apache.kylin.tool;
 
 import java.io.File;
 import java.io.IOException;
-import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
@@ -32,19 +31,18 @@ import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.ClassUtil;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.metadata.project.ProjectInstance;
 import org.apache.kylin.metadata.project.ProjectManager;
 import org.apache.kylin.common.util.ToolUtil;
-import org.apache.kylin.stream.core.util.NamedThreadFactory;
+import org.apache.kylin.tool.extractor.JobInstanceExtractor;
 import org.apache.kylin.tool.extractor.AbstractInfoExtractor;
-import org.apache.kylin.tool.extractor.ClientEnvExtractor;
 import org.apache.kylin.tool.extractor.CubeMetaExtractor;
-import org.apache.kylin.tool.extractor.JStackExtractor;
-import org.apache.kylin.tool.extractor.JobInstanceExtractor;
+import org.apache.kylin.tool.extractor.ClientEnvExtractor;
 import org.apache.kylin.tool.extractor.KylinLogExtractor;
 import org.apache.kylin.tool.extractor.SparkEnvInfoExtractor;
+import org.apache.kylin.tool.extractor.JStackExtractor;
+import org.apache.kylin.tool.extractor.NamedThreadFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -188,27 +186,6 @@ public class DiagnosisInfoCLI extends AbstractInfoExtractor {
             });
         }
 
-        // export HBase
-        if (includeHBase) {
-            executorService.execute(new Runnable() {
-                @Override
-                public void run() {
-                    logger.info("Start to extract HBase usage.");
-                    try {
-                        // use reflection to isolate NoClassDef errors when HBase is not available
-                        String[] hbaseArgs = {"-destDir", new File(exportDir, "hbase").getAbsolutePath(), "-project",
-                                projectNames, "-compress", "false", "-submodule", "true"};
-                        logger.info("HBaseUsageExtractor args: " + Arrays.toString(hbaseArgs));
-                        Object extractor = ClassUtil.newInstance("org.apache.kylin.tool.extractor.HBaseUsageExtractor");
-                        Method execute = extractor.getClass().getMethod("execute", String[].class);
-                        execute.invoke(extractor, (Object) hbaseArgs);
-                    } catch (Exception e) {
-                        logger.error("Error in export HBase usage.", e);
-                    }
-                }
-            });
-        }
-
         // export conf
         if (includeConf) {
             executorService.execute(new Runnable() {
diff --git a/tool/src/main/java/org/apache/kylin/tool/extractor/ClientEnvExtractor.java b/tool/src/main/java/org/apache/kylin/tool/extractor/ClientEnvExtractor.java
index 4c8597b..2b1b7e1 100644
--- a/tool/src/main/java/org/apache/kylin/tool/extractor/ClientEnvExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/extractor/ClientEnvExtractor.java
@@ -35,7 +35,6 @@ import org.apache.kylin.common.KylinConfig;
 import org.apache.kylin.common.util.CliCommandExecutor;
 import org.apache.kylin.common.util.OptionsHelper;
 import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.stream.core.util.NamedThreadFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/tool/src/main/java/org/apache/kylin/tool/extractor/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/extractor/CubeMetaExtractor.java
index 0789011..e5a8fd0 100644
--- a/tool/src/main/java/org/apache/kylin/tool/extractor/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/extractor/CubeMetaExtractor.java
@@ -20,7 +20,6 @@ package org.apache.kylin.tool.extractor;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -55,13 +54,9 @@ import org.apache.kylin.metadata.realization.IRealization;
 import org.apache.kylin.metadata.realization.RealizationRegistry;
 import org.apache.kylin.metadata.realization.RealizationStatusEnum;
 import org.apache.kylin.metadata.realization.RealizationType;
-import org.apache.kylin.metadata.streaming.StreamingConfig;
 import org.apache.kylin.metadata.streaming.StreamingManager;
-import org.apache.kylin.source.kafka.config.KafkaConfig;
 import org.apache.kylin.storage.hybrid.HybridInstance;
 import org.apache.kylin.storage.hybrid.HybridManager;
-import org.apache.kylin.stream.core.source.StreamingSourceConfig;
-import org.apache.kylin.stream.core.source.StreamingSourceConfigManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -132,7 +127,6 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
     private HybridManager hybridManager;
     private CubeManager cubeManager;
     private StreamingManager streamingManager;
-    private StreamingSourceConfigManager streamingSourceConfigManager;
     private CubeDescManager cubeDescManager;
     private ExecutableDao executableDao;
     private RealizationRegistry realizationRegistry;
@@ -198,7 +192,7 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
         executableDao = ExecutableDao.getInstance(kylinConfig);
         realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
         badQueryHistoryManager = BadQueryHistoryManager.getInstance(kylinConfig);
-        streamingSourceConfigManager = StreamingSourceConfigManager.getInstance(kylinConfig);
+//        streamingSourceConfigManager = StreamingSourceConfigManager.getInstance(kylinConfig);
 
         addRequired(ResourceStore.METASTORE_UUID_TAG);
 
@@ -348,9 +342,9 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
             // add tables
             addTables(modelDesc);
             // add streaming stuff
-            addStreamingConfig(cube);
+//            addStreamingConfig(cube);
             // add streamingV2
-            addStreamingV2Config(cube);
+//            addStreamingV2Config(cube);
             // add cube
             addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
             // add project
@@ -432,32 +426,32 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
         }
     }
 
-    private void addStreamingConfig(CubeInstance cube) {
-        streamingManager = StreamingManager.getInstance(kylinConfig);
-        for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) {
-            if (streamingConfig.getName() != null
-                    && streamingConfig.getName().equalsIgnoreCase(cube.getRootFactTable())) {
-                addRequired(StreamingConfig.concatResourcePath(streamingConfig.getName()));
-                addRequired(KafkaConfig.concatResourcePath(streamingConfig.getName()));
-            }
-        }
-    }
-
-    private void addStreamingV2Config(CubeInstance cube) {
-        Collection<StreamingSourceConfig> streamingConfigs;
-        try {
-            streamingConfigs = streamingSourceConfigManager.listAllStreaming();
-        } catch (IOException ioe) {
-            logger.error("", ioe);
-            return;
-        }
-        for (StreamingSourceConfig streamingConfig : streamingConfigs) {
-            if (streamingConfig.getName() != null
-                    && streamingConfig.getName().equalsIgnoreCase(cube.getRootFactTable())) {
-                addRequired(StreamingSourceConfig.concatResourcePath(streamingConfig.getName()));
-            }
-        }
-    }
+//    private void addStreamingConfig(CubeInstance cube) {
+//        streamingManager = StreamingManager.getInstance(kylinConfig);
+//        for (StreamingConfig streamingConfig : streamingManager.listAllStreaming()) {
+//            if (streamingConfig.getName() != null
+//                    && streamingConfig.getName().equalsIgnoreCase(cube.getRootFactTable())) {
+//                addRequired(StreamingConfig.concatResourcePath(streamingConfig.getName()));
+//                addRequired(KafkaConfig.concatResourcePath(streamingConfig.getName()));
+//            }
+//        }
+//    }
+
+//    private void addStreamingV2Config(CubeInstance cube) {
+//        Collection<StreamingSourceConfig> streamingConfigs;
+//        try {
+//            streamingConfigs = streamingSourceConfigManager.listAllStreaming();
+//        } catch (IOException ioe) {
+//            logger.error("", ioe);
+//            return;
+//        }
+//        for (StreamingSourceConfig streamingConfig : streamingConfigs) {
+//            if (streamingConfig.getName() != null
+//                    && streamingConfig.getName().equalsIgnoreCase(cube.getRootFactTable())) {
+//                addRequired(StreamingSourceConfig.concatResourcePath(streamingConfig.getName()));
+//            }
+//        }
+//    }
 
     private void addRequired(String record) {
         logger.info("adding required resource {}", record);
diff --git a/tool/src/main/java/org/apache/kylin/tool/extractor/HBaseUsageExtractor.java b/tool/src/main/java/org/apache/kylin/tool/extractor/HBaseUsageExtractor.java
deleted file mode 100644
index e545ba6..0000000
--- a/tool/src/main/java/org/apache/kylin/tool/extractor/HBaseUsageExtractor.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.tool.extractor;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Locale;
-
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
-import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.util.CliCommandExecutor;
-import org.apache.kylin.common.util.OptionsHelper;
-import org.apache.kylin.common.util.StringUtil;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.CubeSegment;
-import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.metadata.project.ProjectManager;
-import org.apache.kylin.metadata.project.RealizationEntry;
-import org.apache.kylin.metadata.realization.IRealization;
-import org.apache.kylin.metadata.realization.RealizationRegistry;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Lists;
-
-public class HBaseUsageExtractor extends AbstractInfoExtractor {
-
-    private static final Logger logger = LoggerFactory.getLogger(HBaseUsageExtractor.class);
-    @SuppressWarnings("static-access")
-    private static final Option OPTION_CUBE = OptionBuilder.withArgName("cube").hasArg().isRequired(false).withDescription("Specify which cube to extract").create("cube");
-    @SuppressWarnings("static-access")
-    private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
-
-    public static final String HDFS_CHECK_COMMAND = "hadoop fs -ls -R %s/data/%s/%s*";
-    private final String hbaseRootDir;
-    private final String cachedHMasterUrl;
-
-    private List<String> htables = Lists.newArrayList();
-    private Configuration conf;
-    private CubeManager cubeManager;
-    private RealizationRegistry realizationRegistry;
-    private KylinConfig kylinConfig;
-    private ProjectManager projectManager;
-
-    public HBaseUsageExtractor() {
-        super();
-        packageType = "hbase";
-
-        OptionGroup realizationOrProject = new OptionGroup();
-        realizationOrProject.addOption(OPTION_CUBE);
-        realizationOrProject.addOption(OPTION_PROJECT);
-        realizationOrProject.setRequired(true);
-
-        options.addOptionGroup(realizationOrProject);
-        conf = HBaseConfiguration.create();
-        hbaseRootDir = conf.get("hbase.rootdir");
-        cachedHMasterUrl = getHBaseMasterUrl();
-    }
-
-    public static void main(String[] args) {
-        HBaseUsageExtractor extractor = new HBaseUsageExtractor();
-        extractor.execute(args);
-    }
-
-    private String getHBaseMasterUrl() {
-        String host = conf.get("hbase.master.info.bindAddress");
-        if (host.equals("0.0.0.0")) {
-            try {
-                host = MasterAddressTracker.getMasterAddress(new ZooKeeperWatcher(conf, null, null)).getHostname();
-            } catch (IOException | KeeperException io) {
-                return null;
-            }
-        }
-        String port = conf.get("hbase.master.info.port");
-        return "http://" + host + ":" + port + "/";
-    }
-
-    @Override
-    protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
-        if (cachedHMasterUrl == null) {
-            return;
-        }
-        kylinConfig = KylinConfig.getInstanceFromEnv();
-        cubeManager = CubeManager.getInstance(kylinConfig);
-        realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
-        projectManager = ProjectManager.getInstance(kylinConfig);
-
-        if (optionsHelper.hasOption(OPTION_PROJECT)) {
-            String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
-            for (String projectName : StringUtil.splitByComma(projectNames)) {
-                ProjectInstance projectInstance = projectManager.getProject(projectName);
-                if (projectInstance == null) {
-                    throw new IllegalArgumentException("Project " + projectName + " does not exist");
-                }
-                List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
-                for (RealizationEntry realizationEntry : realizationEntries) {
-                    retrieveResourcePath(getRealization(realizationEntry));
-                }
-            }
-        } else if (optionsHelper.hasOption(OPTION_CUBE)) {
-            String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
-            for (String cubeName : StringUtil.splitByComma(cubeNames)) {
-                IRealization realization = cubeManager.getRealization(cubeName);
-                if (realization != null) {
-                    retrieveResourcePath(realization);
-                } else {
-                    throw new IllegalArgumentException("No cube found with name of " + cubeName);
-                }
-            }
-        }
-
-        extractCommonInfo(exportDir, kylinConfig);
-        extractHTables(exportDir);
-    }
-
-    private void extractHTables(File dest) throws IOException {
-        logger.info("These htables are going to be extracted:");
-        for (String htable : htables) {
-            logger.info("{} is required", htable);
-        }
-
-        File tableDir = new File(dest, "table");
-        FileUtils.forceMkdir(tableDir);
-
-        for (String htable : htables) {
-            try {
-                URL srcUrl = new URL(cachedHMasterUrl + "table.jsp?name=" + htable);
-                File destFile = new File(tableDir, htable + ".html");
-                FileUtils.copyURLToFile(srcUrl, destFile);
-            } catch (Exception e) {
-                logger.warn("HTable " + htable + "info fetch failed: ", e);
-            }
-        }
-    }
-
-    private void extractCommonInfo(File dest, KylinConfig config) throws IOException {
-        logger.info("The hbase master info/conf are going to be extracted...");
-        String hbaseNamespace = config.getHBaseStorageNameSpace();
-        String tableNamePrefix = config.getHBaseTableNamePrefix();
-
-        // hbase master page
-        try {
-            File masterDir = new File(dest, "master");
-            FileUtils.forceMkdir(masterDir);
-            URL srcMasterUrl = new URL(cachedHMasterUrl + "master-status");
-            File masterDestFile = new File(masterDir, "master-status.html");
-            FileUtils.copyURLToFile(srcMasterUrl, masterDestFile);
-        } catch (Exception e) {
-            logger.warn("HBase Master status fetch failed: ", e);
-        }
-
-        // hbase conf
-        try {
-            File confDir = new File(dest, "conf");
-            FileUtils.forceMkdir(confDir);
-            URL srcConfUrl = new URL(cachedHMasterUrl + "conf");
-            File destConfFile = new File(confDir, "hbase-conf.xml");
-            FileUtils.copyURLToFile(srcConfUrl, destConfFile);
-        } catch (Exception e) {
-            logger.warn("HBase conf fetch failed: ", e);
-        }
-
-        // hbase jmx
-        try {
-            File jmxDir = new File(dest, "jmx");
-            FileUtils.forceMkdir(jmxDir);
-            URL srcJmxUrl = new URL(cachedHMasterUrl + "jmx");
-            File jmxDestFile = new File(jmxDir, "jmx.html");
-            FileUtils.copyURLToFile(srcJmxUrl, jmxDestFile);
-        } catch (Exception e) {
-            logger.warn("HBase JMX fetch failed: ", e);
-        }
-
-        // dump page
-        try {
-            File dumpDir = new File(dest, "dump");
-            FileUtils.forceMkdir(dumpDir);
-            URL srcDumpUrl = new URL(cachedHMasterUrl + "dump");
-            File dumpDestFile = new File(dumpDir, "dump");
-            FileUtils.copyURLToFile(srcDumpUrl, dumpDestFile);
-        } catch (Exception e) {
-            logger.warn("HBase Dump fetch failed: ", e);
-        }
-
-        // hbase hdfs status
-        try {
-            File hdfsDir = new File(dest, "hdfs");
-            FileUtils.forceMkdir(hdfsDir);
-            CliCommandExecutor cliCommandExecutor = kylinConfig.getCliCommandExecutor();
-            String command = String.format(Locale.ROOT, HDFS_CHECK_COMMAND, hbaseRootDir, hbaseNamespace, tableNamePrefix);
-            logger.info("Execute command {}", command);
-            String output = cliCommandExecutor.execute(command).getSecond();
-            FileUtils.writeStringToFile(new File(hdfsDir, "hdfs-files.list"), output, Charset.defaultCharset());
-        } catch (Exception e) {
-            logger.warn("HBase hdfs status fetch failed: ", e);
-        }
-    }
-
-    private IRealization getRealization(RealizationEntry realizationEntry) {
-        return realizationRegistry.getRealization(realizationEntry.getType(), realizationEntry.getRealization());
-    }
-
-    private void retrieveResourcePath(IRealization realization) {
-
-        logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
-
-        if (realization instanceof CubeInstance) {
-            CubeInstance cube = (CubeInstance) realization;
-            for (CubeSegment segment : cube.getSegments()) {
-                addHTable(segment.getStorageLocationIdentifier());
-            }
-        } else {
-            logger.warn("Unknown realization type: {}", realization.getType());
-        }
-    }
-
-    private void addHTable(String record) {
-        logger.info("adding required resource {}", record);
-        htables.add(record);
-    }
-}
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java b/tool/src/main/java/org/apache/kylin/tool/extractor/NamedThreadFactory.java
similarity index 50%
rename from core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java
rename to tool/src/main/java/org/apache/kylin/tool/extractor/NamedThreadFactory.java
index f473059..fa0c6b9 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/IExtLookupTableCache.java
+++ b/tool/src/main/java/org/apache/kylin/tool/extractor/NamedThreadFactory.java
@@ -16,24 +16,30 @@
  * limitations under the License.
 */
 
-package org.apache.kylin.dict.lookup;
+package org.apache.kylin.tool.extractor;
 
-import org.apache.kylin.metadata.model.TableDesc;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
 
-public interface IExtLookupTableCache {
-    enum CacheState {NONE, IN_BUILDING, AVAILABLE}
+public class NamedThreadFactory implements ThreadFactory {
+    private static AtomicInteger threadNumber = new AtomicInteger(1);
+    private final String namePrefix;
+    private final boolean daemon;
 
-    /**
-     * @param tableDesc
-     * @param extTableSnapshotInfo
-     * @param buildIfNotExist if true, when the cached lookup table not exist, build it.
-     * @return null if no cached lookup table exist
-     */
-    ILookupTable getCachedLookupTable(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, boolean buildIfNotExist);
+    public NamedThreadFactory(String namePrefix, boolean daemon) {
+        this.namePrefix = namePrefix;
+        this.daemon = daemon;
 
-    void buildSnapshotCache(TableDesc tableDesc, ExtTableSnapshotInfo extTableSnapshotInfo, ILookupTable sourceTable);
+    }
 
-    void removeSnapshotCache(ExtTableSnapshotInfo extTableSnapshotInfo);
+    public NamedThreadFactory(String namePrefix) {
+        this(namePrefix, true);
+    }
+
+    public Thread newThread(Runnable runnable) {
+        final Thread thread = new Thread(runnable, namePrefix + "-thread-" + threadNumber.getAndIncrement());
+        thread.setDaemon(daemon);
+        return thread;
+    }
 
-    CacheState getCacheState(ExtTableSnapshotInfo extTableSnapshotInfo);
 }
diff --git a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java
index a2809c0..ac0fbc1 100644
--- a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java
+++ b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/SCCreator.java
@@ -48,10 +48,8 @@ import org.apache.kylin.metadata.TableMetadataManager;
 import org.apache.kylin.metadata.model.DataModelDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.project.ProjectInstance;
-import org.apache.kylin.stream.core.source.StreamingSourceConfig;
 import org.apache.kylin.tool.metrics.systemcube.def.MetricsSinkDesc;
 import org.apache.kylin.tool.metrics.systemcube.streamingv2.KafkaTopicCreator;
-import org.apache.kylin.tool.metrics.systemcube.streamingv2.StreamingMetadataCreator;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -134,7 +132,6 @@ public class SCCreator extends AbstractApplication {
         List<DataModelDesc> kylinModels = Lists.newArrayList();
         List<CubeDesc> cubeDescList = Lists.newArrayList();
         List<CubeInstance> kylinCubeInstances = Lists.newArrayList();
-        List<StreamingSourceConfig> streamingSourceConfigs = Lists.newArrayList();
 
 
         boolean hasHive = false;
@@ -151,11 +148,11 @@ public class SCCreator extends AbstractApplication {
             kylinCubeInstances.addAll(generateKylinCubeInstanceForSystemCube(owner, sinkDesc));
 
             if (sinkDesc.useKafka()) {
-                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsJob(config, sinkDesc));
-                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsJobException(config, sinkDesc));
-                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsQuery(config, sinkDesc));
-                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsQueryRpcCall(config, sinkDesc));
-                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsQueryCube(config, sinkDesc));
+//                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsJob(config, sinkDesc));
+//                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsJobException(config, sinkDesc));
+//                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsQuery(config, sinkDesc));
+//                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsQueryRpcCall(config, sinkDesc));
+//                streamingSourceConfigs.add(StreamingMetadataCreator.generateKylinTableForMetricsQueryCube(config, sinkDesc));
             }
         }
 
@@ -182,9 +179,9 @@ public class SCCreator extends AbstractApplication {
             generateKylinCubeInstanceFileForSystemCube(output, cubeInstance);
         }
 
-        for (StreamingSourceConfig sourceConfig : streamingSourceConfigs) {
-            generateKylinStreamingConfigForSystemCube(output, sourceConfig);
-        }
+//        for (StreamingSourceConfig sourceConfig : streamingSourceConfigs) {
+//            generateKylinStreamingConfigForSystemCube(output, sourceConfig);
+//        }
     }
 
     private List<TableDesc> generateKylinTableForSystemCube(MetricsSinkDesc sinkDesc) {
@@ -251,10 +248,10 @@ public class SCCreator extends AbstractApplication {
                 ModelCreator.MODELDESC_SERIALIZER);
     }
 
-    private void generateKylinStreamingConfigForSystemCube(String output, StreamingSourceConfig streamingConfig) throws IOException {
-        saveSystemCubeMetadataToFile(output + D_STREAMING_V2 + streamingConfig.getName() + ".json", streamingConfig,
-                StreamingMetadataCreator.STREAMING_SOURCE_CONFIG_SERIALIZER);
-    }
+//    private void generateKylinStreamingConfigForSystemCube(String output, StreamingSourceConfig streamingConfig) throws IOException {
+//        saveSystemCubeMetadataToFile(output + D_STREAMING_V2 + streamingConfig.getName() + ".json", streamingConfig,
+//                StreamingMetadataCreator.STREAMING_SOURCE_CONFIG_SERIALIZER);
+//    }
 
     private void generateKylinCubeInstanceFileForSystemCube(String output, CubeInstance cubeInstance)
             throws IOException {
diff --git a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/streamingv2/StreamingMetadataCreator.java b/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/streamingv2/StreamingMetadataCreator.java
deleted file mode 100644
index 93121ce..0000000
--- a/tool/src/main/java/org/apache/kylin/tool/metrics/systemcube/streamingv2/StreamingMetadataCreator.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.tool.metrics.systemcube.streamingv2;
-
-import com.google.common.collect.Lists;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.JsonSerializer;
-import org.apache.kylin.common.persistence.Serializer;
-import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.metrics.lib.impl.kafka.KafkaReservoirReporter;
-import org.apache.kylin.stream.core.source.MessageParserInfo;
-import org.apache.kylin.stream.core.source.StreamingSourceConfig;
-import org.apache.kylin.tool.metrics.systemcube.HiveTableCreator;
-import org.apache.kylin.tool.metrics.systemcube.def.MetricsSinkDesc;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class StreamingMetadataCreator {
-
-    public static final Serializer<StreamingSourceConfig> STREAMING_SOURCE_CONFIG_SERIALIZER = new JsonSerializer<>(StreamingSourceConfig.class);
-
-
-    public static StreamingSourceConfig generateKylinTableForMetricsQuery(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc) {
-        List<Pair<String, String>> columns = Lists.newLinkedList();
-        columns.addAll(HiveTableCreator.getHiveColumnsForMetricsQuery());
-        return generateStreamingV2Config(kylinConfig, sinkDesc, kylinConfig.getKylinMetricsSubjectQuery(), columns);
-    }
-
-    public static StreamingSourceConfig generateKylinTableForMetricsQueryCube(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc) {
-        List<Pair<String, String>> columns = Lists.newLinkedList();
-        columns.addAll(HiveTableCreator.getHiveColumnsForMetricsQueryCube());
-        return generateStreamingV2Config(kylinConfig, sinkDesc, kylinConfig.getKylinMetricsSubjectQueryCube(), columns);
-    }
-
-    public static StreamingSourceConfig generateKylinTableForMetricsQueryRpcCall(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc) {
-        List<Pair<String, String>> columns = Lists.newLinkedList();
-        columns.addAll(HiveTableCreator.getHiveColumnsForMetricsQueryRPC());
-        return generateStreamingV2Config(kylinConfig, sinkDesc, kylinConfig.getKylinMetricsSubjectQueryRpcCall(), columns);
-    }
-
-    public static StreamingSourceConfig generateKylinTableForMetricsJob(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc) {
-        List<Pair<String, String>> columns = Lists.newLinkedList();
-        columns.addAll(HiveTableCreator.getHiveColumnsForMetricsJob());
-        return generateStreamingV2Config(kylinConfig, sinkDesc, kylinConfig.getKylinMetricsSubjectJob(), columns);
-    }
-
-    public static StreamingSourceConfig generateKylinTableForMetricsJobException(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc) {
-        List<Pair<String, String>> columns = Lists.newLinkedList();
-        columns.addAll(HiveTableCreator.getHiveColumnsForMetricsJobException());
-        return generateStreamingV2Config(kylinConfig, sinkDesc, kylinConfig.getKylinMetricsSubjectJobException(), columns);
-    }
-
-    private static StreamingSourceConfig generateStreamingV2Config(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc, String subject, List<Pair<String, String>> columns) {
-        StreamingSourceConfig streamingSourceConfig = new StreamingSourceConfig();
-        MessageParserInfo parserInfo = new MessageParserInfo();
-        parserInfo.setFormatTs(false);
-        parserInfo.setTsColName("KTIMESTAMP");
-        parserInfo.setTsPattern("MS");
-        parserInfo.setTsParser("org.apache.kylin.stream.source.kafka.LongTimeParser");
-        Map<String, String> columnToSourceFieldMapping = new HashMap<>();
-        for (Pair<String, String> col : columns) {
-            columnToSourceFieldMapping.put(col.getKey(), col.getKey());
-        }
-        parserInfo.setColumnToSourceFieldMapping(columnToSourceFieldMapping);
-
-        Map<String, String> properties = new HashMap<>();
-        String topic = KafkaReservoirReporter.decorateTopic(subject);
-        String table = KafkaReservoirReporter.sink.getTableFromSubject(subject);
-
-        properties.put("topic", topic);
-        properties.put("bootstrap.servers", sinkDesc.getTableProperties().get("bootstrap.servers"));
-
-        streamingSourceConfig.setName(table);
-        streamingSourceConfig.setProperties(properties);
-        streamingSourceConfig.setParserInfo(parserInfo);
-
-        streamingSourceConfig.updateRandomUuid();
-        streamingSourceConfig.setLastModified(System.currentTimeMillis());
-
-        return streamingSourceConfig;
-    }
-
-}