You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/02/08 06:03:45 UTC

[8/8] kylin git commit: KYLIN-2428 cleanup duplicated shaded libraries

KYLIN-2428 cleanup duplicated shaded libraries


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6d5ed7ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6d5ed7ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6d5ed7ec

Branch: refs/heads/KYLIN-2428
Commit: 6d5ed7ec16308ec2a2c910710449d0b79e41da84
Parents: 39afa51
Author: Billy Liu <bi...@apache.org>
Authored: Wed Feb 8 14:03:23 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Wed Feb 8 14:03:23 2017 +0800

----------------------------------------------------------------------
 atopcalcite/pom.xml                             |   1 +
 build/bin/kylin.sh                              |   4 +-
 core-common/pom.xml                             |  22 +-
 core-cube/pom.xml                               |  18 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 -------------------
 .../gridtable/GTInvertedIndexOfColumn.java      | 133 -----------
 .../gridtable/SimpleInvertedIndexTest.java      | 196 ----------------
 core-job/pom.xml                                |   9 +
 core-metadata/pom.xml                           |  23 ++
 core-storage/pom.xml                            |  20 ++
 engine-spark/pom.xml                            |   1 +
 jdbc/pom.xml                                    |  14 +-
 pom.xml                                         | 214 ++++++++++--------
 query/pom.xml                                   |   6 +
 server/pom.xml                                  |   4 +
 source-kafka/pom.xml                            |  16 +-
 storage-hbase/pom.xml                           |   4 -
 tool/pom.xml                                    |  33 +++
 18 files changed, 251 insertions(+), 690 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop -->
         <dependency>
             <groupId>org.apache.calcite.avatica</groupId>
             <artifactId>avatica</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index 7813b79..abaca62 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,8 +38,8 @@ function retrieveDependency() {
         source ${dir}/setenv.sh
     fi
 
-    export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
-    export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
+    export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+    export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}:${KYLIN_HOME}/tool/*
     if [ -n "$KAFKA_HOME" ]
     then
         source ${dir}/find-kafka-dependency.sh

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
     <dependencies>
         <!-- Basic Utilities -->
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-lang</groupId>
-            <artifactId>commons-lang</artifactId>
-        </dependency>
-        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
         <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-cli</groupId>
-            <artifactId>commons-cli</artifactId>
-        </dependency>
-        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-email</artifactId>
         </dependency>
@@ -63,9 +47,11 @@
             <artifactId>httpclient</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
         </dependency>
+
+        <!-- Provided -->
         <dependency>
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
             <groupId>commons-collections</groupId>
             <artifactId>commons-collections</artifactId>
         </dependency>
+
+        <!-- Provided -->
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
         <dependency>
-            <groupId>com.n3twork.druid</groupId>
-            <artifactId>extendedset</artifactId>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
         </dependency>
         <dependency>
             <groupId>com.esotericsoftware</groupId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
deleted file mode 100644
index a2c713a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * A thread-safe inverted index of row blocks in memory.
- * 
- * Note function not() must return all blocks, because index only know what block contains a value,
- * but not sure what block does not contain a value.
- * 
- * @author yangli9
- */
-public class GTInvertedIndex {
-
-    private final GTInfo info;
-    private final ImmutableBitSet colPreferIndex;
-    private final ImmutableBitSet colBlocks;
-    private final GTInvertedIndexOfColumn[] index; // for each column
-
-    private volatile int nIndexedBlocks;
-
-    public GTInvertedIndex(GTInfo info) {
-        this.info = info;
-        this.colPreferIndex = info.colPreferIndex;
-        this.colBlocks = info.selectColumnBlocks(colPreferIndex);
-
-        index = new GTInvertedIndexOfColumn[info.getColumnCount()];
-        for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-            int c = colPreferIndex.trueBitAt(i);
-            index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator());
-        }
-    }
-
-    public void add(GTRowBlock block) {
-
-        @SuppressWarnings("unchecked")
-        Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
-        for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-            int c = colPreferIndex.trueBitAt(i);
-            distinctValues[c] = new HashSet<ByteArray>();
-        }
-
-        GTRowBlock.Reader reader = block.getReader(colBlocks);
-        GTRecord record = new GTRecord(info);
-        while (reader.hasNext()) {
-            reader.fetchNext(record);
-            for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-                int c = colPreferIndex.trueBitAt(i);
-                distinctValues[c].add(record.get(c));
-            }
-        }
-
-        for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-            int c = colPreferIndex.trueBitAt(i);
-            index[c].add(distinctValues[c], block.getSequenceId());
-        }
-
-        nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
-    }
-
-    public ConciseSet filter(TupleFilter filter) {
-        return filter(filter, nIndexedBlocks);
-    }
-
-    public ConciseSet filter(TupleFilter filter, int totalBlocks) {
-        // number of indexed blocks may increase as we do evaluation
-        int indexedBlocks = nIndexedBlocks;
-
-        Evaluator evaluator = new Evaluator(indexedBlocks);
-        ConciseSet r = evaluator.evaluate(filter);
-
-        // add blocks that have not been indexed
-        for (int i = indexedBlocks; i < totalBlocks; i++) {
-            r.add(i);
-        }
-
-        return r;
-    }
-
-    private class Evaluator {
-        private int indexedBlocks;
-
-        Evaluator(int indexedBlocks) {
-            this.indexedBlocks = indexedBlocks;
-        }
-
-        public ConciseSet evaluate(TupleFilter filter) {
-            if (filter == null) {
-                return all();
-            }
-
-            if (filter instanceof LogicalTupleFilter)
-                return evalLogical((LogicalTupleFilter) filter);
-
-            if (filter instanceof CompareTupleFilter)
-                return evalCompare((CompareTupleFilter) filter);
-
-            // unable to evaluate
-            return all();
-        }
-
-        @SuppressWarnings("unchecked")
-        private ConciseSet evalCompare(CompareTupleFilter filter) {
-            int col = col(filter);
-            if (index[col] == null)
-                return all();
-
-            switch (filter.getOperator()) {
-            case ISNULL:
-                return index[col].getNull();
-            case ISNOTNULL:
-                return all();
-            case EQ:
-                return index[col].getEquals((ByteArray) filter.getFirstValue());
-            case NEQ:
-                return all();
-            case IN:
-                return index[col].getIn((Iterable<ByteArray>) filter.getValues());
-            case NOTIN:
-                return all();
-            case LT:
-                return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false);
-            case LTE:
-                return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true);
-            case GT:
-                return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false);
-            case GTE:
-                return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false);
-            default:
-                throw new IllegalStateException("Unsupported operator " + filter.getOperator());
-            }
-        }
-
-        private ConciseSet evalLogical(LogicalTupleFilter filter) {
-            List<? extends TupleFilter> children = filter.getChildren();
-
-            switch (filter.getOperator()) {
-            case AND:
-                return evalLogicalAnd(children);
-            case OR:
-                return evalLogicalOr(children);
-            case NOT:
-                return evalLogicalNot(children);
-            default:
-                throw new IllegalStateException("Unsupported operator " + filter.getOperator());
-            }
-        }
-
-        private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) {
-            ConciseSet set = all();
-
-            for (TupleFilter c : children) {
-                ConciseSet t = evaluate(c);
-                if (t == null)
-                    continue; // because it's AND
-
-                set.retainAll(t);
-            }
-            return set;
-        }
-
-        private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) {
-            ConciseSet set = new ConciseSet();
-
-            for (TupleFilter c : children) {
-                ConciseSet t = evaluate(c);
-                if (t == null)
-                    return null; // because it's OR
-
-                set.addAll(t);
-            }
-            return set;
-        }
-
-        private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) {
-            return all();
-        }
-
-        private ConciseSet all() {
-            return not(new ConciseSet());
-        }
-
-        private ConciseSet not(ConciseSet set) {
-            set.add(indexedBlocks);
-            set.complement();
-            return set;
-        }
-
-        private int col(CompareTupleFilter filter) {
-            return filter.getColumn().getColumnDesc().getZeroBasedIndex();
-        }
-
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
deleted file mode 100644
index bfacc0f..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.NavigableMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import com.google.common.collect.Maps;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class GTInvertedIndexOfColumn {
-
-    final private IGTComparator comparator;
-    final private ReentrantReadWriteLock rwLock;
-
-    private int nBlocks;
-    private NavigableMap<ByteArray, ConciseSet> rangeIndex;
-    private ConciseSet nullIndex;
-
-    public GTInvertedIndexOfColumn(IGTComparator comparator) {
-        this.comparator = comparator;
-        this.rwLock = new ReentrantReadWriteLock();
-        this.rangeIndex = Maps.newTreeMap(comparator);
-        this.nullIndex = new ConciseSet();
-    }
-
-    public void add(Iterable<ByteArray> codes, int blockId) {
-        rwLock.writeLock().lock();
-        try {
-            for (ByteArray code : codes) {
-                if (comparator.isNull(code)) {
-                    nullIndex.add(blockId);
-                    continue;
-                }
-                ConciseSet set = rangeIndex.get(code);
-                if (set == null) {
-                    set = new ConciseSet();
-                    rangeIndex.put(code.copy(), set);
-                }
-                set.add(blockId);
-            }
-
-            if (blockId >= nBlocks) {
-                nBlocks = blockId + 1;
-            }
-
-        } finally {
-            rwLock.writeLock().unlock();
-        }
-    }
-
-    public ConciseSet getNull() {
-        rwLock.readLock().lock();
-        try {
-            return nullIndex.clone();
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-
-    public ConciseSet getEquals(ByteArray code) {
-        rwLock.readLock().lock();
-        try {
-            ConciseSet set = rangeIndex.get(code);
-            if (set == null)
-                return new ConciseSet();
-            else
-                return set.clone();
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-
-    public ConciseSet getIn(Iterable<ByteArray> codes) {
-        rwLock.readLock().lock();
-        try {
-            ConciseSet r = new ConciseSet();
-            for (ByteArray code : codes) {
-                ConciseSet set = rangeIndex.get(code);
-                if (set != null)
-                    r.addAll(set);
-            }
-            return r;
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-
-    public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) {
-        rwLock.readLock().lock();
-        try {
-            ConciseSet r = new ConciseSet();
-            if (from == null && to == null) {
-                r.add(nBlocks);
-                r.complement();
-                return r;
-            }
-            NavigableMap<ByteArray, ConciseSet> subMap;
-            if (from == null) {
-                subMap = rangeIndex.headMap(to, toInclusive);
-            } else if (to == null) {
-                subMap = rangeIndex.tailMap(from, fromInclusive);
-            } else {
-                subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive);
-            }
-            for (ConciseSet set : subMap.values()) {
-                r.addAll(set);
-            }
-            return r;
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
deleted file mode 100644
index 5e3e771..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
-
-    GTInfo info;
-    GTInvertedIndex index;
-    ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
-    ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
-    @BeforeClass
-    public static void setUp() throws Exception {
-        staticCreateTestMetadata();
-    }
-
-    @AfterClass
-    public static void after() throws Exception {
-        cleanAfterClass();
-    }
-
-    public SimpleInvertedIndexTest() {
-
-        info = UnitTestSupport.advancedInfo();
-        TblColRef colA = info.colRef(0);
-
-        // block i contains value "i", the last is NULL
-        index = new GTInvertedIndex(info);
-        GTRowBlock mockBlock = GTRowBlock.allocate(info);
-        GTRowBlock.Writer writer = mockBlock.getWriter();
-        GTRecord record = new GTRecord(info);
-        for (int i = 0; i < 10; i++) {
-            record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
-            for (int j = 0; j < info.getRowBlockSize(); j++) {
-                writer.append(record);
-            }
-            writer.readyForFlush();
-            index.add(mockBlock);
-
-            writer.clearForNext();
-        }
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
-        basicResults.add(set(9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
-        basicResults.add(set(0));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
-        basicResults.add(set(0, 5));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
-        basicResults.add(set(0, 1, 2));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
-        basicResults.add(set(0, 1, 2, 3));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
-        basicResults.add(set(4, 5, 6, 7, 8));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
-        basicResults.add(set(3, 4, 5, 6, 7, 8));
-    }
-
-    @Test
-    public void testBasics() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
-        }
-    }
-
-    @Test
-    public void testLogicalAnd() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            for (int j = 0; j < basicFilters.size(); j++) {
-                LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
-                ConciseSet r = basicResults.get(i).clone();
-                r.retainAll(basicResults.get(j));
-                assertEquals(r, index.filter(f));
-            }
-        }
-    }
-
-    @Test
-    public void testLogicalOr() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            for (int j = 0; j < basicFilters.size(); j++) {
-                LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
-                ConciseSet r = basicResults.get(i).clone();
-                r.addAll(basicResults.get(j));
-                assertEquals(r, index.filter(f));
-            }
-        }
-    }
-
-    @Test
-    public void testNotEvaluable() {
-        ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
-        CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
-        assertEquals(all, index.filter(notEvaluable));
-
-        LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
-        assertEquals(all, index.filter(or));
-
-        LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
-        assertEquals(basicResults.get(0), index.filter(and));
-    }
-
-    public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
-        CompareTupleFilter filter = new CompareTupleFilter(op);
-        filter.addChild(columnFilter(col));
-        for (int i : ids) {
-            filter.addChild(constFilter(i));
-        }
-        return filter;
-    }
-
-    public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
-        LogicalTupleFilter filter = new LogicalTupleFilter(op);
-        for (TupleFilter f : filters)
-            filter.addChild(f);
-        return filter;
-    }
-
-    public static ColumnTupleFilter columnFilter(TblColRef col) {
-        return new ColumnTupleFilter(col);
-    }
-
-    public static ConstantTupleFilter constFilter(int id) {
-        byte[] space = new byte[10];
-        ByteBuffer buf = ByteBuffer.wrap(space);
-        StringSerializer stringSerializer = new StringSerializer(DataType.getType("string"));
-        stringSerializer.serialize("" + id, buf);
-        ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
-        return new ConstantTupleFilter(data);
-    }
-
-    public static ConciseSet set(int... ints) {
-        ConciseSet set = new ConciseSet();
-        for (int i : ints)
-            set.add(i);
-        return set;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 36d34c8..9478555 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -39,6 +39,15 @@
             <artifactId>kylin-core-cube</artifactId>
         </dependency>
 
+        <!-- Provided -->
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 87c4438..0832e80 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -39,6 +39,29 @@
             <artifactId>kylin-core-common</artifactId>
         </dependency>
 
+        <!-- Provided -->
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
+
+        <!-- Compiled -->
         <dependency>
             <groupId>net.sf.ehcache</groupId>
             <artifactId>ehcache</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 9bd3f04..8c3da5f 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -39,8 +39,28 @@
             <artifactId>kylin-core-cube</artifactId>
         </dependency>
 
+        <!-- Provided -->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
+
         <!-- Env & Test -->
         <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>org.apache.kylin</groupId>
             <artifactId>kylin-core-common</artifactId>
             <type>test-jar</type>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index be2cdf6..fe6d998 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -70,6 +70,7 @@
         <dependency>
             <groupId>org.reflections</groupId>
             <artifactId>reflections</artifactId>
+            <scope>provided</scope>
         </dependency>
 
         <!-- Hadoop dependency -->

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 874ead6..02890a6 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -43,7 +43,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.calcite.avatica</groupId>
-            <artifactId>avatica</artifactId>
+            <artifactId>avatica-core</artifactId>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -69,9 +69,7 @@
 
                             <artifactSet>
                                 <excludes>
-                                    <exclude>com.google.protobuf:*</exclude>
-                                    <exclude>commons-logging:*</exclude>
-                                    <exclude>commons-codec:*</exclude>
+                                    <exclude>org.slf4j:jcl-over-slf4j:*</exclude>
                                 </excludes>
                             </artifactSet>
 
@@ -85,9 +83,17 @@
                                     <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
                                 </relocation>
                                 <relocation>
+                                    <pattern>com.google.protobuf</pattern>
+                                    <shadedPattern>${shadeBase}.com.google.protobuf</shadedPattern>
+                                </relocation>
+                                <relocation>
                                     <pattern>org.apache.http</pattern>
                                     <shadedPattern>${shadeBase}.org.apache.http</shadedPattern>
                                 </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons</pattern>
+                                    <shadedPattern>${shadeBase}.org.apache.commons</shadedPattern>
+                                </relocation>
                             </relocations>
                             <filters>
                                 <filter>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b82eee2..b109783 100644
--- a/pom.xml
+++ b/pom.xml
@@ -55,15 +55,32 @@
 
         <!-- HBase versions -->
         <hbase-hadoop2.version>1.1.1</hbase-hadoop2.version>
+
+        <!-- Kafka versions -->
         <kafka.version>0.10.1.0</kafka.version>
 
-        <!-- Hadoop deps, keep compatible with hadoop2.version -->
+        <!-- Spark versions -->
+        <spark.version>1.6.3</spark.version>
+        <kryo.version>4.0.0</kryo.version>
+        <reflections.version>0.9.10</reflections.version>
+
+        <!-- Calcite Version -->
+        <calcite.version>1.11.0</calcite.version>
+        <avatica.version>1.9.0</avatica.version>
+
+        <!-- Hadoop Common deps, keep compatible with hadoop2.version -->
         <zookeeper.version>3.4.6</zookeeper.version>
         <curator.version>2.7.1</curator.version>
-        <jackson.version>2.2.4</jackson.version>
         <jsr305.version>3.0.1</jsr305.version>
         <guava.version>14.0</guava.version>
+        <jsch.version>0.1.53</jsch.version>
         <commons-cli.version>1.2</commons-cli.version>
+        <commons-lang.version>2.6</commons-lang.version>
+        <commons-io.version>2.5</commons-io.version>
+        <commons-math3.version>3.6.1</commons-math3.version>
+
+        <!-- Calcite deps, keep compatible with calcite.version -->
+        <jackson.version>2.6.3</jackson.version>
 
         <!-- Test Dependency versions -->
         <antlr.version>3.4</antlr.version>
@@ -75,26 +92,16 @@
         <jamm.version>0.3.1</jamm.version>
 
         <!-- Commons -->
-        <commons-lang.version>2.6</commons-lang.version>
         <commons-lang3.version>3.4</commons-lang3.version>
         <commons-collections.version>3.2.2</commons-collections.version>
-        <commons-io.version>2.5</commons-io.version>
         <commons-daemon.version>1.0.15</commons-daemon.version>
         <commons-email.version>1.4</commons-email.version>
-        <commons-math3.version>3.6.1</commons-math3.version>
-
-        <!-- Spark -->
-        <spark.version>1.6.3</spark.version>
 
         <!-- Utility -->
         <log4j.version>1.2.17</log4j.version>
         <slf4j.version>1.7.21</slf4j.version>
-        <reflections.version>0.9.10</reflections.version>
-        <jsch.version>0.1.53</jsch.version>
         <xerces.version>2.11.0</xerces.version>
         <xalan.version>2.7.2</xalan.version>
-        <extendedset.version>1.3.4</extendedset.version>
-        <kryo.version>4.0.0</kryo.version>
         <ehcache.version>2.10.2.2.21</ehcache.version>
         <apache-httpclient.version>4.2.5</apache-httpclient.version>
         <roaring.version>0.6.18</roaring.version>
@@ -111,10 +118,6 @@
         <opensaml.version>2.6.4</opensaml.version>
         <aspectj.version>1.8.9</aspectj.version>
 
-        <!-- Calcite Version -->
-        <calcite.version>1.11.0</calcite.version>
-        <avatica.version>1.9.0</avatica.version>
-
         <!-- Sonar -->
         <sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
         <sonar.dynamicAnalysis>reuseReports</sonar.dynamicAnalysis>
@@ -390,6 +393,66 @@
                 <version>${hadoop2.version}</version>
                 <optional>true</optional>
             </dependency>
+            <dependency>
+                <groupId>org.apache.zookeeper</groupId>
+                <artifactId>zookeeper</artifactId>
+                <version>${zookeeper.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-framework</artifactId>
+                <version>${curator.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-recipes</artifactId>
+                <version>${curator.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>com.google.code.findbugs</groupId>
+                <artifactId>jsr305</artifactId>
+                <version>${jsr305.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>com.google.guava</groupId>
+                <artifactId>guava</artifactId>
+                <version>${guava.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>com.jcraft</groupId>
+                <artifactId>jsch</artifactId>
+                <version>${jsch.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>commons-cli</groupId>
+                <artifactId>commons-cli</artifactId>
+                <version>${commons-cli.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>commons-lang</groupId>
+                <artifactId>commons-lang</artifactId>
+                <version>${commons-lang.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-math3</artifactId>
+                <version>${commons-math3.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>commons-io</groupId>
+                <artifactId>commons-io</artifactId>
+                <version>${commons-io.version}</version>
+                <scope>provided</scope>
+            </dependency>
 
             <!-- HBase2 dependencies -->
             <dependency>
@@ -461,6 +524,11 @@
             </dependency>
             <dependency>
                 <groupId>org.apache.calcite.avatica</groupId>
+                <artifactId>avatica-core</artifactId>
+                <version>${avatica.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.apache.calcite.avatica</groupId>
                 <artifactId>avatica</artifactId>
                 <version>${avatica.version}</version>
             </dependency>
@@ -476,23 +544,45 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
+
             <!-- Spark dependency -->
             <dependency>
                 <groupId>org.apache.spark</groupId>
                 <artifactId>spark-core_2.10</artifactId>
                 <version>${spark.version}</version>
+                <scope>provided</scope>
             </dependency>
             <dependency>
                 <groupId>org.apache.spark</groupId>
                 <artifactId>spark-sql_2.10</artifactId>
                 <version>${spark.version}</version>
+                <scope>provided</scope>
             </dependency>
             <dependency>
                 <groupId>org.apache.spark</groupId>
                 <artifactId>spark-hive_2.10</artifactId>
                 <version>${spark.version}</version>
+                <scope>provided</scope>
             </dependency>
             <dependency>
+                <groupId>com.esotericsoftware</groupId>
+                <artifactId>kryo-shaded</artifactId>
+                <version>${kryo.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <dependency>
+                <groupId>org.reflections</groupId>
+                <artifactId>reflections</artifactId>
+                <version>${reflections.version}</version>
+                <scope>provided</scope>
+            </dependency>
+            <!-- Kafka dependency -->
+            <dependency>
                 <groupId>org.apache.kafka</groupId>
                 <artifactId>kafka_2.10</artifactId>
                 <version>${kafka.version}</version>
@@ -501,26 +591,6 @@
 
             <!-- Other dependencies -->
             <dependency>
-                <groupId>junit</groupId>
-                <artifactId>junit</artifactId>
-                <version>${junit.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.zookeeper</groupId>
-                <artifactId>zookeeper</artifactId>
-                <version>${zookeeper.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>commons-cli</groupId>
-                <artifactId>commons-cli</artifactId>
-                <version>${commons-cli.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>commons-lang</groupId>
-                <artifactId>commons-lang</artifactId>
-                <version>${commons-lang.version}</version>
-            </dependency>
-            <dependency>
                 <groupId>org.apache.commons</groupId>
                 <artifactId>commons-lang3</artifactId>
                 <version>${commons-lang3.version}</version>
@@ -531,21 +601,15 @@
                 <version>${commons-collections.version}</version>
             </dependency>
             <dependency>
-                <groupId>org.apache.commons</groupId>
-                <artifactId>commons-math3</artifactId>
-                <version>${commons-math3.version}</version>
-            </dependency>
-
-            <dependency>
-                <groupId>commons-io</groupId>
-                <artifactId>commons-io</artifactId>
-                <version>${commons-io.version}</version>
-            </dependency>
-            <dependency>
                 <groupId>commons-daemon</groupId>
                 <artifactId>commons-daemon</artifactId>
                 <version>${commons-daemon.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.apache.commons</groupId>
+                <artifactId>commons-email</artifactId>
+                <version>${commons-email.version}</version>
+            </dependency>
 
             <!-- Logging -->
             <dependency>
@@ -571,40 +635,18 @@
                 <version>${slf4j.version}</version>
             </dependency>
 
+            <!-- Test -->
             <dependency>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-databind</artifactId>
-                <version>${jackson.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.commons</groupId>
-                <artifactId>commons-email</artifactId>
-                <version>${commons-email.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.google.code.findbugs</groupId>
-                <artifactId>jsr305</artifactId>
-                <version>${jsr305.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.google.guava</groupId>
-                <artifactId>guava</artifactId>
-                <version>${guava.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.reflections</groupId>
-                <artifactId>reflections</artifactId>
-                <version>${reflections.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.jcraft</groupId>
-                <artifactId>jsch</artifactId>
-                <version>${jsch.version}</version>
+                <groupId>junit</groupId>
+                <artifactId>junit</artifactId>
+                <version>${junit.version}</version>
+                <scope>test</scope>
             </dependency>
             <dependency>
                 <groupId>org.dbunit</groupId>
                 <artifactId>dbunit</artifactId>
                 <version>${dbunit.version}</version>
+                <scope>test</scope>
             </dependency>
             <dependency>
                 <groupId>org.apache.maven</groupId>
@@ -615,6 +657,7 @@
                 <groupId>com.h2database</groupId>
                 <artifactId>h2</artifactId>
                 <version>${h2.version}</version>
+                <scope>test</scope>
             </dependency>
             <dependency>
                 <groupId>xerces</groupId>
@@ -627,31 +670,10 @@
                 <version>${xalan.version}</version>
             </dependency>
             <dependency>
-                <groupId>com.n3twork.druid</groupId>
-                <artifactId>extendedset</artifactId>
-                <version>${extendedset.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>com.esotericsoftware</groupId>
-                <artifactId>kryo-shaded</artifactId>
-                <version>${kryo.version}</version>
-            </dependency>
-            <dependency>
                 <groupId>com.github.jbellis</groupId>
                 <artifactId>jamm</artifactId>
                 <version>${jamm.version}</version>
             </dependency>
-
-            <dependency>
-                <groupId>org.apache.curator</groupId>
-                <artifactId>curator-framework</artifactId>
-                <version>${curator.version}</version>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.curator</groupId>
-                <artifactId>curator-recipes</artifactId>
-                <version>${curator.version}</version>
-            </dependency>
             <dependency>
                 <groupId>org.apache.httpcomponents</groupId>
                 <artifactId>httpclient</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index 6ab74a7..d35c74c 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -47,10 +47,16 @@
             <groupId>org.apache.kylin</groupId>
             <artifactId>kylin-core-storage</artifactId>
         </dependency>
+
+        <!-- Provided -->
         <dependency>
             <groupId>commons-cli</groupId>
             <artifactId>commons-cli</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
 
         <!-- Env & Test -->
         <dependency>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/server/pom.xml
----------------------------------------------------------------------
diff --git a/server/pom.xml b/server/pom.xml
index 454ed6b..3c96e60 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -233,6 +233,10 @@
                     <artifactId>jetty-all</artifactId>
                     <groupId>org.eclipse.jetty.aggregate</groupId>
                 </exclusion>
+                <exclusion>
+                    <artifactId>calcite-core</artifactId>
+                    <groupId>org.apache.calcite</groupId>
+                </exclusion>
             </exclusions>
         </dependency>
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/source-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/source-kafka/pom.xml b/source-kafka/pom.xml
index e2fe448..da5e033 100644
--- a/source-kafka/pom.xml
+++ b/source-kafka/pom.xml
@@ -35,23 +35,19 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-engine-mr</artifactId>
+            <artifactId>kylin-core-common</artifactId>
         </dependency>
-
         <dependency>
             <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-core-common</artifactId>
+            <artifactId>kylin-engine-mr</artifactId>
         </dependency>
 
+        <!-- Provided -->
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka_2.10</artifactId>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
         </dependency>
+
         <!-- Env & Test -->
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -76,9 +72,5 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
-
-
-
     </dependencies>
-
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/storage-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/storage-hbase/pom.xml b/storage-hbase/pom.xml
index 3aea531..054b2e7 100644
--- a/storage-hbase/pom.xml
+++ b/storage-hbase/pom.xml
@@ -119,12 +119,8 @@
                                     <include>org.apache.kylin:kylin-core-metadata</include>
                                     <include>org.apache.kylin:kylin-core-dictionary</include>
                                     <include>org.apache.kylin:kylin-core-cube</include>
-                                    <include>com.ning:compress-lzf</include>
                                     <include>org.roaringbitmap:RoaringBitmap</include>
                                     <include>com.tdunning:t-digest</include>
-                                    <!-- below for inverted index only -->
-                                    <include>com.n3twork.druid:extendedset</include>
-                                    <include>org.apache.commons:commons-lang3</include>
                                 </includes>
                             </artifactSet>
                             <filters>

http://git-wip-us.apache.org/repos/asf/kylin/blob/6d5ed7ec/tool/pom.xml
----------------------------------------------------------------------
diff --git a/tool/pom.xml b/tool/pom.xml
index 91040d4..e9bb9ba 100644
--- a/tool/pom.xml
+++ b/tool/pom.xml
@@ -54,6 +54,33 @@
             <artifactId>kylin-source-hive</artifactId>
         </dependency>
 
+        <!-- As KylinConfigCLI will be called before any dependency loaded, has to define the following as compile in tool -->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-log4j12</artifactId>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <scope>compile</scope>
+        </dependency>
+
         <!--Env-->
         <dependency>
             <groupId>org.apache.hbase</groupId>
@@ -100,6 +127,12 @@
                                 <includes>
                                     <!-- shade the httpcore to avoid the lower version conflict with HBase one -->
                                     <include>org.apache.httpcomponents:httpcore</include>
+                                    <include>org.slf4j:slf4j-api</include>
+                                    <include>org.slf4j:slf4j-log4j12</include>
+                                    <include>log4j:log4j</include>
+                                    <include>commons-io:commons-io</include>
+                                    <include>commons-lang:commons-lang</include>
+                                    <include>com.google.guava:guava</include>
                                     <include>org.apache.kylin:kylin-tool</include>
                                 </includes>
                             </artifactSet>