You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/26 16:08:33 UTC

[10/34] incubator-kylin git commit: KYLIN-960 organize existing storage module

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java b/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
deleted file mode 100644
index 2edf11e..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements.  See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership.  The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License.  You may obtain a copy of the License at
-// * 
-// *     http://www.apache.org/licenses/LICENSE-2.0
-// * 
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-//*/
-//
-//package org.apache.kylin.storage.minicluster;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.sql.Connection;
-//import java.sql.DriverManager;
-//import java.sql.SQLException;
-//
-//import org.apache.commons.io.FileUtils;
-//import org.apache.hadoop.hdfs.MiniDFSCluster;
-//import org.apache.hadoop.hive.conf.HiveConf;
-//import org.apache.hadoop.hive.service.HiveInterface;
-//import org.apache.hadoop.mapred.JobConf;
-//import org.apache.hadoop.mapred.MiniMRCluster;
-//
-///**
-// * This is a test case to verify whether the query can be executed on Hive minicluster;
-// * You need set $HADOOP_HOME environment parameter before run it;
-// * @author shaoshi
-// *
-// */
-//public class HiveMiniClusterTest extends HiveJDBCClientTest {
-//    public static final File HIVE_BASE_DIR = new File("target/hive");
-//    public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir");
-//    public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir");
-//    public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb");
-//    public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs");
-//    public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp");
-//    public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse");
-//    public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata");
-//    public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp");
-//    protected HiveInterface client;
-//
-//    protected MiniDFSCluster miniDFS;
-//    protected MiniMRCluster miniMR;
-//
-//    //@Before
-//    public void setup() {
-//        super.setup();
-//        startHiveMiniCluster();
-//    }
-//
-//    protected void startHiveMiniCluster() {
-//        //Create and configure location for hive to dump junk in target folder
-//        try {
-//            FileUtils.forceMkdir(HIVE_BASE_DIR);
-//            FileUtils.forceMkdir(HIVE_SCRATCH_DIR);
-//            FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR);
-//            FileUtils.forceMkdir(HIVE_LOGS_DIR);
-//            FileUtils.forceMkdir(HIVE_TMP_DIR);
-//            FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR);
-//            FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR);
-//            FileUtils.forceMkdir(HIVE_TESTDATA_DIR);
-//        } catch (IOException e1) {
-//            e1.printStackTrace();
-//            System.exit(1);
-//        }
-//
-//        System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true");
-//        System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath());
-//        System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath());
-//        System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath());
-//        System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath());
-//        System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath());
-//        System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath());
-//        System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath());
-//        System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log");
-//
-//        // custom properties
-//        System.setProperty("hive.server2.long.polling.timeout", "5000");
-//
-//        HiveConf conf = new HiveConf();
-//
-//        /* Build MiniDFSCluster */
-//        try {
-//            miniDFS = new MiniDFSCluster.Builder(conf).build();
-//
-//            /* Build MiniMR Cluster */
-//            int numTaskTrackers = 1;
-//            int numTaskTrackerDirectories = 1;
-//            String[] racks = null;
-//            String[] hosts = null;
-//            miniMR = new MiniMRCluster(numTaskTrackers, miniDFS.getFileSystem().getUri().toString(), numTaskTrackerDirectories, racks, hosts, new JobConf(conf));
-//            JobConf jobConf = miniMR.createJobConf(new JobConf(conf));
-//            System.out.println("-------" + jobConf.get("fs.defaultFS"));
-//            System.out.println("-------" + miniDFS.getFileSystem().getUri().toString());
-//            System.setProperty("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
-//        } catch (IOException e) {
-//            e.printStackTrace();
-//            System.exit(1);
-//        }
-//    }
-//
-//    //@After
-//    public void tearDown() {
-//        if (miniMR != null)
-//            miniMR.shutdown();
-//
-//        if (miniDFS != null)
-//            miniDFS.shutdown();
-//
-//        super.tearDown();
-//    }
-//
-//    protected Connection getHiveConnection() throws SQLException {
-//        return DriverManager.getConnection("jdbc:hive2:///", "", "");
-//    }
-//
-//    public static void main(String[] args) throws SQLException {
-//        HiveMiniClusterTest test = new HiveMiniClusterTest();
-//        test.runTests();
-//    }
-//}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
deleted file mode 100644
index 0ef61b6..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
+++ /dev/null
@@ -1,146 +0,0 @@
-package org.apache.kylin.storage.test;
-
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.IdentityUtils;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.ICachableStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.cache.CacheFledgedDynamicStorageEngine;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
-
-/**
- */
-public class DynamicCacheTest {
-
-    class TsOnlyTuple implements ITuple {
-        private TblColRef partitionCol;
-        private String tsStr;
-
-        public TsOnlyTuple(TblColRef partitionCol, String tsStr) {
-            this.partitionCol = partitionCol;
-            this.tsStr = tsStr;
-        }
-
-        @Override
-        public List<String> getAllFields() {
-            throw new NotImplementedException();
-        }
-
-        @Override
-        public List<TblColRef> getAllColumns() {
-            throw new NotImplementedException();
-        }
-
-        @Override
-        public Object[] getAllValues() {
-            throw new NotImplementedException();
-        }
-
-        @Override
-        public ITuple makeCopy() {
-            return new TsOnlyTuple(this.partitionCol, this.tsStr);
-        }
-
-        @Override
-        public Object getValue(TblColRef col) {
-            if (col.equals(partitionCol)) {
-                return Tuple.dateToEpicDays(this.tsStr);
-            } else {
-                throw new NotImplementedException();
-            }
-        }
-    }
-
-    @Test
-    public void basicTest() {
-
-        final StorageContext context = new StorageContext();
-        final List<TblColRef> groups = StorageMockUtils.buildGroups();
-        final TblColRef partitionCol = groups.get(0);
-        final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-        final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
-
-        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef> newArrayList(), aggregations);
-
-        ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01");
-        ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01");
-        final List<ITuple> allTuples = Lists.newArrayList(aTuple, bTuple);
-
-        //counts for verifying
-        final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
-        final List<Integer> returnedRowPerSearch = Lists.newArrayList();
-
-        CacheFledgedDynamicStorageEngine dynamicCache = new CacheFledgedDynamicStorageEngine(new ICachableStorageQuery() {
-            @Override
-            public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
-                Range<Long> tsRagneInQuery = TsConditionExtractor.extractTsCondition(partitionCol, sqlDigest.filter);
-                List<ITuple> ret = Lists.newArrayList();
-                for (ITuple tuple : allTuples) {
-                    if (tsRagneInQuery.contains(Tuple.getTs(tuple, partitionCol))) {
-                        ret.add(tuple);
-                    }
-                }
-
-                underlyingSEHitCount.incrementAndGet();
-                returnedRowPerSearch.add(ret.size());
-
-                return new SimpleTupleIterator(ret.iterator());
-            }
-
-            @Override
-            public boolean isDynamic() {
-                return true;
-            }
-
-            @Override
-            public Range<Long> getVolatilePeriod() {
-                return Ranges.greaterThan(DateFormat.stringToMillis("2011-02-01"));
-            }
-
-            @Override
-            public String getStorageUUID() {
-                return "111ca32a-a33e-4b69-12aa-0bb8b1f8c191";
-            }
-        }, partitionCol);
-
-        sqlDigest.filter = StorageMockUtils.buildTs2010Filter(groups.get(0));
-        ITupleIterator firstIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
-        IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
-        while (firstIterator.hasNext()) {
-            firstResults.put(firstIterator.next(), null);
-        }
-        firstIterator.close();
-
-        sqlDigest.filter = StorageMockUtils.buildTs2011Filter(groups.get(0));
-        ITupleIterator secondIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
-        IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
-        while (secondIterator.hasNext()) {
-            secondResults.put(secondIterator.next(), null);
-        }
-        secondIterator.close();
-
-        Assert.assertEquals(2, firstResults.size());
-        IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
-        Assert.assertEquals(2, underlyingSEHitCount.get());
-        Assert.assertEquals(new Integer(2), returnedRowPerSearch.get(0));
-        Assert.assertEquals(new Integer(1), returnedRowPerSearch.get(1));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
deleted file mode 100644
index 50eb56d..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.test;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageFactory;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
-import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class ITStorageTest extends HBaseMetadataTestCase {
-
-    private IStorageQuery storageEngine;
-    private CubeInstance cube;
-    private StorageContext context;
-
-    @BeforeClass
-    public static void setupResource() throws Exception {
-    }
-
-    @AfterClass
-    public static void tearDownResource() {
-    }
-
-    @Before
-    public void setUp() throws Exception {
-        this.createTestMetadata();
-
-        CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
-        cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
-        Assert.assertNotNull(cube);
-        storageEngine = StorageFactory.createQuery(cube);
-        String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
-        context = new StorageContext();
-        context.setConnUrl(url);
-    }
-
-    @After
-    public void tearDown() throws Exception {
-        this.cleanupTestMetadata();
-    }
-
-    @Test(expected = ScanOutOfLimitException.class)
-    @Ignore
-    public void testScanOutOfLimit() {
-        context.setThreshold(1);
-        List<TblColRef> groups = StorageMockUtils.buildGroups();
-        List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-
-        search(groups, aggregations, null, context);
-    }
-
-    @Test
-    public void test01() {
-        List<TblColRef> groups = StorageMockUtils.buildGroups();
-        List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-        TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
-
-        int count = search(groups, aggregations, filter, context);
-        assertTrue(count > 0);
-    }
-
-    /*
-        @Test
-        public void test02() {
-            List<TblColRef> groups = buildGroups();
-            List<FunctionDesc> aggregations = buildAggregations();
-            TupleFilter filter = buildFilter2(groups.get(1));
-
-            int count = search(groups, aggregations, filter, context);
-            assertTrue(count > 0);
-        }
-
-        @Test
-        public void test03() {
-            List<TblColRef> groups = buildGroups();
-            List<FunctionDesc> aggregations = buildAggregations();
-            TupleFilter filter = buildAndFilter(groups);
-
-            int count = search(groups, aggregations, filter, context);
-            assertTrue(count > 0);
-        }
-
-        @Test
-        public void test04() {
-            List<TblColRef> groups = buildGroups();
-            List<FunctionDesc> aggregations = buildAggregations();
-            TupleFilter filter = buildOrFilter(groups);
-
-            int count = search(groups, aggregations, filter, context);
-            assertTrue(count > 0);
-        }
-
-        @Test
-        public void test05() {
-            List<TblColRef> groups = buildGroups();
-            List<FunctionDesc> aggregations = buildAggregations();
-
-            int count = search(groups, aggregations, null, context);
-            assertTrue(count > 0);
-        }
-    */
-    private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
-        int count = 0;
-        ITupleIterator iterator = null;
-        try {
-            SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
-            iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
-            while (iterator.hasNext()) {
-                ITuple tuple = iterator.next();
-                System.out.println("Tuple = " + tuple);
-                count++;
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            if (iterator != null)
-                iterator.close();
-        }
-        return count;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
deleted file mode 100644
index 358ad39..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-package org.apache.kylin.storage.test;
-
-import java.util.Collections;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.kylin.common.util.IdentityUtils;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.ICachableStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.cache.CacheFledgedStaticStorageEngine;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-
-/**
- */
-public class StaticCacheTest {
-
-    @Test
-    public void basicTest() {
-
-        final StorageContext context = new StorageContext();
-        final List<TblColRef> groups = StorageMockUtils.buildGroups();
-        final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-        final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
-        final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
-        final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
-
-        final List<ITuple> ret = Lists.newArrayList();
-        ret.add(new Tuple(tupleInfo));
-        ret.add(new Tuple(tupleInfo));
-        ret.add(new Tuple(tupleInfo));
-
-        final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
-
-        CacheFledgedStaticStorageEngine cacheFledgedStaticStorageEngine = new CacheFledgedStaticStorageEngine(new ICachableStorageQuery() {
-            @Override
-            public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
-                underlyingSEHitCount.incrementAndGet();
-                return new SimpleTupleIterator(ret.iterator());
-            }
-
-            @Override
-            public boolean isDynamic() {
-                return false;
-            }
-
-            @Override
-            public Range<Long> getVolatilePeriod() {
-                return null;
-            }
-
-            @Override
-            public String getStorageUUID() {
-                return "111ca32a-a33e-4b69-12aa-0bb8b1f8c092";
-            }
-        });
-
-        ITupleIterator firstIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
-        IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
-        while (firstIterator.hasNext()) {
-            firstResults.put(firstIterator.next(), null);
-        }
-        firstIterator.close();
-
-        ITupleIterator secondIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
-        IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
-        while (secondIterator.hasNext()) {
-            secondResults.put(secondIterator.next(), null);
-        }
-        secondIterator.close();
-
-        ITupleIterator thirdIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
-        IdentityHashMap<ITuple, Void> thirdResults = new IdentityHashMap<>();
-        while (thirdIterator.hasNext()) {
-            thirdResults.put(thirdIterator.next(), null);
-        }
-        thirdIterator.close();
-
-        Assert.assertEquals(3, firstResults.size());
-        IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
-        IdentityUtils.collectionReferenceEquals(thirdResults.keySet(), secondResults.keySet());
-
-        Assert.assertEquals(1, underlyingSEHitCount.get());
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
deleted file mode 100644
index b616589..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
+++ /dev/null
@@ -1,139 +0,0 @@
-package org.apache.kylin.storage.test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.TupleInfo;
-
-import com.google.common.collect.ImmutableList;
-
-/**
- */
-public class StorageMockUtils {
-    public static TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
-        TupleInfo info = new TupleInfo();
-        int idx = 0;
-
-        for (TblColRef col : groups) {
-            info.setField(col.getName(), col, idx++);
-        }
-
-        TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
-        for (FunctionDesc func : aggregations) {
-            TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
-            info.setField(col.getName(), col, idx++);
-        }
-
-        return info;
-    }
-
-    public static List<TblColRef> buildGroups() {
-        List<TblColRef> groups = new ArrayList<TblColRef>();
-
-        TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
-        ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
-        TblColRef cf1 = new TblColRef(c1);
-        groups.add(cf1);
-
-        TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
-        ColumnDesc c2 = ColumnDesc.mockup(t2, 14, "META_CATEG_NAME", "string");
-        TblColRef cf2 = new TblColRef(c2);
-        groups.add(cf2);
-
-        return groups;
-    }
-
-    public static List<FunctionDesc> buildAggregations() {
-        List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
-
-        TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
-        TblColRef priceCol = new TblColRef(ColumnDesc.mockup(t1, 7, "PRICE", "decimal(19,4)"));
-        TblColRef sellerCol = new TblColRef(ColumnDesc.mockup(t1, 9, "SELLER_ID", "bigint"));
-
-        FunctionDesc f1 = new FunctionDesc();
-        f1.setExpression("SUM");
-        ParameterDesc p1 = new ParameterDesc();
-        p1.setType("column");
-        p1.setValue("PRICE");
-        p1.setColRefs(ImmutableList.of(priceCol));
-        f1.setParameter(p1);
-        functions.add(f1);
-
-        FunctionDesc f2 = new FunctionDesc();
-        f2.setExpression("COUNT_DISTINCT");
-        ParameterDesc p2 = new ParameterDesc();
-        p2.setType("column");
-        p2.setValue("SELLER_ID");
-        p2.setColRefs(ImmutableList.of(sellerCol));
-        f2.setParameter(p2);
-        functions.add(f2);
-
-        return functions;
-    }
-
-    public static CompareTupleFilter buildTs2010Filter(TblColRef column) {
-        CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
-        ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
-        compareFilter.addChild(columnFilter1);
-        ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2010-01-01");
-        compareFilter.addChild(constantFilter1);
-        return compareFilter;
-    }
-
-    public static CompareTupleFilter buildTs2011Filter(TblColRef column) {
-        CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
-        ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
-        compareFilter.addChild(columnFilter1);
-        ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2011-01-01");
-        compareFilter.addChild(constantFilter1);
-        return compareFilter;
-    }
-
-    public static CompareTupleFilter buildFilter1(TblColRef column) {
-        CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
-        ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
-        compareFilter.addChild(columnFilter1);
-        ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
-        compareFilter.addChild(constantFilter1);
-        return compareFilter;
-    }
-
-    public static CompareTupleFilter buildFilter2(TblColRef column) {
-        CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
-        ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
-        compareFilter.addChild(columnFilter2);
-        ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
-        compareFilter.addChild(constantFilter2);
-        return compareFilter;
-    }
-
-    @SuppressWarnings("unused")
-    public static TupleFilter buildAndFilter(List<TblColRef> columns) {
-        CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
-        CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
-        LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
-        andFilter.addChild(compareFilter1);
-        andFilter.addChild(compareFilter2);
-        return andFilter;
-    }
-
-    @SuppressWarnings("unused")
-    public static TupleFilter buildOrFilter(List<TblColRef> columns) {
-        CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
-        CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
-        LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
-        logicFilter.addChild(compareFilter1);
-        logicFilter.addChild(compareFilter2);
-        return logicFilter;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d9440a1..7bbb54a 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -26,7 +26,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.kylin</groupId>
-            <artifactId>kylin-storage</artifactId>
+            <artifactId>kylin-storage-hbase</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
 

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
index 2a9e3e9..3d8a6f8 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
@@ -46,7 +46,7 @@ import org.apache.kylin.invertedindex.index.Slice;
 import org.apache.kylin.invertedindex.model.IIDesc;
 import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
 import org.apache.kylin.invertedindex.model.IIRow;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
 import org.apache.kylin.streaming.MicroStreamBatch;
 import org.apache.kylin.streaming.MicroStreamBatchConsumer;
 import org.apache.kylin.streaming.StreamingManager;