You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/26 16:08:33 UTC
[10/34] incubator-kylin git commit: KYLIN-960 organize existing
storage module
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java b/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
deleted file mode 100644
index 2edf11e..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/minicluster/HiveMiniClusterTest.java
+++ /dev/null
@@ -1,131 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one
-// * or more contributor license agreements. See the NOTICE file
-// * distributed with this work for additional information
-// * regarding copyright ownership. The ASF licenses this file
-// * to you under the Apache License, Version 2.0 (the
-// * "License"); you may not use this file except in compliance
-// * with the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-//*/
-//
-//package org.apache.kylin.storage.minicluster;
-//
-//import java.io.File;
-//import java.io.IOException;
-//import java.sql.Connection;
-//import java.sql.DriverManager;
-//import java.sql.SQLException;
-//
-//import org.apache.commons.io.FileUtils;
-//import org.apache.hadoop.hdfs.MiniDFSCluster;
-//import org.apache.hadoop.hive.conf.HiveConf;
-//import org.apache.hadoop.hive.service.HiveInterface;
-//import org.apache.hadoop.mapred.JobConf;
-//import org.apache.hadoop.mapred.MiniMRCluster;
-//
-///**
-// * This is a test case to verify whether the query can be executed on Hive minicluster;
-// * You need set $HADOOP_HOME environment parameter before run it;
-// * @author shaoshi
-// *
-// */
-//public class HiveMiniClusterTest extends HiveJDBCClientTest {
-// public static final File HIVE_BASE_DIR = new File("target/hive");
-// public static final File HIVE_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/scratchdir");
-// public static final File HIVE_LOCAL_SCRATCH_DIR = new File(HIVE_BASE_DIR + "/localscratchdir");
-// public static final File HIVE_METADB_DIR = new File(HIVE_BASE_DIR + "/metastoredb");
-// public static final File HIVE_LOGS_DIR = new File(HIVE_BASE_DIR + "/logs");
-// public static final File HIVE_TMP_DIR = new File(HIVE_BASE_DIR + "/tmp");
-// public static final File HIVE_WAREHOUSE_DIR = new File(HIVE_BASE_DIR + "/warehouse");
-// public static final File HIVE_TESTDATA_DIR = new File(HIVE_BASE_DIR + "/testdata");
-// public static final File HIVE_HADOOP_TMP_DIR = new File(HIVE_BASE_DIR + "/hadooptmp");
-// protected HiveInterface client;
-//
-// protected MiniDFSCluster miniDFS;
-// protected MiniMRCluster miniMR;
-//
-// //@Before
-// public void setup() {
-// super.setup();
-// startHiveMiniCluster();
-// }
-//
-// protected void startHiveMiniCluster() {
-// //Create and configure location for hive to dump junk in target folder
-// try {
-// FileUtils.forceMkdir(HIVE_BASE_DIR);
-// FileUtils.forceMkdir(HIVE_SCRATCH_DIR);
-// FileUtils.forceMkdir(HIVE_LOCAL_SCRATCH_DIR);
-// FileUtils.forceMkdir(HIVE_LOGS_DIR);
-// FileUtils.forceMkdir(HIVE_TMP_DIR);
-// FileUtils.forceMkdir(HIVE_WAREHOUSE_DIR);
-// FileUtils.forceMkdir(HIVE_HADOOP_TMP_DIR);
-// FileUtils.forceMkdir(HIVE_TESTDATA_DIR);
-// } catch (IOException e1) {
-// e1.printStackTrace();
-// System.exit(1);
-// }
-//
-// System.setProperty("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=" + HIVE_METADB_DIR.getAbsolutePath() + ";create=true");
-// System.setProperty("hive.metastore.warehouse.dir", HIVE_WAREHOUSE_DIR.getAbsolutePath());
-// System.setProperty("hive.exec.scratchdir", HIVE_SCRATCH_DIR.getAbsolutePath());
-// System.setProperty("hive.exec.local.scratchdir", HIVE_LOCAL_SCRATCH_DIR.getAbsolutePath());
-// System.setProperty("hive.metastore.metadb.dir", HIVE_METADB_DIR.getAbsolutePath());
-// System.setProperty("test.log.dir", HIVE_LOGS_DIR.getAbsolutePath());
-// System.setProperty("hive.querylog.location", HIVE_TMP_DIR.getAbsolutePath());
-// System.setProperty("hadoop.tmp.dir", HIVE_HADOOP_TMP_DIR.getAbsolutePath());
-// System.setProperty("derby.stream.error.file", HIVE_BASE_DIR.getAbsolutePath() + "/derby.log");
-//
-// // custom properties
-// System.setProperty("hive.server2.long.polling.timeout", "5000");
-//
-// HiveConf conf = new HiveConf();
-//
-// /* Build MiniDFSCluster */
-// try {
-// miniDFS = new MiniDFSCluster.Builder(conf).build();
-//
-// /* Build MiniMR Cluster */
-// int numTaskTrackers = 1;
-// int numTaskTrackerDirectories = 1;
-// String[] racks = null;
-// String[] hosts = null;
-// miniMR = new MiniMRCluster(numTaskTrackers, miniDFS.getFileSystem().getUri().toString(), numTaskTrackerDirectories, racks, hosts, new JobConf(conf));
-// JobConf jobConf = miniMR.createJobConf(new JobConf(conf));
-// System.out.println("-------" + jobConf.get("fs.defaultFS"));
-// System.out.println("-------" + miniDFS.getFileSystem().getUri().toString());
-// System.setProperty("mapred.job.tracker", jobConf.get("mapred.job.tracker"));
-// } catch (IOException e) {
-// e.printStackTrace();
-// System.exit(1);
-// }
-// }
-//
-// //@After
-// public void tearDown() {
-// if (miniMR != null)
-// miniMR.shutdown();
-//
-// if (miniDFS != null)
-// miniDFS.shutdown();
-//
-// super.tearDown();
-// }
-//
-// protected Connection getHiveConnection() throws SQLException {
-// return DriverManager.getConnection("jdbc:hive2:///", "", "");
-// }
-//
-// public static void main(String[] args) throws SQLException {
-// HiveMiniClusterTest test = new HiveMiniClusterTest();
-// test.runTests();
-// }
-//}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
deleted file mode 100644
index 0ef61b6..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/DynamicCacheTest.java
+++ /dev/null
@@ -1,146 +0,0 @@
-package org.apache.kylin.storage.test;
-
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.kylin.common.util.DateFormat;
-import org.apache.kylin.common.util.IdentityUtils;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.ICachableStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.cache.CacheFledgedDynamicStorageEngine;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.TsConditionExtractor;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
-
-/**
- */
-public class DynamicCacheTest {
-
- class TsOnlyTuple implements ITuple {
- private TblColRef partitionCol;
- private String tsStr;
-
- public TsOnlyTuple(TblColRef partitionCol, String tsStr) {
- this.partitionCol = partitionCol;
- this.tsStr = tsStr;
- }
-
- @Override
- public List<String> getAllFields() {
- throw new NotImplementedException();
- }
-
- @Override
- public List<TblColRef> getAllColumns() {
- throw new NotImplementedException();
- }
-
- @Override
- public Object[] getAllValues() {
- throw new NotImplementedException();
- }
-
- @Override
- public ITuple makeCopy() {
- return new TsOnlyTuple(this.partitionCol, this.tsStr);
- }
-
- @Override
- public Object getValue(TblColRef col) {
- if (col.equals(partitionCol)) {
- return Tuple.dateToEpicDays(this.tsStr);
- } else {
- throw new NotImplementedException();
- }
- }
- }
-
- @Test
- public void basicTest() {
-
- final StorageContext context = new StorageContext();
- final List<TblColRef> groups = StorageMockUtils.buildGroups();
- final TblColRef partitionCol = groups.get(0);
- final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
- final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
-
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef> newArrayList(), aggregations);
-
- ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01");
- ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01");
- final List<ITuple> allTuples = Lists.newArrayList(aTuple, bTuple);
-
- //counts for verifying
- final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
- final List<Integer> returnedRowPerSearch = Lists.newArrayList();
-
- CacheFledgedDynamicStorageEngine dynamicCache = new CacheFledgedDynamicStorageEngine(new ICachableStorageQuery() {
- @Override
- public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- Range<Long> tsRagneInQuery = TsConditionExtractor.extractTsCondition(partitionCol, sqlDigest.filter);
- List<ITuple> ret = Lists.newArrayList();
- for (ITuple tuple : allTuples) {
- if (tsRagneInQuery.contains(Tuple.getTs(tuple, partitionCol))) {
- ret.add(tuple);
- }
- }
-
- underlyingSEHitCount.incrementAndGet();
- returnedRowPerSearch.add(ret.size());
-
- return new SimpleTupleIterator(ret.iterator());
- }
-
- @Override
- public boolean isDynamic() {
- return true;
- }
-
- @Override
- public Range<Long> getVolatilePeriod() {
- return Ranges.greaterThan(DateFormat.stringToMillis("2011-02-01"));
- }
-
- @Override
- public String getStorageUUID() {
- return "111ca32a-a33e-4b69-12aa-0bb8b1f8c191";
- }
- }, partitionCol);
-
- sqlDigest.filter = StorageMockUtils.buildTs2010Filter(groups.get(0));
- ITupleIterator firstIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
- IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
- while (firstIterator.hasNext()) {
- firstResults.put(firstIterator.next(), null);
- }
- firstIterator.close();
-
- sqlDigest.filter = StorageMockUtils.buildTs2011Filter(groups.get(0));
- ITupleIterator secondIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
- IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
- while (secondIterator.hasNext()) {
- secondResults.put(secondIterator.next(), null);
- }
- secondIterator.close();
-
- Assert.assertEquals(2, firstResults.size());
- IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
- Assert.assertEquals(2, underlyingSEHitCount.get());
- Assert.assertEquals(new Integer(2), returnedRowPerSearch.get(0));
- Assert.assertEquals(new Integer(1), returnedRowPerSearch.get(1));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java b/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
deleted file mode 100644
index 50eb56d..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/ITStorageTest.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.storage.test;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.cube.CubeInstance;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.storage.IStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.StorageFactory;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
-import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class ITStorageTest extends HBaseMetadataTestCase {
-
- private IStorageQuery storageEngine;
- private CubeInstance cube;
- private StorageContext context;
-
- @BeforeClass
- public static void setupResource() throws Exception {
- }
-
- @AfterClass
- public static void tearDownResource() {
- }
-
- @Before
- public void setUp() throws Exception {
- this.createTestMetadata();
-
- CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
- cube = cubeMgr.getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_EMPTY");
- Assert.assertNotNull(cube);
- storageEngine = StorageFactory.createQuery(cube);
- String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
- context = new StorageContext();
- context.setConnUrl(url);
- }
-
- @After
- public void tearDown() throws Exception {
- this.cleanupTestMetadata();
- }
-
- @Test(expected = ScanOutOfLimitException.class)
- @Ignore
- public void testScanOutOfLimit() {
- context.setThreshold(1);
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
-
- search(groups, aggregations, null, context);
- }
-
- @Test
- public void test01() {
- List<TblColRef> groups = StorageMockUtils.buildGroups();
- List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
- TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- /*
- @Test
- public void test02() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildFilter2(groups.get(1));
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test03() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildAndFilter(groups);
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test04() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
- TupleFilter filter = buildOrFilter(groups);
-
- int count = search(groups, aggregations, filter, context);
- assertTrue(count > 0);
- }
-
- @Test
- public void test05() {
- List<TblColRef> groups = buildGroups();
- List<FunctionDesc> aggregations = buildAggregations();
-
- int count = search(groups, aggregations, null, context);
- assertTrue(count > 0);
- }
- */
- private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
- int count = 0;
- ITupleIterator iterator = null;
- try {
- SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
- iterator = storageEngine.search(context, sqlDigest, StorageMockUtils.newTupleInfo(groups, aggregations));
- while (iterator.hasNext()) {
- ITuple tuple = iterator.next();
- System.out.println("Tuple = " + tuple);
- count++;
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- if (iterator != null)
- iterator.close();
- }
- return count;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java b/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
deleted file mode 100644
index 358ad39..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/StaticCacheTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-package org.apache.kylin.storage.test;
-
-import java.util.Collections;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.kylin.common.util.IdentityUtils;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.realization.SQLDigest;
-import org.apache.kylin.metadata.tuple.ITuple;
-import org.apache.kylin.metadata.tuple.ITupleIterator;
-import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
-import org.apache.kylin.storage.ICachableStorageQuery;
-import org.apache.kylin.storage.StorageContext;
-import org.apache.kylin.storage.cache.CacheFledgedStaticStorageEngine;
-import org.apache.kylin.storage.tuple.Tuple;
-import org.apache.kylin.storage.tuple.TupleInfo;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Range;
-
-/**
- */
-public class StaticCacheTest {
-
- @Test
- public void basicTest() {
-
- final StorageContext context = new StorageContext();
- final List<TblColRef> groups = StorageMockUtils.buildGroups();
- final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
- final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
- final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
- final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
-
- final List<ITuple> ret = Lists.newArrayList();
- ret.add(new Tuple(tupleInfo));
- ret.add(new Tuple(tupleInfo));
- ret.add(new Tuple(tupleInfo));
-
- final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
-
- CacheFledgedStaticStorageEngine cacheFledgedStaticStorageEngine = new CacheFledgedStaticStorageEngine(new ICachableStorageQuery() {
- @Override
- public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
- underlyingSEHitCount.incrementAndGet();
- return new SimpleTupleIterator(ret.iterator());
- }
-
- @Override
- public boolean isDynamic() {
- return false;
- }
-
- @Override
- public Range<Long> getVolatilePeriod() {
- return null;
- }
-
- @Override
- public String getStorageUUID() {
- return "111ca32a-a33e-4b69-12aa-0bb8b1f8c092";
- }
- });
-
- ITupleIterator firstIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
- IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
- while (firstIterator.hasNext()) {
- firstResults.put(firstIterator.next(), null);
- }
- firstIterator.close();
-
- ITupleIterator secondIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
- IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
- while (secondIterator.hasNext()) {
- secondResults.put(secondIterator.next(), null);
- }
- secondIterator.close();
-
- ITupleIterator thirdIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
- IdentityHashMap<ITuple, Void> thirdResults = new IdentityHashMap<>();
- while (thirdIterator.hasNext()) {
- thirdResults.put(thirdIterator.next(), null);
- }
- thirdIterator.close();
-
- Assert.assertEquals(3, firstResults.size());
- IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
- IdentityUtils.collectionReferenceEquals(thirdResults.keySet(), secondResults.keySet());
-
- Assert.assertEquals(1, underlyingSEHitCount.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java b/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
deleted file mode 100644
index b616589..0000000
--- a/storage/src/test/java/org/apache/kylin/storage/test/StorageMockUtils.java
+++ /dev/null
@@ -1,139 +0,0 @@
-package org.apache.kylin.storage.test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.model.ColumnDesc;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.ParameterDesc;
-import org.apache.kylin.metadata.model.TableDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.tuple.TupleInfo;
-
-import com.google.common.collect.ImmutableList;
-
-/**
- */
-public class StorageMockUtils {
- public static TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
- TupleInfo info = new TupleInfo();
- int idx = 0;
-
- for (TblColRef col : groups) {
- info.setField(col.getName(), col, idx++);
- }
-
- TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
- for (FunctionDesc func : aggregations) {
- TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
- info.setField(col.getName(), col, idx++);
- }
-
- return info;
- }
-
- public static List<TblColRef> buildGroups() {
- List<TblColRef> groups = new ArrayList<TblColRef>();
-
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
- TblColRef cf1 = new TblColRef(c1);
- groups.add(cf1);
-
- TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
- ColumnDesc c2 = ColumnDesc.mockup(t2, 14, "META_CATEG_NAME", "string");
- TblColRef cf2 = new TblColRef(c2);
- groups.add(cf2);
-
- return groups;
- }
-
- public static List<FunctionDesc> buildAggregations() {
- List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
-
- TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
- TblColRef priceCol = new TblColRef(ColumnDesc.mockup(t1, 7, "PRICE", "decimal(19,4)"));
- TblColRef sellerCol = new TblColRef(ColumnDesc.mockup(t1, 9, "SELLER_ID", "bigint"));
-
- FunctionDesc f1 = new FunctionDesc();
- f1.setExpression("SUM");
- ParameterDesc p1 = new ParameterDesc();
- p1.setType("column");
- p1.setValue("PRICE");
- p1.setColRefs(ImmutableList.of(priceCol));
- f1.setParameter(p1);
- functions.add(f1);
-
- FunctionDesc f2 = new FunctionDesc();
- f2.setExpression("COUNT_DISTINCT");
- ParameterDesc p2 = new ParameterDesc();
- p2.setType("column");
- p2.setValue("SELLER_ID");
- p2.setColRefs(ImmutableList.of(sellerCol));
- f2.setParameter(p2);
- functions.add(f2);
-
- return functions;
- }
-
- public static CompareTupleFilter buildTs2010Filter(TblColRef column) {
- CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
- ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter1);
- ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2010-01-01");
- compareFilter.addChild(constantFilter1);
- return compareFilter;
- }
-
- public static CompareTupleFilter buildTs2011Filter(TblColRef column) {
- CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
- ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter1);
- ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2011-01-01");
- compareFilter.addChild(constantFilter1);
- return compareFilter;
- }
-
- public static CompareTupleFilter buildFilter1(TblColRef column) {
- CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
- ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter1);
- ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
- compareFilter.addChild(constantFilter1);
- return compareFilter;
- }
-
- public static CompareTupleFilter buildFilter2(TblColRef column) {
- CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
- ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
- compareFilter.addChild(columnFilter2);
- ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
- compareFilter.addChild(constantFilter2);
- return compareFilter;
- }
-
- @SuppressWarnings("unused")
- public static TupleFilter buildAndFilter(List<TblColRef> columns) {
- CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
- CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
- LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
- andFilter.addChild(compareFilter1);
- andFilter.addChild(compareFilter2);
- return andFilter;
- }
-
- @SuppressWarnings("unused")
- public static TupleFilter buildOrFilter(List<TblColRef> columns) {
- CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
- CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
- LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
- logicFilter.addChild(compareFilter1);
- logicFilter.addChild(compareFilter2);
- return logicFilter;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d9440a1..7bbb54a 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -26,7 +26,7 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-storage</artifactId>
+ <artifactId>kylin-storage-hbase</artifactId>
<version>${project.parent.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
----------------------------------------------------------------------
diff --git a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
index 2a9e3e9..3d8a6f8 100644
--- a/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
+++ b/streaming/src/main/java/org/apache/kylin/streaming/invertedindex/IIStreamConsumer.java
@@ -46,7 +46,7 @@ import org.apache.kylin.invertedindex.index.Slice;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.invertedindex.model.IIRow;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.apache.kylin.streaming.MicroStreamBatch;
import org.apache.kylin.streaming.MicroStreamBatchConsumer;
import org.apache.kylin.streaming.StreamingManager;