You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/08/26 16:08:53 UTC
[30/34] incubator-kylin git commit: KYLIN-960 organize existing
storage module
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/main/java/org/apache/kylin/storage/translate/FuzzyValueCombination.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/FuzzyValueCombination.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/FuzzyValueCombination.java
new file mode 100644
index 0000000..199a219
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/FuzzyValueCombination.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.translate;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+/**
+ * @author yangli9
+ *
+ */
+public class FuzzyValueCombination {
+
+ private static class Dim {
+ TblColRef col;
+ Set<String> values;
+ }
+
+ private static final Set<String> SINGLE_NULL_SET = Sets.newHashSet();
+ static {
+ SINGLE_NULL_SET.add(null);
+ }
+
+ public static List<Map<TblColRef, String>> calculate(Map<TblColRef, Set<String>> fuzzyValues, long cap) {
+ Dim[] dims = toDims(fuzzyValues);
+ capDims(dims, cap);
+ return combination(dims);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static List<Map<TblColRef, String>> combination(Dim[] dims) {
+
+ List<Map<TblColRef, String>> result = Lists.newArrayList();
+
+ int emptyDims = 0;
+ for (Dim dim : dims) {
+ if (dim.values.isEmpty()) {
+ dim.values = SINGLE_NULL_SET;
+ emptyDims++;
+ }
+ }
+ if (emptyDims == dims.length) {
+ return result;
+ }
+
+ Map<TblColRef, String> r = Maps.newHashMap();
+ Iterator<String>[] iters = new Iterator[dims.length];
+ int level = 0;
+ while (true) {
+ Dim dim = dims[level];
+ if (iters[level] == null) {
+ iters[level] = dim.values.iterator();
+ }
+
+ Iterator<String> it = iters[level];
+ if (it.hasNext() == false) {
+ if (level == 0)
+ break;
+ r.remove(dim.col);
+ iters[level] = null;
+ level--;
+ continue;
+ }
+
+ r.put(dim.col, it.next());
+ if (level == dims.length - 1) {
+ result.add(new HashMap<TblColRef, String>(r));
+ } else {
+ level++;
+ }
+ }
+ return result;
+ }
+
+ private static Dim[] toDims(Map<TblColRef, Set<String>> fuzzyValues) {
+ Dim[] dims = new Dim[fuzzyValues.size()];
+ int i = 0;
+ for (Entry<TblColRef, Set<String>> entry : fuzzyValues.entrySet()) {
+ dims[i] = new Dim();
+ dims[i].col = entry.getKey();
+ dims[i].values = entry.getValue();
+ if (dims[i].values == null)
+ dims[i].values = Collections.emptySet();
+ i++;
+ }
+ return dims;
+ }
+
+ private static void capDims(Dim[] dims, long cap) {
+ Arrays.sort(dims, new Comparator<Dim>() {
+ @Override
+ public int compare(Dim o1, Dim o2) {
+ return -(o1.values.size() - o2.values.size());
+ }
+ });
+
+ for (Dim dim : dims) {
+ if (combCount(dims) < cap)
+ break;
+ dim.values = Collections.emptySet();
+ }
+ }
+
+ private static long combCount(Dim[] dims) {
+ long count = 1;
+ for (Dim dim : dims) {
+ count *= Math.max(dim.values.size(), 1);
+ }
+ return count;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
new file mode 100644
index 0000000..a5a9611
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/translate/HBaseKeyRange.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.storage.translate;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.kylin.common.debug.BackdoorToggles;
+import org.apache.kylin.common.util.Bytes;
+import org.apache.kylin.common.util.BytesUtil;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.Pair;
+import org.apache.kylin.cube.CubeSegment;
+import org.apache.kylin.cube.cuboid.Cuboid;
+import org.apache.kylin.cube.kv.AbstractRowKeyEncoder;
+import org.apache.kylin.cube.kv.FuzzyKeyEncoder;
+import org.apache.kylin.cube.kv.FuzzyMaskEncoder;
+import org.apache.kylin.cube.kv.RowConstants;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ *
+ * @author xjiang
+ *
+ */
+public class HBaseKeyRange implements Comparable<HBaseKeyRange> {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseKeyRange.class);
+
+ private static final int FUZZY_VALUE_CAP = 20;
+ private static final byte[] ZERO_TAIL_BYTES = new byte[] { 0 };
+
+ private final CubeSegment cubeSeg;
+ private final Cuboid cuboid;
+ private final List<Collection<ColumnValueRange>> flatOrAndFilter; // OR-AND filter, (A AND B AND ..) OR (C AND D AND ..) OR ..
+
+ private byte[] startKey;
+ private byte[] stopKey;
+ private List<Pair<byte[], byte[]>> fuzzyKeys;
+
+ private String startKeyString;
+ private String stopKeyString;
+ private String fuzzyKeyString;
+
+ private long partitionColumnStartDate = Long.MIN_VALUE;
+ private long partitionColumnEndDate = Long.MAX_VALUE;
+
+ public HBaseKeyRange(CubeSegment cubeSeg, Cuboid cuboid, byte[] startKey, byte[] stopKey, List<Pair<byte[], byte[]>> fuzzyKeys, List<Collection<ColumnValueRange>> flatColumnValueFilter, long partitionColumnStartDate, long partitionColumnEndDate) {
+ this.cubeSeg = cubeSeg;
+ this.cuboid = cuboid;
+ this.startKey = startKey;
+ this.stopKey = stopKey;
+ this.fuzzyKeys = fuzzyKeys;
+ this.flatOrAndFilter = flatColumnValueFilter;
+ this.partitionColumnStartDate = partitionColumnStartDate;
+ this.partitionColumnEndDate = partitionColumnEndDate;
+ initDebugString();
+ }
+
+ public HBaseKeyRange(Collection<TblColRef> dimensionColumns, Collection<ColumnValueRange> andDimensionRanges, CubeSegment cubeSeg, CubeDesc cubeDesc) {
+ this.cubeSeg = cubeSeg;
+ long cuboidId = this.calculateCuboidID(cubeDesc, dimensionColumns);
+ this.cuboid = Cuboid.findById(cubeDesc, cuboidId);
+ this.flatOrAndFilter = Lists.newLinkedList();
+ this.flatOrAndFilter.add(andDimensionRanges);
+ init(andDimensionRanges);
+ initDebugString();
+ }
+
+ private long calculateCuboidID(CubeDesc cube, Collection<TblColRef> dimensions) {
+ long cuboidID = 0;
+ for (TblColRef column : dimensions) {
+ int index = cube.getRowkey().getColumnBitIndex(column);
+ cuboidID |= 1L << index;
+ }
+ return cuboidID;
+ }
+
+ private void init(Collection<ColumnValueRange> andDimensionRanges) {
+ int size = andDimensionRanges.size();
+ Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size);
+ Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size);
+ Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size);
+ for (ColumnValueRange dimRange : andDimensionRanges) {
+ TblColRef column = dimRange.getColumn();
+ startValues.put(column, dimRange.getBeginValue());
+ stopValues.put(column, dimRange.getEndValue());
+ fuzzyValues.put(column, dimRange.getEqualValues());
+
+ TblColRef partitionDateColumnRef = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
+ if (column.equals(partitionDateColumnRef)) {
+ initPartitionRange(dimRange);
+ }
+ }
+
+ AbstractRowKeyEncoder encoder = AbstractRowKeyEncoder.createInstance(cubeSeg, cuboid);
+
+ encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
+
+ this.startKey = encoder.encode(startValues);
+
+ encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
+
+ // In order to make stopRow inclusive add a trailing 0 byte. #See
+ // Scan.setStopRow(byte [] stopRow)
+ this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES);
+
+ // restore encoder defaults for later reuse (note
+ // AbstractRowKeyEncoder.createInstance() caches instances)
+ encoder.setBlankByte(AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE);
+
+ // always fuzzy match cuboid ID to lock on the selected cuboid
+ this.fuzzyKeys = buildFuzzyKeys(fuzzyValues);
+ }
+
+ private void initPartitionRange(ColumnValueRange dimRange) {
+ if (null != dimRange.getBeginValue()) {
+ this.partitionColumnStartDate = DateFormat.stringToMillis(dimRange.getBeginValue());
+ }
+ if (null != dimRange.getEndValue()) {
+ this.partitionColumnEndDate = DateFormat.stringToMillis(dimRange.getEndValue());
+ }
+ }
+
+ private void initDebugString() {
+ this.startKeyString = BytesUtil.toHex(this.startKey);
+ this.stopKeyString = BytesUtil.toHex(this.stopKey);
+ StringBuilder buf = new StringBuilder();
+ for (Pair<byte[], byte[]> fuzzyKey : this.fuzzyKeys) {
+ buf.append(BytesUtil.toHex(fuzzyKey.getFirst()));
+ buf.append(" ");
+ buf.append(BytesUtil.toHex(fuzzyKey.getSecond()));
+ buf.append(System.lineSeparator());
+ }
+ this.fuzzyKeyString = buf.toString();
+ }
+
+ private List<Pair<byte[], byte[]>> buildFuzzyKeys(Map<TblColRef, Set<String>> fuzzyValueSet) {
+ ArrayList<Pair<byte[], byte[]>> result = new ArrayList<Pair<byte[], byte[]>>();
+
+ // debug/profiling purpose
+ if (BackdoorToggles.getDisableFuzzyKey()) {
+ logger.info("The execution of this query will not use fuzzy key");
+ return result;
+ }
+
+ FuzzyKeyEncoder fuzzyKeyEncoder = new FuzzyKeyEncoder(cubeSeg, cuboid);
+ FuzzyMaskEncoder fuzzyMaskEncoder = new FuzzyMaskEncoder(cubeSeg, cuboid);
+
+ List<Map<TblColRef, String>> fuzzyValues = FuzzyValueCombination.calculate(fuzzyValueSet, FUZZY_VALUE_CAP);
+ for (Map<TblColRef, String> fuzzyValue : fuzzyValues) {
+ result.add(new Pair<byte[], byte[]>(fuzzyKeyEncoder.encode(fuzzyValue), fuzzyMaskEncoder.encode(fuzzyValue)));
+ }
+ return result;
+ }
+
+ public CubeSegment getCubeSegment() {
+ return this.cubeSeg;
+ }
+
+ public Cuboid getCuboid() {
+ return cuboid;
+ }
+
+ public byte[] getStartKey() {
+ return startKey;
+ }
+
+ public byte[] getStopKey() {
+ return stopKey;
+ }
+
+ public List<Pair<byte[], byte[]>> getFuzzyKeys() {
+ return fuzzyKeys;
+ }
+
+ public String getStartKeyAsString() {
+ return startKeyString;
+ }
+
+ public String getStopKeyAsString() {
+ return stopKeyString;
+ }
+
+ public String getFuzzyKeyAsString() {
+ return fuzzyKeyString;
+ }
+
+ public List<Collection<ColumnValueRange>> getFlatOrAndFilter() {
+ return flatOrAndFilter;
+ }
+
+ public long getPartitionColumnStartDate() {
+ return partitionColumnStartDate;
+ }
+
+ public long getPartitionColumnEndDate() {
+ return partitionColumnEndDate;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((cubeSeg == null) ? 0 : cubeSeg.hashCode());
+ result = prime * result + ((cuboid == null) ? 0 : cuboid.hashCode());
+ result = prime * result + ((fuzzyKeyString == null) ? 0 : fuzzyKeyString.hashCode());
+ result = prime * result + ((startKeyString == null) ? 0 : startKeyString.hashCode());
+ result = prime * result + ((stopKeyString == null) ? 0 : stopKeyString.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ HBaseKeyRange other = (HBaseKeyRange) obj;
+ if (cubeSeg == null) {
+ if (other.cubeSeg != null)
+ return false;
+ } else if (!cubeSeg.equals(other.cubeSeg))
+ return false;
+ if (cuboid == null) {
+ if (other.cuboid != null)
+ return false;
+ } else if (!cuboid.equals(other.cuboid))
+ return false;
+ if (fuzzyKeyString == null) {
+ if (other.fuzzyKeyString != null)
+ return false;
+ } else if (!fuzzyKeyString.equals(other.fuzzyKeyString))
+ return false;
+ if (startKeyString == null) {
+ if (other.startKeyString != null)
+ return false;
+ } else if (!startKeyString.equals(other.startKeyString))
+ return false;
+ if (stopKeyString == null) {
+ if (other.stopKeyString != null)
+ return false;
+ } else if (!stopKeyString.equals(other.stopKeyString))
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(HBaseKeyRange other) {
+ return Bytes.compareTo(this.startKey, other.startKey);
+ }
+
+ public boolean hitSegment() {
+ return cubeSeg.getDateRangeStart() <= getPartitionColumnEndDate() && cubeSeg.getDateRangeEnd() >= getPartitionColumnStartDate();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java b/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
new file mode 100644
index 0000000..ccbe845
--- /dev/null
+++ b/core-storage/src/main/java/org/apache/kylin/storage/util/SizeOfUtil.java
@@ -0,0 +1,21 @@
+package org.apache.kylin.storage.util;
+
+import net.sf.ehcache.pool.sizeof.ReflectionSizeOf;
+
+/**
+ */
+public final class SizeOfUtil {
+
+ private SizeOfUtil() {
+ }
+
+ private static final ReflectionSizeOf DEFAULT_SIZE_OF = new ReflectionSizeOf();
+
+ public static final long deepSizeOf(Object obj) {
+ return DEFAULT_SIZE_OF.deepSizeOf(Integer.MAX_VALUE, true, obj).getCalculated();
+ }
+
+ public static final long sizeOf(Object obj) {
+ return DEFAULT_SIZE_OF.sizeOf(obj);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
new file mode 100644
index 0000000..d2c8e11
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/DynamicCacheTest.java
@@ -0,0 +1,144 @@
+package org.apache.kylin.storage.cache;
+
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.common.util.IdentityUtils;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.storage.ICachableStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+import com.google.common.collect.Ranges;
+
+/**
+ */
+public class DynamicCacheTest {
+
+ class TsOnlyTuple implements ITuple {
+ private TblColRef partitionCol;
+ private String tsStr;
+
+ public TsOnlyTuple(TblColRef partitionCol, String tsStr) {
+ this.partitionCol = partitionCol;
+ this.tsStr = tsStr;
+ }
+
+ @Override
+ public List<String> getAllFields() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public List<TblColRef> getAllColumns() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Object[] getAllValues() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public ITuple makeCopy() {
+ return new TsOnlyTuple(this.partitionCol, this.tsStr);
+ }
+
+ @Override
+ public Object getValue(TblColRef col) {
+ if (col.equals(partitionCol)) {
+ return Tuple.dateToEpicDays(this.tsStr);
+ } else {
+ throw new NotImplementedException();
+ }
+ }
+ }
+
+ @Test
+ public void basicTest() {
+
+ final StorageContext context = new StorageContext();
+ final List<TblColRef> groups = StorageMockUtils.buildGroups();
+ final TblColRef partitionCol = groups.get(0);
+ final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
+
+ SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", null, null, Lists.<TblColRef> newArrayList(), groups, Lists.newArrayList(partitionCol), Lists.<TblColRef> newArrayList(), aggregations);
+
+ ITuple aTuple = new TsOnlyTuple(partitionCol, "2011-02-01");
+ ITuple bTuple = new TsOnlyTuple(partitionCol, "2012-02-01");
+ final List<ITuple> allTuples = Lists.newArrayList(aTuple, bTuple);
+
+ //counts for verifying
+ final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
+ final List<Integer> returnedRowPerSearch = Lists.newArrayList();
+
+ CacheFledgedDynamicStorageEngine dynamicCache = new CacheFledgedDynamicStorageEngine(new ICachableStorageQuery() {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ Range<Long> tsRagneInQuery = TsConditionExtractor.extractTsCondition(partitionCol, sqlDigest.filter);
+ List<ITuple> ret = Lists.newArrayList();
+ for (ITuple tuple : allTuples) {
+ if (tsRagneInQuery.contains(Tuple.getTs(tuple, partitionCol))) {
+ ret.add(tuple);
+ }
+ }
+
+ underlyingSEHitCount.incrementAndGet();
+ returnedRowPerSearch.add(ret.size());
+
+ return new SimpleTupleIterator(ret.iterator());
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return true;
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return Ranges.greaterThan(DateFormat.stringToMillis("2011-02-01"));
+ }
+
+ @Override
+ public String getStorageUUID() {
+ return "111ca32a-a33e-4b69-12aa-0bb8b1f8c191";
+ }
+ }, partitionCol);
+
+ sqlDigest.filter = StorageMockUtils.buildTs2010Filter(groups.get(0));
+ ITupleIterator firstIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
+ while (firstIterator.hasNext()) {
+ firstResults.put(firstIterator.next(), null);
+ }
+ firstIterator.close();
+
+ sqlDigest.filter = StorageMockUtils.buildTs2011Filter(groups.get(0));
+ ITupleIterator secondIterator = dynamicCache.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
+ while (secondIterator.hasNext()) {
+ secondResults.put(secondIterator.next(), null);
+ }
+ secondIterator.close();
+
+ Assert.assertEquals(2, firstResults.size());
+ IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
+ Assert.assertEquals(2, underlyingSEHitCount.get());
+ Assert.assertEquals(new Integer(2), returnedRowPerSearch.get(0));
+ Assert.assertEquals(new Integer(1), returnedRowPerSearch.get(1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
new file mode 100644
index 0000000..4422fbd
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/EhcacheTest.java
@@ -0,0 +1,70 @@
+package org.apache.kylin.storage.cache;
+
+import java.util.List;
+
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.Configuration;
+import net.sf.ehcache.config.PersistenceConfiguration;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+@Ignore("trial for dev")
+public class EhcacheTest {
+
+ @Test
+ public void basicTest() throws InterruptedException {
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+
+ Configuration conf = new Configuration();
+ conf.setMaxBytesLocalHeap("100M");
+ CacheManager cacheManager = CacheManager.create(conf);
+
+ //Create a Cache specifying its configuration.
+ Cache testCache = //Create a Cache specifying its configuration.
+ new Cache(new CacheConfiguration("test", 0).//
+ memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU).//
+ eternal(false).//
+ timeToIdleSeconds(86400).//
+ diskExpiryThreadIntervalSeconds(0).//
+ //maxBytesLocalHeap(1000, MemoryUnit.MEGABYTES).//
+ persistence(new PersistenceConfiguration().strategy(PersistenceConfiguration.Strategy.NONE)));
+
+ cacheManager.addCache(testCache);
+
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ byte[] blob = new byte[(1024 * 40 * 1024)];//400M
+
+ List<String> manyObjects = Lists.newArrayList();
+ for (int i = 0; i < 10000; i++) {
+ manyObjects.add(new String("" + i));
+ }
+ testCache.put(new Element("0", manyObjects));
+
+ testCache.put(new Element("1", blob));
+ System.out.println(testCache.get("1") == null);
+ System.out.println(testCache.getSize());
+ System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ testCache.put(new Element("2", blob));
+ System.out.println(testCache.get("1") == null);
+ System.out.println(testCache.getSize());
+ System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+ testCache.put(new Element("3", blob));
+ System.out.println(testCache.get("1") == null);
+ System.out.println(testCache.get("2") == null);
+ System.out.println(testCache.get("3") == null);
+ System.out.println(testCache.getSize());
+ System.out.println(testCache.getStatistics().getLocalHeapSizeInBytes());
+ System.out.println("runtime used memory: " + (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1024 / 1024 + "M");
+
+ cacheManager.shutdown();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
new file mode 100644
index 0000000..d81331e
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StaticCacheTest.java
@@ -0,0 +1,97 @@
+package org.apache.kylin.storage.cache;
+
+import java.util.Collections;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.kylin.common.util.IdentityUtils;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.realization.SQLDigest;
+import org.apache.kylin.metadata.tuple.ITuple;
+import org.apache.kylin.metadata.tuple.ITupleIterator;
+import org.apache.kylin.metadata.tuple.SimpleTupleIterator;
+import org.apache.kylin.storage.ICachableStorageQuery;
+import org.apache.kylin.storage.StorageContext;
+import org.apache.kylin.storage.tuple.Tuple;
+import org.apache.kylin.storage.tuple.TupleInfo;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Range;
+
+/**
+ */
+public class StaticCacheTest {
+
+ @Test
+ public void basicTest() {
+
+ final StorageContext context = new StorageContext();
+ final List<TblColRef> groups = StorageMockUtils.buildGroups();
+ final List<FunctionDesc> aggregations = StorageMockUtils.buildAggregations();
+ final TupleFilter filter = StorageMockUtils.buildFilter1(groups.get(0));
+ final SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
+ final TupleInfo tupleInfo = StorageMockUtils.newTupleInfo(groups, aggregations);
+
+ final List<ITuple> ret = Lists.newArrayList();
+ ret.add(new Tuple(tupleInfo));
+ ret.add(new Tuple(tupleInfo));
+ ret.add(new Tuple(tupleInfo));
+
+ final AtomicInteger underlyingSEHitCount = new AtomicInteger(0);
+
+ CacheFledgedStaticStorageEngine cacheFledgedStaticStorageEngine = new CacheFledgedStaticStorageEngine(new ICachableStorageQuery() {
+ @Override
+ public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
+ underlyingSEHitCount.incrementAndGet();
+ return new SimpleTupleIterator(ret.iterator());
+ }
+
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+
+ @Override
+ public Range<Long> getVolatilePeriod() {
+ return null;
+ }
+
+ @Override
+ public String getStorageUUID() {
+ return "111ca32a-a33e-4b69-12aa-0bb8b1f8c092";
+ }
+ });
+
+ ITupleIterator firstIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> firstResults = new IdentityHashMap<>();
+ while (firstIterator.hasNext()) {
+ firstResults.put(firstIterator.next(), null);
+ }
+ firstIterator.close();
+
+ ITupleIterator secondIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> secondResults = new IdentityHashMap<>();
+ while (secondIterator.hasNext()) {
+ secondResults.put(secondIterator.next(), null);
+ }
+ secondIterator.close();
+
+ ITupleIterator thirdIterator = cacheFledgedStaticStorageEngine.search(context, sqlDigest, tupleInfo);
+ IdentityHashMap<ITuple, Void> thirdResults = new IdentityHashMap<>();
+ while (thirdIterator.hasNext()) {
+ thirdResults.put(thirdIterator.next(), null);
+ }
+ thirdIterator.close();
+
+ Assert.assertEquals(3, firstResults.size());
+ IdentityUtils.collectionReferenceEquals(firstResults.keySet(), secondResults.keySet());
+ IdentityUtils.collectionReferenceEquals(thirdResults.keySet(), secondResults.keySet());
+
+ Assert.assertEquals(1, underlyingSEHitCount.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
new file mode 100644
index 0000000..1bf8e6a
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/cache/StorageMockUtils.java
@@ -0,0 +1,139 @@
+package org.apache.kylin.storage.cache;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.kylin.metadata.filter.ColumnTupleFilter;
+import org.apache.kylin.metadata.filter.CompareTupleFilter;
+import org.apache.kylin.metadata.filter.ConstantTupleFilter;
+import org.apache.kylin.metadata.filter.LogicalTupleFilter;
+import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.ColumnDesc;
+import org.apache.kylin.metadata.model.FunctionDesc;
+import org.apache.kylin.metadata.model.ParameterDesc;
+import org.apache.kylin.metadata.model.TableDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.storage.tuple.TupleInfo;
+
+import com.google.common.collect.ImmutableList;
+
+/**
+ */
+public class StorageMockUtils {
+ public static TupleInfo newTupleInfo(List<TblColRef> groups, List<FunctionDesc> aggregations) {
+ TupleInfo info = new TupleInfo();
+ int idx = 0;
+
+ for (TblColRef col : groups) {
+ info.setField(col.getName(), col, idx++);
+ }
+
+ TableDesc sourceTable = groups.get(0).getColumnDesc().getTable();
+ for (FunctionDesc func : aggregations) {
+ TblColRef col = new TblColRef(func.newFakeRewriteColumn(sourceTable));
+ info.setField(col.getName(), col, idx++);
+ }
+
+ return info;
+ }
+
+ public static List<TblColRef> buildGroups() {
+ List<TblColRef> groups = new ArrayList<TblColRef>();
+
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ ColumnDesc c1 = ColumnDesc.mockup(t1, 2, "CAL_DT", "date");
+ TblColRef cf1 = new TblColRef(c1);
+ groups.add(cf1);
+
+ TableDesc t2 = TableDesc.mockup("DEFAULT.TEST_CATEGORY_GROUPINGS");
+ ColumnDesc c2 = ColumnDesc.mockup(t2, 14, "META_CATEG_NAME", "string");
+ TblColRef cf2 = new TblColRef(c2);
+ groups.add(cf2);
+
+ return groups;
+ }
+
+ public static List<FunctionDesc> buildAggregations() {
+ List<FunctionDesc> functions = new ArrayList<FunctionDesc>();
+
+ TableDesc t1 = TableDesc.mockup("DEFAULT.TEST_KYLIN_FACT");
+ TblColRef priceCol = new TblColRef(ColumnDesc.mockup(t1, 7, "PRICE", "decimal(19,4)"));
+ TblColRef sellerCol = new TblColRef(ColumnDesc.mockup(t1, 9, "SELLER_ID", "bigint"));
+
+ FunctionDesc f1 = new FunctionDesc();
+ f1.setExpression("SUM");
+ ParameterDesc p1 = new ParameterDesc();
+ p1.setType("column");
+ p1.setValue("PRICE");
+ p1.setColRefs(ImmutableList.of(priceCol));
+ f1.setParameter(p1);
+ functions.add(f1);
+
+ FunctionDesc f2 = new FunctionDesc();
+ f2.setExpression("COUNT_DISTINCT");
+ ParameterDesc p2 = new ParameterDesc();
+ p2.setType("column");
+ p2.setValue("SELLER_ID");
+ p2.setColRefs(ImmutableList.of(sellerCol));
+ f2.setParameter(p2);
+ functions.add(f2);
+
+ return functions;
+ }
+
+ public static CompareTupleFilter buildTs2010Filter(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter1);
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2010-01-01");
+ compareFilter.addChild(constantFilter1);
+ return compareFilter;
+ }
+
+ public static CompareTupleFilter buildTs2011Filter(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.GT);
+ ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter1);
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2011-01-01");
+ compareFilter.addChild(constantFilter1);
+ return compareFilter;
+ }
+
+ public static CompareTupleFilter buildFilter1(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter1 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter1);
+ ConstantTupleFilter constantFilter1 = new ConstantTupleFilter("2012-05-23");
+ compareFilter.addChild(constantFilter1);
+ return compareFilter;
+ }
+
+ public static CompareTupleFilter buildFilter2(TblColRef column) {
+ CompareTupleFilter compareFilter = new CompareTupleFilter(TupleFilter.FilterOperatorEnum.EQ);
+ ColumnTupleFilter columnFilter2 = new ColumnTupleFilter(column);
+ compareFilter.addChild(columnFilter2);
+ ConstantTupleFilter constantFilter2 = new ConstantTupleFilter("ClothinShoes & Accessories");
+ compareFilter.addChild(constantFilter2);
+ return compareFilter;
+ }
+
+ @SuppressWarnings("unused")
+ public static TupleFilter buildAndFilter(List<TblColRef> columns) {
+ CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
+ CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
+ LogicalTupleFilter andFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.AND);
+ andFilter.addChild(compareFilter1);
+ andFilter.addChild(compareFilter2);
+ return andFilter;
+ }
+
+ @SuppressWarnings("unused")
+ public static TupleFilter buildOrFilter(List<TblColRef> columns) {
+ CompareTupleFilter compareFilter1 = buildFilter1(columns.get(0));
+ CompareTupleFilter compareFilter2 = buildFilter2(columns.get(1));
+ LogicalTupleFilter logicFilter = new LogicalTupleFilter(TupleFilter.FilterOperatorEnum.OR);
+ logicFilter.addChild(compareFilter1);
+ logicFilter.addChild(compareFilter2);
+ return logicFilter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java b/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
new file mode 100644
index 0000000..12fdd53
--- /dev/null
+++ b/core-storage/src/test/java/org/apache/kylin/storage/hybrid/HybridManagerTest.java
@@ -0,0 +1,43 @@
+package org.apache.kylin.storage.hybrid;
+
+import org.apache.kylin.common.util.JsonUtil;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeInstance;
+import org.apache.kylin.metadata.realization.IRealization;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ */
+public class HybridManagerTest extends LocalFileMetadataTestCase {
+
+ @Before
+ public void setUp() throws Exception {
+ this.createTestMetadata();
+ }
+
+ @After
+ public void after() throws Exception {
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testBasics() throws Exception {
+ HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_ready");
+ System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance));
+
+ IRealization[] realizations = hybridInstance.getRealizations();
+ Assert.assertEquals(realizations.length, 2);
+
+ IRealization lastReal = hybridInstance.getLatestRealization();
+ Assert.assertTrue(lastReal instanceof CubeInstance);
+ Assert.assertEquals(lastReal.getName(), "test_kylin_cube_with_slr_ready_2_segments");
+
+ }
+
+ public HybridManager getHybridManager() {
+ return HybridManager.getInstance(getTestConfig());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
index 95d3551..4c8b652 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/MRUtil.java
@@ -12,7 +12,7 @@ import org.apache.kylin.engine.mr.IMROutput2.IMRBatchMergeOutputSide2;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.TableSourceFactory;
-import org.apache.kylin.storage.StorageFactory2;
+import org.apache.kylin.storage.StorageFactory;
public class MRUtil {
@@ -33,23 +33,23 @@ public class MRUtil {
}
public static IMRBatchCubingOutputSide getBatchCubingOutputSide(CubeSegment seg) {
- return StorageFactory2.createEngineAdapter(seg, IMROutput.class).getBatchCubingOutputSide(seg);
+ return StorageFactory.createEngineAdapter(seg, IMROutput.class).getBatchCubingOutputSide(seg);
}
public static IMRBatchMergeOutputSide getBatchMergeOutputSide(CubeSegment seg) {
- return StorageFactory2.createEngineAdapter(seg, IMROutput.class).getBatchMergeOutputSide(seg);
+ return StorageFactory.createEngineAdapter(seg, IMROutput.class).getBatchMergeOutputSide(seg);
}
public static IMRBatchCubingOutputSide2 getBatchCubingOutputSide2(CubeSegment seg) {
- return StorageFactory2.createEngineAdapter(seg, IMROutput2.class).getBatchCubingOutputSide(seg);
+ return StorageFactory.createEngineAdapter(seg, IMROutput2.class).getBatchCubingOutputSide(seg);
}
public static IMRBatchMergeInputSide2 getBatchMergeInputSide2(CubeSegment seg) {
- return StorageFactory2.createEngineAdapter(seg, IMROutput2.class).getBatchMergeInputSide(seg);
+ return StorageFactory.createEngineAdapter(seg, IMROutput2.class).getBatchMergeInputSide(seg);
}
public static IMRBatchMergeOutputSide2 getBatchMergeOutputSide2(CubeSegment seg) {
- return StorageFactory2.createEngineAdapter(seg, IMROutput2.class).getBatchMergeOutputSide(seg);
+ return StorageFactory.createEngineAdapter(seg, IMROutput2.class).getBatchMergeOutputSide(seg);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
index 0317d1f..2fd5229 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCubing.java
@@ -59,7 +59,7 @@ import org.apache.kylin.job.common.OptionsHelper;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.apache.kylin.storage.hbase.steps.CreateHTableJob;
import org.apache.kylin.storage.hbase.steps.CubeHTableUtil;
import org.apache.spark.SparkConf;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java b/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
index d24a72d..122758a 100644
--- a/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
+++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/BuildCubeWithSparkTest.java
@@ -27,9 +27,6 @@ import java.util.TimeZone;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
@@ -37,8 +34,6 @@ import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
-import org.apache.kylin.engine.mr.CubingJob;
-import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.job.DeployUtil;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.job.execution.AbstractExecutable;
@@ -47,9 +42,7 @@ import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.lock.MockJobLock;
import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
-import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/pom.xml
----------------------------------------------------------------------
diff --git a/job/pom.xml b/job/pom.xml
index 1fa9d70..9342de9 100644
--- a/job/pom.xml
+++ b/job/pom.xml
@@ -48,27 +48,25 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-storage</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
+ <artifactId>kylin-source-hive</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-source-hive</artifactId>
+ <artifactId>kylin-invertedindex</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-invertedindex</artifactId>
+ <artifactId>kylin-streaming</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-streaming</artifactId>
+ <artifactId>kylin-storage-hbase</artifactId>
<version>${project.parent.version}</version>
</dependency>
-
+
<dependency>
<groupId>commons-daemon</groupId>
<artifactId>commons-daemon</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
index d3f6d68..d7b088b 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IIBulkLoadJob.java
@@ -31,7 +31,7 @@ import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
/**
* @author ysong1
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
index 08b7a46..90f85fa 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHFileJob.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.ToolRunner;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.engine.mr.common.AbstractHadoopJob;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
index 6149d27..752c3be 100644
--- a/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
+++ b/job/src/main/java/org/apache/kylin/job/hadoop/invertedindex/IICreateHTableJob.java
@@ -38,7 +38,7 @@ import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.invertedindex.model.IIKeyValueCodec;
import org.apache.kylin.metadata.realization.IRealizationConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI;
import org.apache.kylin.storage.hbase.util.LZOSupportnessChecker;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
index 188da6e..38787a8 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
@@ -42,24 +42,19 @@ import org.apache.kylin.cube.inmemcubing.InMemCubeBuilder;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.util.CubingUtils;
import org.apache.kylin.dict.Dictionary;
-import org.apache.kylin.dict.DictionaryInfo;
-import org.apache.kylin.dict.DictionaryManager;
import org.apache.kylin.engine.mr.HadoopUtil;
import org.apache.kylin.engine.mr.common.BatchConstants;
import org.apache.kylin.engine.mr.steps.FactDistinctColumnsReducer;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.source.ReadableTable.TableSignature;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.HBaseCuboidWriter;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseCuboidWriter;
import org.apache.kylin.storage.hbase.steps.CubeHTableUtil;
import org.apache.kylin.streaming.MicroStreamBatch;
import org.apache.kylin.streaming.MicroStreamBatchConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.Maps;
-
/**
*/
public class CubeStreamConsumer implements MicroStreamBatchConsumer {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
index fb6911d..0811451 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/StreamingBootstrap.java
@@ -66,7 +66,7 @@ import org.apache.kylin.invertedindex.model.IIDesc;
import org.apache.kylin.job.monitor.StreamingMonitor;
import org.apache.kylin.metadata.model.IntermediateColumnDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.apache.kylin.streaming.JsonStreamParser;
import org.apache.kylin.streaming.KafkaClusterConfig;
import org.apache.kylin.streaming.KafkaConsumer;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java b/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
index 20bccfe..7906ada 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/GridTableHBaseBenchmark.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
import org.apache.kylin.common.util.Bytes;
import org.apache.kylin.common.util.Pair;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import com.google.common.collect.Lists;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
index b652c2f..2156574 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithEngineTest.java
@@ -48,7 +48,7 @@ import org.apache.kylin.job.execution.DefaultChainedExecutable;
import org.apache.kylin.job.execution.ExecutableState;
import org.apache.kylin.job.impl.threadpool.DefaultScheduler;
import org.apache.kylin.job.manager.ExecutableManager;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
index 092b1fb..d3f84a0 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildCubeWithStreamTest.java
@@ -46,7 +46,7 @@ import org.apache.kylin.common.util.DateFormat;
import org.apache.kylin.job.hadoop.cube.StorageCleanupJob;
import org.apache.kylin.job.streaming.BootstrapConfig;
import org.apache.kylin.job.streaming.StreamingBootstrap;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.apache.kylin.streaming.StreamingConfig;
import org.apache.kylin.streaming.StreamingManager;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
index 645f7c0..1cf1a0f 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithEngineTest.java
@@ -21,7 +21,6 @@ package org.apache.kylin.job;
import static org.junit.Assert.assertEquals;
import java.io.File;
-import java.io.IOException;
import java.lang.reflect.Method;
import java.text.SimpleDateFormat;
import java.util.List;
@@ -50,7 +49,7 @@ import org.apache.kylin.job.invertedindex.IIJob;
import org.apache.kylin.job.invertedindex.IIJobBuilder;
import org.apache.kylin.job.manager.ExecutableManager;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.apache.kylin.storage.hbase.util.ZookeeperJobLock;
import org.junit.After;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 0c57e4b..360ad8b 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -68,7 +68,7 @@ import org.apache.kylin.job.hadoop.invertedindex.IICreateHTableJob;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.source.hive.HiveTableReader;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.apache.kylin.streaming.StreamBuilder;
import org.apache.kylin.streaming.StreamMessage;
import org.apache.kylin.streaming.invertedindex.IIStreamConsumer;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java b/job/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
index d3094d5..7f12069 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployLocalMetaToRemoteTest.java
@@ -25,7 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/DeployUtil.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/DeployUtil.java b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
index 4f0ec5c..5e95983 100644
--- a/job/src/test/java/org/apache/kylin/job/DeployUtil.java
+++ b/job/src/test/java/org/apache/kylin/job/DeployUtil.java
@@ -86,32 +86,6 @@ public class DeployUtil {
config().overrideSparkJobJarPath(getSparkJobJarFile().getAbsolutePath());
}
- public static void deployJobJars() throws IOException {
- File originalJobJar = getJobJarFile();
- File originalCoprocessorJar = getCoprocessorJarFile();
-
- String jobJarPath = config().getKylinJobJarPath();
- if (StringUtils.isEmpty(jobJarPath)) {
- throw new RuntimeException("deployJobJars cannot find job jar");
- }
-
- File targetJobJar = new File(jobJarPath);
- File jobJarRenamedAsTarget = new File(originalJobJar.getParentFile(), targetJobJar.getName());
- if (originalJobJar.equals(jobJarRenamedAsTarget) == false) {
- FileUtils.copyFile(originalJobJar, jobJarRenamedAsTarget);
- }
-
- File targetCoprocessorJar = new File(config().getCoprocessorLocalJar());
- File coprocessorJarRenamedAsTarget = new File(originalCoprocessorJar.getParentFile(), targetCoprocessorJar.getName());
- if (originalCoprocessorJar.equals(coprocessorJarRenamedAsTarget) == false) {
- FileUtils.copyFile(originalCoprocessorJar, coprocessorJarRenamedAsTarget);
- }
-
- CliCommandExecutor cmdExec = config().getCliCommandExecutor();
- cmdExec.copyFile(jobJarRenamedAsTarget.getAbsolutePath(), targetJobJar.getParent());
- cmdExec.copyFile(coprocessorJarRenamedAsTarget.getAbsolutePath(), targetCoprocessorJar.getParent());
- }
-
private static String getPomVersion() {
try {
MavenXpp3Reader pomReader = new MavenXpp3Reader();
@@ -127,7 +101,7 @@ public class DeployUtil {
}
private static File getCoprocessorJarFile() {
- return new File("../storage/target", "kylin-storage-" + getPomVersion() + "-coprocessor.jar");
+ return new File("../storage-hbase/target", "kylin-storage-hbase-" + getPomVersion() + "-coprocessor.jar");
}
private static File getSparkJobJarFile() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
index d30b829..2d8bb05 100644
--- a/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
+++ b/job/src/test/java/org/apache/kylin/job/ExportHBaseData.java
@@ -31,8 +31,8 @@ import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.CliCommandExecutor;
import org.apache.kylin.common.util.SSHClient;
import org.apache.kylin.engine.mr.common.BatchConstants;
-import org.apache.kylin.storage.hbase.HBaseConnection;
-import org.apache.kylin.storage.hbase.HBaseMiniclusterHelper;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseMiniclusterHelper;
public class ExportHBaseData {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
index 37362d5..6a615cb 100644
--- a/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
+++ b/job/src/test/java/org/apache/kylin/job/ITKafkaBasedIIStreamBuilderTest.java
@@ -40,7 +40,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.job.streaming.StreamingBootstrap;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
index 811948b..dcd460b 100644
--- a/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
+++ b/job/src/test/java/org/apache/kylin/job/hadoop/invertedindex/IITest.java
@@ -33,14 +33,14 @@ import org.apache.kylin.metadata.filter.TupleFilter;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.ParameterDesc;
import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorFilter;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorProjector;
-import org.apache.kylin.storage.hbase.coprocessor.CoprocessorRowType;
-import org.apache.kylin.storage.hbase.coprocessor.FilterDecorator;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.ClearTextDictionary;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.EndpointAggregators;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.IIEndpoint;
-import org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorFilter;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorProjector;
+import org.apache.kylin.storage.hbase.common.coprocessor.CoprocessorRowType;
+import org.apache.kylin.storage.hbase.common.coprocessor.FilterDecorator;
+import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.ClearTextDictionary;
+import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.EndpointAggregators;
+import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.IIEndpoint;
+import org.apache.kylin.storage.hbase.ii.coprocessor.endpoint.generated.IIProtos;
import org.apache.kylin.streaming.MicroStreamBatch;
import org.apache.kylin.streaming.ParsedStreamMessage;
import org.apache.kylin.streaming.StreamMessage;
@@ -230,7 +230,7 @@ public class IITest extends LocalFileMetadataTestCase {
Assert.assertEquals(2, response.getRowsList().size());
System.out.println(response.getRowsList().size());
Set<String> answers = Sets.newHashSet("120.4747", "26.8551");
- for (org.apache.kylin.storage.hbase.coprocessor.endpoint.generated.IIProtos.IIResponseInternal.IIRow responseRow : response.getRowsList()) {
+ for (IIProtos.IIResponseInternal.IIRow responseRow : response.getRowsList()) {
byte[] measuresBytes = responseRow.getMeasures().toByteArray();
List<Object> metrics = aggregators.deserializeMetricValues(measuresBytes, 0);
Assert.assertTrue(answers.contains(metrics.get(0)));
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamConsumerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamConsumerTest.java b/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamConsumerTest.java
index ac65798..be4fa26 100644
--- a/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamConsumerTest.java
+++ b/job/src/test/java/org/apache/kylin/job/streaming/CubeStreamConsumerTest.java
@@ -21,7 +21,7 @@ import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.cube.CubeUpdate;
import org.apache.kylin.job.DeployUtil;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.apache.kylin.streaming.StreamBuilder;
import org.apache.kylin.streaming.StreamMessage;
import org.junit.Before;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java b/job/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
index c2cf383..8218d51 100644
--- a/job/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
+++ b/job/src/test/java/org/apache/kylin/source/hive/ITHiveSourceTableLoaderTest.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.util.Set;
import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java b/job/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
index cd4cdc6..57c0be3 100644
--- a/job/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
+++ b/job/src/test/java/org/apache/kylin/source/hive/ITHiveTableReaderTest.java
@@ -20,7 +20,7 @@ package org.apache.kylin.source.hive;
import java.io.IOException;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/job/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java b/job/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
index 2e47953..3d4fbba 100644
--- a/job/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
+++ b/job/src/test/java/org/apache/kylin/source/hive/ITSnapshotManagerTest.java
@@ -28,7 +28,7 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.ReadableTable;
import org.apache.kylin.source.ReadableTable.TableReader;
import org.apache.kylin.source.TableSourceFactory;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4818c19..17d09f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -596,7 +596,6 @@
<module>source-kafka</module>
<module>storage-hbase</module>
<module>job</module>
- <module>storage</module>
<module>query</module>
<module>server</module>
<module>jdbc</module>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/query/pom.xml
----------------------------------------------------------------------
diff --git a/query/pom.xml b/query/pom.xml
index f8a5dab..39d8faa 100644
--- a/query/pom.xml
+++ b/query/pom.xml
@@ -43,7 +43,7 @@
</dependency>
<dependency>
<groupId>org.apache.kylin</groupId>
- <artifactId>kylin-storage</artifactId>
+ <artifactId>kylin-core-storage</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
@@ -79,6 +79,12 @@
<dependency>
<groupId>org.apache.kylin</groupId>
<artifactId>kylin-storage-hbase</artifactId>
+ <scope>test</scope>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kylin</groupId>
+ <artifactId>kylin-storage-hbase</artifactId>
<type>test-jar</type>
<scope>test</scope>
<version>${project.parent.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
index bea0e36..c6c1ce1 100644
--- a/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
+++ b/query/src/main/java/org/apache/kylin/query/enumerator/OLAPEnumerator.java
@@ -24,6 +24,7 @@ import java.util.Properties;
import org.apache.calcite.DataContext;
import org.apache.calcite.jdbc.CalciteConnection;
import org.apache.calcite.linq4j.Enumerator;
+import org.apache.kylin.dict.DictCodeSystem;
import org.apache.kylin.metadata.filter.CompareTupleFilter;
import org.apache.kylin.metadata.filter.TimeConditionLiteralsReplacer;
import org.apache.kylin.metadata.filter.TupleFilter;
@@ -32,8 +33,7 @@ import org.apache.kylin.metadata.tuple.ITuple;
import org.apache.kylin.metadata.tuple.ITupleIterator;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.storage.IStorageQuery;
-import org.apache.kylin.storage.StorageFactory;
-import org.apache.kylin.storage.hbase.coprocessor.DictCodeSystem;
+import org.apache.kylin.storage.StorageQueryFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,7 +111,7 @@ public class OLAPEnumerator implements Enumerator<Object[]> {
olapContext.resetSQLDigest();
// query storage engine
- IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
+ IStorageQuery storageEngine = StorageQueryFactory.createQuery(olapContext.realization);
ITupleIterator iterator = storageEngine.search(olapContext.storageContext, olapContext.getSQLDigest(), olapContext.returnTupleInfo);
if (logger.isDebugEnabled()) {
logger.debug("return TupleIterator...");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java b/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
index 2106c06..bbff87d 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITCombinationTest.java
@@ -22,7 +22,7 @@ import java.sql.SQLException;
import java.util.Arrays;
import java.util.Collection;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.runner.RunWith;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
----------------------------------------------------------------------
diff --git a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
index ce8d3ee..a08d595 100644
--- a/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
+++ b/query/src/test/java/org/apache/kylin/query/test/ITKylinQueryTest.java
@@ -31,8 +31,8 @@ import org.apache.kylin.metadata.project.ProjectInstance;
import org.apache.kylin.query.enumerator.OLAPQuery;
import org.apache.kylin.query.relnode.OLAPContext;
import org.apache.kylin.query.schema.OLAPSchemaFactory;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.dbunit.database.DatabaseConnection;
import org.dbunit.database.IDatabaseConnection;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/script/prepare_libs.sh b/script/prepare_libs.sh
index fe0bc1b..7eb857e 100755
--- a/script/prepare_libs.sh
+++ b/script/prepare_libs.sh
@@ -11,7 +11,7 @@ echo "copy lib file"
rm -rf lib
mkdir lib
cp job/target/kylin-job-${version}-job.jar lib/kylin-job-${version}.jar
-cp storage/target/kylin-storage-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
+cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar lib/kylin-coprocessor-${version}.jar
cp jdbc/target/kylin-jdbc-${version}.jar lib/kylin-jdbc-${version}.jar
# Copied file becomes 000 for some env (e.g. my Cygwin)
chmod 644 lib/kylin-job-${version}.jar
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
index e0f7487..7484742 100644
--- a/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
+++ b/server/src/main/java/org/apache/kylin/rest/DebugTomcat.java
@@ -79,7 +79,7 @@ public class DebugTomcat {
if (devJobJar != null) {
conf.overrideMRJobJarPath(devJobJar.getAbsolutePath());
}
- File devCoprocessorJar = findFile("../storage/target", "kylin-storage-.*-SNAPSHOT-coprocessor.jar");
+ File devCoprocessorJar = findFile("../storage-hbase/target", "kylin-storage-hbase-.*-SNAPSHOT-coprocessor.jar");
if (devCoprocessorJar != null) {
conf.overrideCoprocessorLocalJar(devCoprocessorJar.getAbsolutePath());
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 14c231b..b8ec976 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -50,7 +50,7 @@ import org.apache.kylin.rest.response.GeneralResponse;
import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.service.CubeService;
import org.apache.kylin.rest.service.JobService;
-import org.apache.kylin.storage.hbase.coprocessor.observer.ObserverEnabler;
+import org.apache.kylin.storage.hbase.cube.v1.coprocessor.observer.ObserverEnabler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
index 8cc1ea1..b73118a 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/QueryController.java
@@ -49,7 +49,7 @@ import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.service.QueryService;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.storage.cache.AbstractCacheFledgedStorageEngine;
-import org.apache.kylin.storage.hbase.ScanOutOfLimitException;
+import org.apache.kylin.storage.exception.ScanOutOfLimitException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
index 0cf3822..21f364a 100644
--- a/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
+++ b/server/src/main/java/org/apache/kylin/rest/security/RealAclHBaseStorage.java
@@ -6,7 +6,7 @@ import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.rest.service.AclService;
import org.apache.kylin.rest.service.UserService;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.h2.util.StringUtils;
/**
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index b5365b5..bf339cc 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -62,7 +62,7 @@ import org.apache.kylin.rest.response.HBaseResponse;
import org.apache.kylin.rest.response.MetricsResponse;
import org.apache.kylin.rest.security.AclPermission;
import org.apache.kylin.source.hive.HiveSourceTableLoader;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.apache.kylin.storage.hbase.util.HBaseRegionSizeCalculator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
index 82ab387..4f47790 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/QueryService.java
@@ -64,7 +64,7 @@ import org.apache.kylin.rest.request.SQLRequest;
import org.apache.kylin.rest.response.SQLResponse;
import org.apache.kylin.rest.util.QueryUtil;
import org.apache.kylin.rest.util.Serializer;
-import org.apache.kylin.storage.hbase.HBaseConnection;
+import org.apache.kylin.storage.hbase.steps.HBaseConnection;
import org.h2.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
----------------------------------------------------------------------
diff --git a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
index 63b05a9..1546576 100644
--- a/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
+++ b/server/src/test/java/org/apache/kylin/jdbc/ITJDBCDriverTest.java
@@ -29,7 +29,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.commons.io.FileUtils;
-import org.apache.kylin.storage.hbase.HBaseMetadataTestCase;
+import org.apache.kylin.storage.hbase.steps.HBaseMetadataTestCase;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.webapp.WebAppContext;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/11611cc2/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
index a9e95a4..f9f87c6 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveTableSource.java
@@ -23,6 +23,8 @@ import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.source.ITableSource;
import org.apache.kylin.source.ReadableTable;
+@SuppressWarnings("unused")
+//used by reflection
public class HiveTableSource implements ITableSource {
@SuppressWarnings("unchecked")